imsciences 0.9.6.3__py3-none-any.whl → 0.9.6.4__py3-none-any.whl

imsciences/pull.py

@@ -380,265 +380,368 @@ class datapull:
     ###############################################################  Seasonality  ##########################################################################
 
     def pull_seasonality(self, week_commencing, start_date, countries):
-        # ---------------------------------------------------------------------
-        # 0. Setup: dictionary for 'week_commencing' to Python weekday() integer
-        # ---------------------------------------------------------------------
-        day_dict = {"mon": 0, "tue": 1, "wed": 2, "thu": 3, "fri": 4, "sat": 5, "sun": 6}
-
-        # ---------------------------------------------------------------------
-        # 1. Create daily date range from start_date to today
-        # ---------------------------------------------------------------------
-        date_range = pd.date_range(
-            start=pd.to_datetime(start_date), 
-            end=datetime.today(), 
-            freq="D"
-        )
-        df_daily = pd.DataFrame(date_range, columns=["Date"])
+            """
+            Generates a DataFrame with weekly seasonality features.
 
-        # ---------------------------------------------------------------------
-        # 1.1 Identify "week_start" for each daily row, based on week_commencing
-        # ---------------------------------------------------------------------
-        df_daily['week_start'] = df_daily["Date"].apply(
-            lambda x: x - pd.Timedelta(days=(x.weekday() - day_dict[week_commencing]) % 7)
-        )
+            Args:
+                week_commencing (str): The starting day of the week ('mon', 'tue', ..., 'sun').
+                start_date (str): The start date in 'YYYY-MM-DD' format.
+                countries (list): A list of country codes (e.g., ['GB', 'US']) for holidays.
 
-        # ---------------------------------------------------------------------
-        # 2. Build a weekly index (df_weekly_start) with dummy columns
-        # ---------------------------------------------------------------------
-        df_weekly_start = df_daily[['week_start']].drop_duplicates().reset_index(drop=True)
-        df_weekly_start.rename(columns={'week_start': "Date"}, inplace=True)
-        
-        # Set index to weekly "start of week"
-        df_weekly_start.index = np.arange(1, len(df_weekly_start) + 1)
-        df_weekly_start.set_index("Date", inplace=True)
-
-        # Create individual weekly dummies
-        dummy_columns = {}
-        for i in range(len(df_weekly_start)):
-            col_name = f"dum_{df_weekly_start.index[i].strftime('%Y_%m_%d')}"
-            dummy_columns[col_name] = [0] * len(df_weekly_start)
-            dummy_columns[col_name][i] = 1
-
-        df_dummies = pd.DataFrame(dummy_columns, index=df_weekly_start.index)
-        df_weekly_start = pd.concat([df_weekly_start, df_dummies], axis=1)
-
-        # ---------------------------------------------------------------------
-        # 3. Public holidays (daily) from 'holidays' package + each holiday name
-        # ---------------------------------------------------------------------
-        for country in countries:
-            country_holidays = holidays.CountryHoliday(
-                country, 
-                years=range(int(start_date[:4]), datetime.today().year + 1)
+            Returns:
+                pd.DataFrame: A DataFrame indexed by week start date, containing various
+                            seasonal dummy variables, holidays, trend, and constant.
+                            The date column is named 'OBS'.
+            """
+            # ---------------------------------------------------------------------
+            # 0. Setup: dictionary for 'week_commencing' to Python weekday() integer
+            # ---------------------------------------------------------------------
+            day_dict = {"mon": 0, "tue": 1, "wed": 2, "thu": 3, "fri": 4, "sat": 5, "sun": 6}
+            if week_commencing not in day_dict:
+                raise ValueError(f"Invalid week_commencing value: {week_commencing}. Use one of {list(day_dict.keys())}")
+
+            # ---------------------------------------------------------------------
+            # 1. Create daily date range from start_date to today
+            # ---------------------------------------------------------------------
+            try:
+                start_dt = pd.to_datetime(start_date)
+            except ValueError:
+                raise ValueError(f"Invalid start_date format: {start_date}. Use 'YYYY-MM-DD'")
+
+            end_dt = datetime.today()
+            # Ensure end date is not before start date
+            if end_dt < start_dt:
+                end_dt = start_dt + timedelta(days=1) # Or handle as error if preferred
+
+            date_range = pd.date_range(
+                start=start_dt,
+                end=end_dt,
+                freq="D"
             )
-            # Daily indicator: 1 if that date is a holiday
-            df_daily[f"seas_holiday_{country.lower()}"] = df_daily["Date"].apply(
-                lambda x: 1 if x in country_holidays else 0
+            df_daily = pd.DataFrame(date_range, columns=["Date"])
+
+            # ---------------------------------------------------------------------
+            # 1.1 Identify "week_start" for each daily row, based on week_commencing
+            # ---------------------------------------------------------------------
+            start_day_int = day_dict[week_commencing]
+            df_daily['week_start'] = df_daily["Date"].apply(
+                lambda x: x - pd.Timedelta(days=(x.weekday() - start_day_int) % 7)
             )
-            # Create columns for specific holiday names
-            for date_hol, name in country_holidays.items():
-                col_name = f"seas_{name.replace(' ', '_').lower()}_{country.lower()}"
-                if col_name not in df_daily.columns:
-                    df_daily[col_name] = 0
-                df_daily.loc[df_daily["Date"] == pd.Timestamp(date_hol), col_name] = 1
-
-        # ---------------------------------------------------------------------
-        # 3.1 Additional Special Days (Father's Day, Mother's Day, etc.)
-        #     We'll add daily columns for each. 
-        # ---------------------------------------------------------------------
-        # Initialize columns
-        extra_cols = [
-            "seas_valentines_day", 
-            "seas_halloween", 
-            "seas_fathers_day_us_uk",
-            "seas_mothers_day_us",
-            "seas_mothers_day_uk",
-            "seas_good_friday",
-            "seas_easter_monday",
-            "seas_black_friday",
-            "seas_cyber_monday",
-        ]
-        for c in extra_cols:
-            df_daily[c] = 0  # default zero
-
-        # Helper: nth_weekday_of_month(year, month, weekday, nth=1 => first, 2 => second, etc.)
-        # weekday: Monday=0, Tuesday=1, ... Sunday=6
-        def nth_weekday_of_month(year, month, weekday, nth):
-            """
-            Returns date of the nth <weekday> in <month> of <year>.
-            E.g. nth_weekday_of_month(2023, 6, 6, 3) => 3rd Sunday of June 2023.
-            """
-            # 1st day of the month
-            d = datetime(year, month, 1)
-            # What is the weekday of day #1?
-            w = d.weekday()  # Monday=0, Tuesday=1, ... Sunday=6
-            # If we want, e.g. Sunday=6, we see how many days to add
-            delta = (weekday - w) % 7
-            # This is the first <weekday> in that month
-            first_weekday = d + timedelta(days=delta)
-            # Now add 7*(nth-1) days
-            return first_weekday + timedelta(days=7 * (nth-1))
-
-        def get_good_friday(year):
-            """Good Friday is 2 days before Easter Sunday."""
-            return easter(year) - timedelta(days=2)
-
-        def get_easter_monday(year):
-            """Easter Monday is 1 day after Easter Sunday."""
-            return easter(year) + timedelta(days=1)
-
-        def get_black_friday(year):
-            """
-            Black Friday = day after US Thanksgiving, 
-            and US Thanksgiving is the 4th Thursday in November.
-            """
-            # 4th Thursday in November
-            fourth_thursday = nth_weekday_of_month(year, 11, 3, 4)  # weekday=3 => Thursday
-            return fourth_thursday + timedelta(days=1)
-
-        def get_cyber_monday(year):
-            """Cyber Monday = Monday after US Thanksgiving, i.e. 4 days after 4th Thursday in Nov."""
-            # 4th Thursday in November
-            fourth_thursday = nth_weekday_of_month(year, 11, 3, 4)
-            return fourth_thursday + timedelta(days=4)  # Monday after Thanksgiving
-
-        # Loop over each year in range
-        start_yr = int(start_date[:4])
-        end_yr = datetime.today().year
-
-        for yr in range(start_yr, end_yr + 1):
-            # Valentines = Feb 14
-            valentines_day = datetime(yr, 2, 14)
-            # Halloween = Oct 31
-            halloween_day  = datetime(yr, 10, 31)
-            # Father's Day (US & UK) = 3rd Sunday in June
-            fathers_day    = nth_weekday_of_month(yr, 6, 6, 3)  # Sunday=6
-            # Mother's Day US = 2nd Sunday in May
-            mothers_day_us = nth_weekday_of_month(yr, 5, 6, 2)
-            mothering_sunday = easter(yr) - timedelta(days=21)
-            # If for some reason that's not a Sunday (rare corner cases), shift to Sunday:
-            while mothering_sunday.weekday() != 6:  # Sunday=6
-                mothering_sunday -= timedelta(days=1)
-
-            # Good Friday, Easter Monday
-            gf = get_good_friday(yr)
-            em = get_easter_monday(yr)
-
-            # Black Friday, Cyber Monday
-            bf = get_black_friday(yr)
-            cm = get_cyber_monday(yr)
-
-            # Mark them in df_daily if in range
-            for special_date, col in [
-                (valentines_day, "seas_valentines_day"),
-                (halloween_day,  "seas_halloween"),
-                (fathers_day,    "seas_fathers_day_us_uk"),
-                (mothers_day_us, "seas_mothers_day_us"),
-                (mothering_sunday, "seas_mothers_day_uk"),
-                (gf, "seas_good_friday"),
-                (em, "seas_easter_monday"),
-                (bf, "seas_black_friday"),
-                (cm, "seas_cyber_monday"),
-            ]:
-                # Convert to pd.Timestamp:
-                special_ts = pd.Timestamp(special_date)
-
-                # Only set if it's within your daily range
-                if (special_ts >= df_daily["Date"].min()) and (special_ts <= df_daily["Date"].max()):
-                    df_daily.loc[df_daily["Date"] == special_ts, col] = 1
-
-        # ---------------------------------------------------------------------
-        # 4. Add daily indicators for last day & last Friday of month
-        #    Then aggregate them to weekly level using .max()
-        # ---------------------------------------------------------------------
-        # Last day of month (daily)
-        df_daily["seas_last_day_of_month"] = df_daily["Date"].apply(
-            lambda d: 1 if d == d.to_period("M").to_timestamp("M") else 0
-        )
-
-        # Last Friday of month (daily)
-        def is_last_friday(date):
-            # last day of the month
-            last_day_of_month = date.to_period("M").to_timestamp("M")
-            last_day_weekday = last_day_of_month.weekday()  # Monday=0,...Sunday=6
-            # Determine how many days we go back from the last day to get Friday (weekday=4)
-            if last_day_weekday >= 4:
-                days_to_subtract = last_day_weekday - 4
-            else:
-                days_to_subtract = last_day_weekday + 3
-            last_friday = last_day_of_month - pd.Timedelta(days=days_to_subtract)
-            return 1 if date == last_friday else 0
-
-        df_daily["seas_last_friday_of_month"] = df_daily["Date"].apply(is_last_friday)
-
-        # ---------------------------------------------------------------------
-        # 5. Weekly aggregation for holiday columns & monthly dummies
-        # ---------------------------------------------------------------------
-        # For monthly dummies, create a daily col "Month", then get_dummies
-        df_daily["Month"] = df_daily["Date"].dt.month_name().str.lower()
-        df_monthly_dummies = pd.get_dummies(
-            df_daily, 
-            prefix="seas", 
-            columns=["Month"], 
-            dtype=int
-        )
-        # Recalculate 'week_start' (already in df_daily, but just to be sure)
-        df_monthly_dummies['week_start'] = df_daily['week_start']
-
-        # Group monthly dummies by .sum() or .mean()—we often spread them across the week
-        df_monthly_dummies = (
-            df_monthly_dummies
-            .groupby('week_start')
-            .sum(numeric_only=True)    # sum the daily flags
-            .reset_index()
-            .rename(columns={'week_start': "Date"})
-            .set_index("Date")
-        )
-        # Spread monthly dummies by 7 to distribute across that week
-        monthly_cols = [c for c in df_monthly_dummies.columns if c.startswith("seas_month_")]
-        df_monthly_dummies[monthly_cols] = df_monthly_dummies[monthly_cols] / 7
-
-        # Group holiday & special-day columns by .max() => binary at weekly level
-        df_holidays = (
-            df_daily
-            .groupby('week_start')
-            .max(numeric_only=True)   # if any day=1 in that week, entire week=1
-            .reset_index()
-            .rename(columns={'week_start': "Date"})
-            .set_index("Date")
-        )
 
-        # ---------------------------------------------------------------------
-        # 6. Combine weekly start, monthly dummies, holiday flags
-        # ---------------------------------------------------------------------
-        df_combined = pd.concat([df_weekly_start, df_monthly_dummies], axis=1)
-        df_combined = pd.concat([df_combined, df_holidays], axis=1)
-        df_combined = df_combined.loc[:, ~df_combined.columns.duplicated()]
-
-        # ---------------------------------------------------------------------
-        # 7. Create weekly dummies for Week of Year & yearly dummies
-        # ---------------------------------------------------------------------
-        df_combined.reset_index(inplace=True)
-        df_combined.rename(columns={"index": "old_index"}, inplace=True)  # just in case
-        
-        df_combined["Week"] = df_combined["Date"].dt.isocalendar().week
-        df_combined = pd.get_dummies(df_combined, prefix="seas", columns=["Week"], dtype=int)
-        
-        df_combined["Year"] = df_combined["Date"].dt.year
-        df_combined = pd.get_dummies(df_combined, prefix="seas", columns=["Year"], dtype=int)
-        
-        # ---------------------------------------------------------------------
-        # 8. Add constant & trend
-        # ---------------------------------------------------------------------
-        df_combined["Constant"] = 1
-        df_combined["Trend"] = df_combined.index + 1
-        
-        # ---------------------------------------------------------------------
-        # 9. Rename Date -> OBS and return
-        # ---------------------------------------------------------------------
-        df_combined.rename(columns={"Date": "OBS"}, inplace=True)
-        
-        return df_combined
+            # ---------------------------------------------------------------------
+            # 1.2 Calculate ISO week number for each DAY (for later aggregation)
+            #     Also calculate Year for each DAY to handle year transitions correctly
+            # ---------------------------------------------------------------------
+            df_daily['iso_week_daily'] = df_daily['Date'].dt.isocalendar().week.astype(int)
+            df_daily['iso_year_daily'] = df_daily['Date'].dt.isocalendar().year.astype(int)
+
+
+            # ---------------------------------------------------------------------
+            # 2. Build a weekly index (df_weekly_start) based on unique week_start dates
+            # ---------------------------------------------------------------------
+            df_weekly_start = df_daily[['week_start']].drop_duplicates().sort_values('week_start').reset_index(drop=True)
+            df_weekly_start.rename(columns={'week_start': "Date"}, inplace=True)
+            df_weekly_start.set_index("Date", inplace=True)
+
+            # Create individual weekly dummies (optional, uncomment if needed)
+            # dummy_columns = {}
+            # for i, date_index in enumerate(df_weekly_start.index):
+            #     col_name = f"dum_{date_index.strftime('%Y_%m_%d')}"
+            #     dummy_columns[col_name] = [0] * len(df_weekly_start)
+            #     dummy_columns[col_name][i] = 1
+            # df_dummies = pd.DataFrame(dummy_columns, index=df_weekly_start.index)
+            # df_weekly_start = pd.concat([df_weekly_start, df_dummies], axis=1)
+
+
+            # ---------------------------------------------------------------------
+            # 3. Public holidays (daily) from 'holidays' package + each holiday name
+            # ---------------------------------------------------------------------
+            start_year = start_dt.year
+            end_year = end_dt.year
+            years_range = range(start_year, end_year + 1)
+
+            for country in countries:
+                try:
+                    country_holidays = holidays.CountryHoliday(
+                        country,
+                        years=years_range,
+                        observed=False # Typically you want the actual date, not observed substitute
+                    )
+                    # Handle cases like UK where specific subdivisions might be needed for some holidays
+                    # Example: if country == 'GB': country_holidays.observed = True # If observed are needed
+                except KeyError:
+                    print(f"Warning: Country code '{country}' not found in holidays library. Skipping.")
+                    continue # Skip to next country
+
+                # Daily indicator: 1 if that date is a holiday
+                df_daily[f"seas_holiday_{country.lower()}"] = df_daily["Date"].apply(
+                    lambda x: 1 if x in country_holidays else 0
+                )
+                # Create columns for specific holiday names
+                for date_hol, name in sorted(country_holidays.items()): # Sort for consistent column order
+                    # Clean name: lower, replace space with underscore, remove non-alphanumeric (except underscore)
+                    clean_name = ''.join(c for c in name if c.isalnum() or c == ' ').strip().replace(' ', '_').lower()
+                    clean_name = clean_name.replace('_(observed)', '').replace("'", "") # specific cleaning
+                    col_name = f"seas_{clean_name}_{country.lower()}"
+
+                    # Only create column if the holiday occurs within the df_daily date range
+                    if pd.Timestamp(date_hol).year in years_range:
+                        if col_name not in df_daily.columns:
+                            df_daily[col_name] = 0
+                        # Ensure date_hol is within the actual daily range before assigning
+                        if (pd.Timestamp(date_hol) >= df_daily["Date"].min()) and (pd.Timestamp(date_hol) <= df_daily["Date"].max()):
+                            df_daily.loc[df_daily["Date"] == pd.Timestamp(date_hol), col_name] = 1
+
+            # ---------------------------------------------------------------------
+            # 3.1 Additional Special Days (Father's Day, Mother's Day, etc.)
+            # ---------------------------------------------------------------------
+            extra_cols = [
+                "seas_valentines_day",
+                "seas_halloween",
+                "seas_fathers_day_us_uk", # Note: UK/US is 3rd Sun Jun, others vary
+                "seas_mothers_day_us",    # Note: US is 2nd Sun May
+                "seas_mothers_day_uk",    # Note: UK Mothering Sunday varies with Easter
+                "seas_good_friday",
+                "seas_easter_monday",
+                "seas_black_friday",      # US-centric, but globally adopted
+                "seas_cyber_monday",      # US-centric, but globally adopted
+            ]
+            for c in extra_cols:
+                if c not in df_daily.columns: # Avoid overwriting if already created by holidays pkg
+                    df_daily[c] = 0
+
+            # Helper: nth_weekday_of_month(year, month, weekday, nth)
+            def nth_weekday_of_month(year, month, weekday, nth):
+                d = datetime(year, month, 1)
+                w = d.weekday()
+                delta = (weekday - w + 7) % 7 # Ensure positive delta
+                first_weekday = d + timedelta(days=delta)
+                target_date = first_weekday + timedelta(days=7 * (nth - 1))
+                # Check if the calculated date is still in the same month
+                if target_date.month == month:
+                    return target_date
+                else:
+                    # This can happen if nth is too large (e.g., 5th Friday)
+                    # Return the last occurrence of that weekday in the month instead
+                    return target_date - timedelta(days=7)
+
+
+            def get_good_friday(year):
+                return easter(year) - timedelta(days=2)
+
+            def get_easter_monday(year):
+                return easter(year) + timedelta(days=1)
+
+            def get_black_friday(year):
+                # US Thanksgiving is 4th Thursday in November (weekday=3)
+                thanksgiving = nth_weekday_of_month(year, 11, 3, 4)
+                return thanksgiving + timedelta(days=1)
+
+            def get_cyber_monday(year):
+                # Monday after US Thanksgiving
+                thanksgiving = nth_weekday_of_month(year, 11, 3, 4)
+                return thanksgiving + timedelta(days=4)
+
+            def get_mothering_sunday_uk(year):
+                # Fourth Sunday in Lent (3 weeks before Easter Sunday)
+                # Lent starts on Ash Wednesday, 46 days before Easter.
+                # Easter Sunday is day 0. Sunday before is -7, etc.
+                # 4th Sunday in Lent is 3 weeks before Easter.
+                return easter(year) - timedelta(days=21)
+
+
+            # Loop over each year in range
+            for yr in range(start_year, end_year + 1):
+                try: # Wrap calculations in try-except for robustness
+                    # Valentines = Feb 14
+                    valentines_day = datetime(yr, 2, 14)
+                    # Halloween = Oct 31
+                    halloween_day  = datetime(yr, 10, 31)
+                    # Father's Day (US & UK) = 3rd Sunday (6) in June
+                    fathers_day    = nth_weekday_of_month(yr, 6, 6, 3)
+                    # Mother's Day US = 2nd Sunday (6) in May
+                    mothers_day_us = nth_weekday_of_month(yr, 5, 6, 2)
+                    # Mother's Day UK (Mothering Sunday)
+                    mothering_sunday = get_mothering_sunday_uk(yr)
+
+                    # Good Friday, Easter Monday
+                    gf = get_good_friday(yr)
+                    em = get_easter_monday(yr)
+
+                    # Black Friday, Cyber Monday
+                    bf = get_black_friday(yr)
+                    cm = get_cyber_monday(yr)
+
+                    # Mark them in df_daily if in range
+                    special_days_map = [
+                        (valentines_day, "seas_valentines_day"),
+                        (halloween_day,  "seas_halloween"),
+                        (fathers_day,    "seas_fathers_day_us_uk"),
+                        (mothers_day_us, "seas_mothers_day_us"),
+                        (mothering_sunday,"seas_mothers_day_uk"),
+                        (gf, "seas_good_friday"),
+                        (em, "seas_easter_monday"),
+                        (bf, "seas_black_friday"),
+                        (cm, "seas_cyber_monday"),
+                    ]
+
+                    for special_date, col in special_days_map:
+                        if special_date is not None: # nth_weekday_of_month can return None edge cases
+                            special_ts = pd.Timestamp(special_date)
+                            # Only set if it's within the daily range AND column exists
+                            if (special_ts >= df_daily["Date"].min()) and \
+                            (special_ts <= df_daily["Date"].max()) and \
+                            (col in df_daily.columns):
+                                df_daily.loc[df_daily["Date"] == special_ts, col] = 1
+                except Exception as e:
+                    print(f"Warning: Could not calculate special days for year {yr}: {e}")
+
+
+            # ---------------------------------------------------------------------
+            # 4. Add daily indicators for last day & last Friday of month
+            # ---------------------------------------------------------------------
+            df_daily["is_last_day_of_month"] = df_daily["Date"].dt.is_month_end
+
+            def is_last_friday(date):
+                # Check if it's a Friday first
+                if date.weekday() != 4: # Friday is 4
+                    return 0
+                # Check if next Friday is in the next month
+                next_friday = date + timedelta(days=7)
+                return 1 if next_friday.month != date.month else 0
+
+            df_daily["is_last_friday_of_month"] = df_daily["Date"].apply(is_last_friday)
+
+            # Rename for clarity prefix
+            df_daily.rename(columns={
+                "is_last_day_of_month": "seas_last_day_of_month",
+                "is_last_friday_of_month": "seas_last_friday_of_month"
+            }, inplace=True)
+
+
+            # ---------------------------------------------------------------------
+            # 5. Weekly aggregation
+            # ---------------------------------------------------------------------
+
+            # --- Aggregate flags using MAX (1 if any day in week is flagged) ---
+            # Select only columns that are indicators/flags (intended for max aggregation)
+            flag_cols = [col for col in df_daily.columns if col.startswith('seas_') or col.startswith('is_')]
+            # Ensure 'week_start' is present for grouping
+            df_to_agg = df_daily[['week_start'] + flag_cols]
+
+            df_weekly_flags = (
+                df_to_agg
+                .groupby('week_start')
+                .max() # if any day=1 in that week, entire week=1
+                .reset_index()
+                .rename(columns={'week_start': "Date"})
+                .set_index("Date")
+            )
 
+            # --- Aggregate Week Number using MODE ---
+            # Define aggregation function for mode (handling potential multi-modal cases by taking the first)
+            def get_mode(x):
+                modes = pd.Series.mode(x)
+                return modes[0] if not modes.empty else np.nan # Return first mode or NaN
+
+            df_weekly_iso_week_year = (
+                df_daily[['week_start', 'iso_week_daily', 'iso_year_daily']]
+                .groupby('week_start')
+                .agg(
+                    # Find the most frequent week number and year within the group
+                    Week=('iso_week_daily', get_mode),
+                    Year=('iso_year_daily', get_mode)
+                )
+                .reset_index()
+                .rename(columns={'week_start': 'Date'})
+                .set_index('Date')
+            )
+            # Convert Week/Year back to integer type after aggregation
+            df_weekly_iso_week_year['Week'] = df_weekly_iso_week_year['Week'].astype(int)
+            df_weekly_iso_week_year['Year'] = df_weekly_iso_week_year['Year'].astype(int)
+
+
+            # --- Monthly dummies (spread evenly across week) ---
+            df_daily["Month"] = df_daily["Date"].dt.month_name().str.lower()
+            df_monthly_dummies_daily = pd.get_dummies(
+                df_daily[["week_start", "Month"]], # Only need these columns
+                prefix="seas_month",
+                columns=["Month"],
+                dtype=float # Use float for division
+            )
+            # Sum daily dummies within the week
+            df_monthly_dummies_summed = df_monthly_dummies_daily.groupby('week_start').sum()
+            # Divide by number of days in that specific week group (usually 7, except potentially start/end)
+            days_in_week = df_daily.groupby('week_start').size()
+            df_weekly_monthly_dummies = df_monthly_dummies_summed.div(days_in_week, axis=0)
+
+            # Reset index to merge
+            df_weekly_monthly_dummies.reset_index(inplace=True)
+            df_weekly_monthly_dummies.rename(columns={'week_start': 'Date'}, inplace=True)
+            df_weekly_monthly_dummies.set_index('Date', inplace=True)
+
+
+            # ---------------------------------------------------------------------
+            # 6. Combine all weekly components
+            # ---------------------------------------------------------------------
+            # Start with the basic weekly index
+            df_combined = df_weekly_start.copy()
+
+            # Join the other aggregated DataFrames
+            df_combined = df_combined.join(df_weekly_flags, how='left')
+            df_combined = df_combined.join(df_weekly_iso_week_year, how='left')
+            df_combined = df_combined.join(df_weekly_monthly_dummies, how='left')
+
+            # Fill potential NaNs created by joins (e.g., if a flag column didn't exist) with 0
+            # Exclude 'Week' and 'Year' which should always be present
+            cols_to_fill = df_combined.columns.difference(['Week', 'Year'])
+            df_combined[cols_to_fill] = df_combined[cols_to_fill].fillna(0)
+
+            # Ensure correct types for flag columns (int)
+            for col in df_weekly_flags.columns:
+                if col in df_combined.columns:
+                    df_combined[col] = df_combined[col].astype(int)
+
+            # Ensure correct types for month columns (float)
+            for col in df_weekly_monthly_dummies.columns:
+                if col in df_combined.columns:
+                    df_combined[col] = df_combined[col].astype(float)
+
+
+            # ---------------------------------------------------------------------
+            # 7. Create weekly dummies for Week of Year & yearly dummies from aggregated cols
+            # ---------------------------------------------------------------------
+            df_combined.reset_index(inplace=True) # 'Date', 'Week', 'Year' become columns
+
+            # Create dummies from the aggregated 'Week' column
+            df_combined = pd.get_dummies(df_combined, prefix="seas", columns=["Week"], dtype=int, prefix_sep='_')
+
+            # Create dummies from the aggregated 'Year' column
+            df_combined = pd.get_dummies(df_combined, prefix="seas", columns=["Year"], dtype=int, prefix_sep='_')
+
+            # ---------------------------------------------------------------------
+            # 8. Add constant & trend
+            # ---------------------------------------------------------------------
+            df_combined["Constant"] = 1
+            df_combined.reset_index(drop=True, inplace=True) # Ensure index is 0, 1, 2... for trend
+            df_combined["Trend"] = df_combined.index + 1
+
+            # ---------------------------------------------------------------------
+            # 9. Rename Date -> OBS and select final columns
+            # ---------------------------------------------------------------------
+            df_combined.rename(columns={"Date": "OBS"}, inplace=True)
+
+            # Reorder columns - OBS first, then Constant, Trend, then seasonal features
+            cols_order = ['OBS', 'Constant', 'Trend'] + \
+                        sorted([col for col in df_combined.columns if col.startswith('seas_')]) + \
+                        sorted([col for col in df_combined.columns if col.startswith('dum_')]) # If individual week dummies were enabled
+
+            # Filter out columns not in the desired order list (handles case where dum_ cols are off)
+            final_cols = [col for col in cols_order if col in df_combined.columns]
+            df_combined = df_combined[final_cols]
+
+            return df_combined
+    
     def pull_weather(self, week_commencing, start_date, country_codes) -> pd.DataFrame:
         """
         Pull weather data for a given week-commencing day and one or more country codes.

{imsciences-0.9.6.3.dist-info → imsciences-0.9.6.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: imsciences
-Version: 0.9.6.3
+Version: 0.9.6.4
 Summary: IMS Data Processing Package
 Author: IMS
 Author-email: cam@im-sciences.com

{imsciences-0.9.6.3.dist-info → imsciences-0.9.6.4.dist-info}/RECORD

@@ -1,12 +1,12 @@
 imsciences/__init__.py,sha256=_HuYeLbDMTdt7GpKI4r6-d7yRPZgcAQ7yOW0-ydR2Yo,117
 imsciences/geo.py,sha256=eenng7_BP_E2WD5Wt1G_oNxQS8W3t6lycRwJ91ngysY,15808
 imsciences/mmm.py,sha256=qMh0ccOepehfCcux7EeG8cq6piSEoFEz5iiJbDBWOS4,82214
-imsciences/pull.py,sha256=F83xlklM_lyPffMMZasHWLxDaeUHtOnUQGAUsiV7ves,88073
+imsciences/pull.py,sha256=aLgqFxbuCnSsyhlC3aWXcloeQGIGxY8KKeN8kNnns9c,95091
 imsciences/unittesting.py,sha256=U177_Txg0Lqn49zYRu5bl9OVe_X7MkNJ6V_Zd6DHOsU,45656
 imsciences/vis.py,sha256=2izdHQhmWEReerRqIxhY4Ai10VjL7xoUqyWyZC7-2XI,8931
-imsciences-0.9.6.3.dist-info/LICENSE.txt,sha256=lVq2QwcExPX4Kl2DHeEkRrikuItcDB1Pr7yF7FQ8_z8,1108
-imsciences-0.9.6.3.dist-info/METADATA,sha256=RMcthCSyWmU6IBsXGL-nYqw0RP06pzjPKK3dzOQcU-8,18846
-imsciences-0.9.6.3.dist-info/PKG-INFO-TomG-HP-290722,sha256=RMcthCSyWmU6IBsXGL-nYqw0RP06pzjPKK3dzOQcU-8,18846
-imsciences-0.9.6.3.dist-info/WHEEL,sha256=ixB2d4u7mugx_bCBycvM9OzZ5yD7NmPXFRtKlORZS2Y,91
-imsciences-0.9.6.3.dist-info/top_level.txt,sha256=hsENS-AlDVRh8tQJ6-426iUQlla9bPcGc0-UlFF0_iU,11
-imsciences-0.9.6.3.dist-info/RECORD,,
+imsciences-0.9.6.4.dist-info/LICENSE.txt,sha256=lVq2QwcExPX4Kl2DHeEkRrikuItcDB1Pr7yF7FQ8_z8,1108
+imsciences-0.9.6.4.dist-info/METADATA,sha256=3UJiLbs9Uqjv6e1GBIppcQYVH3PUR5_yo2cs951MMq4,18846
+imsciences-0.9.6.4.dist-info/PKG-INFO-TomG-HP-290722,sha256=RMcthCSyWmU6IBsXGL-nYqw0RP06pzjPKK3dzOQcU-8,18846
+imsciences-0.9.6.4.dist-info/WHEEL,sha256=ixB2d4u7mugx_bCBycvM9OzZ5yD7NmPXFRtKlORZS2Y,91
+imsciences-0.9.6.4.dist-info/top_level.txt,sha256=hsENS-AlDVRh8tQJ6-426iUQlla9bPcGc0-UlFF0_iU,11
+imsciences-0.9.6.4.dist-info/RECORD,,