imsciences 0.9.5.2__py3-none-any.whl → 0.9.5.4__py3-none-any.whl

imsciences/mmm.py

@@ -6,6 +6,9 @@ import re
 from datetime import datetime, timedelta
 import subprocess
 import json
+from sklearn.model_selection import train_test_split
+import xgboost as xgb
+from sklearn.ensemble import RandomForestRegressor
 
 class dataprocessing:
     
@@ -180,7 +183,12 @@ class dataprocessing:
         print("   - Description: Maps dates to the start of the current ISO week based on a specified weekday.")
         print("   - Usage: week_commencing_2_week_commencing_conversion_isoweekday(df, date_col, week_commencing='mon')")
         print("   - Example: week_commencing_2_week_commencing_conversion_isoweekday(df, 'date_col', week_commencing='fri')")
-              
+        
+        print("\n35. seasonality_feature_extraction")
+        print("   - Description: Splits data into train/test sets, trains XGBoost and Random Forest on all features, extracts top features based on feature importance, merges them, optionally retrains models on top and combined features, and returns a dict of results.")
+        print("   - Usage: seasonality_feature_extraction(df, kpi_var, n_features=10, test_size=0.1, random_state=42, shuffle=False)")
+        print("   - Example: seasonality_feature_extraction(df, 'kpi_total_sales', n_features=5, test_size=0.2, random_state=123, shuffle=True)")
+
     def get_wd_levels(self, levels):
         """
         Gets the current wd of whoever is working on it and gives the options to move the number of levels up.
@@ -492,15 +500,15 @@ class dataprocessing:
 
         return combined_df
     
-    def pivot_table(self, df, index_col, columns, values_col, filters_dict=None, fill_value=0, aggfunc="sum", margins=False, margins_name="Total", datetime_trans_needed=True, date_format="%Y-%m-%d", reverse_header_order=False, fill_missing_weekly_dates=False, week_commencing="W-MON"):       
+    def pivot_table(self, df, index_col, columns, values_col, filters_dict=None, fill_value=0, aggfunc="sum", margins=False, margins_name="Total", datetime_trans_needed=True, date_format="%Y-%m-%d", reverse_header_order=False, fill_missing_weekly_dates=True, week_commencing="W-MON"):       
         """
         Provides the ability to create pivot tables, filtering the data to get to data you want and then pivoting on certain columns
 
         Args:
             df (pandas.DataFrame): The DataFrame containing the data.
             index_col (str): Name of Column for your pivot table to index on
-            columns (str): Name of Columns for your pivot table.
-            values_col (str): Name of Values Columns for your pivot table.
+            columns (str or list): Name of Column(s) for your pivot table. Can be a single column or a list of columns.
+            values_col (str or list): Name of Values Column(s) for your pivot table. Can be a single column or a list of columns.
             filters_dict (dict, optional): Dictionary of conditions for the boolean mask i.e. what to filter your df on to get to your chosen cell. Defaults to None
             fill_value (int, optional): The value to replace nan with. Defaults to 0.
             aggfunc (str, optional): The method on which to aggregate the values column. Defaults to sum.
@@ -514,14 +522,19 @@ class dataprocessing:
         Returns:
             pandas.DataFrame: The pivot table specified
         """
-        
         # Validate inputs
         if index_col not in df.columns:
             raise ValueError(f"index_col '{index_col}' not found in DataFrame.")
-        if columns not in df.columns:
-            raise ValueError(f"columns '{columns}' not found in DataFrame.")
-        if values_col not in df.columns:
-            raise ValueError(f"values_col '{values_col}' not found in DataFrame.")
+
+        columns = [columns] if isinstance(columns, str) else columns
+        for col in columns:
+            if col not in df.columns:
+                raise ValueError(f"columns '{col}' not found in DataFrame.")
+
+        values_col = [values_col] if isinstance(values_col, str) else values_col
+        for col in values_col:
+            if col not in df.columns:
+                raise ValueError(f"values_col '{col}' not found in DataFrame.")
 
         # Apply filters if provided
         if filters_dict:
@@ -1412,4 +1425,133 @@ class dataprocessing:
         new_col = f"week_start_{week_commencing}"
         df[new_col] = df[date_col].apply(map_to_week_start)
         
-        return df
+        return df
+    
+    def seasonality_feature_extraction(self, df, kpi_var, n_features=10, test_size=0.1, random_state=42, shuffle=False):
+        """
+        1) Uses the provided dataframe (df), where:
+        - df['kpi_total_sales'] is the target (y).
+        - df['OBS'] is a date or index column (excluded from features).
+        
+        2) Splits data into train/test using the specified test_size, random_state, and shuffle.
+        3) Trains XGBoost and Random Forest on all features.
+        4) Extracts the top n_features from each model.
+        5) Merges their unique top features.
+        6) Optionally retrains each model on the combined top features.
+        7) Returns performance metrics and the fitted models.
+
+        Parameters
+        ----------
+        df : pd.DataFrame
+            The input dataframe that contains kpi_var (target) and 'OBS' (date/index).
+        n_features : int, optional
+            Number of top features to extract from each model (default=10).
+        test_size : float, optional
+            Test size for train_test_split (default=0.1).
+        random_state : int, optional
+            Random state for reproducibility (default=42).
+        shuffle : bool, optional
+            Whether to shuffle the data before splitting (default=False).
+
+        Returns
+        -------
+        dict
+            A dictionary containing:
+            - "top_features_xgb": list of top n_features from XGBoost
+            - "top_features_rf": list of top n_features from Random Forest
+            - "combined_features": merged unique feature list
+            - "performance": dictionary of performance metrics
+            - "models": dictionary of fitted models
+        """
+        # ---------------------------------------------------------------------
+        # 1. Prepare your data (X, y)
+        # ---------------------------------------------------------------------
+        # Extract target and features
+        y = df[kpi_var]
+        X = df.drop(columns=['OBS', kpi_var])
+
+        # Split into train/test
+        X_train, X_test, y_train, y_test = train_test_split(
+            X, y,
+            test_size=test_size,
+            random_state=random_state,
+            shuffle=shuffle
+        )
+
+        # ---------------------------------------------------------------------
+        # 2. XGBoost Approach (on all features)
+        # ---------------------------------------------------------------------
+        # (A) Train full model on ALL features
+        xgb_model_full = xgb.XGBRegressor(random_state=random_state)
+        xgb_model_full.fit(X_train, y_train)
+
+        # (B) Get feature importances
+        xgb_importances = xgb_model_full.feature_importances_
+        xgb_feat_importance_df = (
+            pd.DataFrame({
+                'feature': X.columns,
+                'importance': xgb_importances
+            })
+            .sort_values('importance', ascending=False)
+            .reset_index(drop=True)
+        )
+
+        # (C) Select top N features
+        top_features_xgb = xgb_feat_importance_df['feature'].head(n_features).tolist()
+
+        # (D) Subset data to top N features
+        X_train_xgb_topN = X_train[top_features_xgb]
+
+        # (E) Retrain XGBoost on these top N features
+        xgb_model_topN = xgb.XGBRegressor(random_state=random_state)
+        xgb_model_topN.fit(X_train_xgb_topN, y_train)
+
+        # ---------------------------------------------------------------------
+        # 3. Random Forest Approach (on all features)
+        # ---------------------------------------------------------------------
+        rf_model_full = RandomForestRegressor(random_state=random_state)
+        rf_model_full.fit(X_train, y_train)
+
+        # (B) Get feature importances
+        rf_importances = rf_model_full.feature_importances_
+        rf_feat_importance_df = (
+            pd.DataFrame({
+                'feature': X.columns,
+                'importance': rf_importances
+            })
+            .sort_values('importance', ascending=False)
+            .reset_index(drop=True)
+        )
+
+        # (C) Select top N features
+        top_features_rf = rf_feat_importance_df['feature'].head(n_features).tolist()
+
+        # (D) Subset data to top N features
+        X_train_rf_topN = X_train[top_features_rf]
+
+        # (E) Retrain Random Forest on these top N features
+        rf_model_topN = RandomForestRegressor(random_state=random_state)
+        rf_model_topN.fit(X_train_rf_topN, y_train)
+
+        # ---------------------------------------------------------------------
+        # 4. Combine top features from both models
+        # ---------------------------------------------------------------------
+        combined_features = list(set(top_features_xgb + top_features_rf))
+
+        # Create new training/testing data with the combined features
+        X_train_combined = X_train[combined_features]
+
+        # (Optional) Retrain XGBoost on combined features
+        xgb_model_combined = xgb.XGBRegressor(random_state=random_state)
+        xgb_model_combined.fit(X_train_combined, y_train)
+
+        # (Optional) Retrain Random Forest on combined features
+        rf_model_combined = RandomForestRegressor(random_state=random_state)
+        rf_model_combined.fit(X_train_combined, y_train)
+
+        # Organize all results to return
+        output = {
+            "combined_features": combined_features,
+        }
+
+        return output

{imsciences-0.9.5.2.dist-info → imsciences-0.9.5.4.dist-info}/METADATA

@@ -1,10 +1,10 @@
 Metadata-Version: 2.1
 Name: imsciences
-Version: 0.9.5.2
+Version: 0.9.5.4
 Summary: IMS Data Processing Package
 Author: IMS
 Author-email: cam@im-sciences.com
-Keywords: python,data processing,apis
+Keywords: data processing,apis,data analysis,data visualization,machine learning
 Classifier: Development Status :: 3 - Alpha
 Classifier: Intended Audience :: Developers
 Classifier: Programming Language :: Python :: 3
@@ -17,6 +17,8 @@ Requires-Dist: pandas
 Requires-Dist: plotly
 Requires-Dist: numpy
 Requires-Dist: fredapi
+Requires-Dist: xgboost
+Requires-Dist: scikit-learn
 Requires-Dist: bs4
 Requires-Dist: yfinance
 Requires-Dist: holidays
@@ -222,6 +224,11 @@ Table of Contents
 - **Usage**: `week_commencing_2_week_commencing_conversion_isoweekday(df, date_col, week_commencing='mon')`
 - **Example**: `week_commencing_2_week_commencing_conversion_isoweekday(df, 'date_col', week_commencing='fri')`
 
+## 35. `seasonality_feature_extraction`
+- **Description**: Splits data into train/test sets, trains XGBoost and Random Forest on all features, extracts top features based on feature importance, merges them, optionally retrains models on top and combined features, and returns a dict of results.
+- **Usage**: `seasonality_feature_extraction(df, kpi_var, n_features=10, test_size=0.1, random_state=42, shuffle=False)`
+- **Example**: `seasonality_feature_extraction(df, 'kpi_total_sales', n_features=5, test_size=0.2, random_state=123, shuffle=True)`
+
 ---
 
 ## Data Processing for Incrementality Testing

{imsciences-0.9.5.2.dist-info → imsciences-0.9.5.4.dist-info}/RECORD

@@ -6,7 +6,7 @@ imsciences/datafunctions-IMS-24Ltp-3.py,sha256=3Snv-0iE_03StmyjtT-riOU9f4v8TaJWL
 imsciences/datafunctions.py,sha256=WZrXNLO-SYrCuFt0pAbha74psMOZPY7meWJ7yWEbRpk,169953
 imsciences/datapull.py,sha256=TPY0LDgOkcKTBk8OekbD0Grg5x0SomAK2dZ7MuT6X1E,19000
 imsciences/geo.py,sha256=J8AkLk1Nyty3VBkPFqcseXjtlSvXVNkHW_nymERz3nA,13472
-imsciences/mmm.py,sha256=M91Qs_ijtY4EytTo1rruWCOVGnV7DMVaYUhNpP1NVNc,73920
+imsciences/mmm.py,sha256=w2A90eJPvMH0Mp3jh8booKaLGm0BKFqW-H92FR4OpV8,80490
 imsciences/pull.py,sha256=bGz8B7bBQ5b9hrx3ipCFTWl_eebEb7rPL4dANKiVWTY,74015
 imsciences/unittesting.py,sha256=DYGqVCsZHrs_tZ-EXDW8q8CdlcsTnG8HsnmWjEE521c,45691
 imsciences/vis.py,sha256=2izdHQhmWEReerRqIxhY4Ai10VjL7xoUqyWyZC7-2XI,8931
@@ -14,9 +14,9 @@ imsciencesdataprocessing/__init__.py,sha256=quSwsLs6IuLoA5Rzi0ZD40xZaQudwDteF7_a
 imsciencesdataprocessing/datafunctions.py,sha256=vE1vsZ8xOSbR9Bwlp9SWXwEHXQ0nFydwGkvzHXf2f1Y,41
 imsdataprocessing/__init__.py,sha256=quSwsLs6IuLoA5Rzi0ZD40xZaQudwDteF7_ai9JfTPk,32
 imsdataprocessing/datafunctions.py,sha256=vE1vsZ8xOSbR9Bwlp9SWXwEHXQ0nFydwGkvzHXf2f1Y,41
-imsciences-0.9.5.2.dist-info/LICENSE.txt,sha256=lVq2QwcExPX4Kl2DHeEkRrikuItcDB1Pr7yF7FQ8_z8,1108
-imsciences-0.9.5.2.dist-info/METADATA,sha256=Zvfy7-y18I-ZFj4aTL18nJOPp57k4uwj5p5ajze5GwI,16994
-imsciences-0.9.5.2.dist-info/PKG-INFO-IMS-24Ltp-3,sha256=yqZbigwHjnYoqyI81PGz_AeofRFfOrwH_Vyawyef-mg,854
-imsciences-0.9.5.2.dist-info/WHEEL,sha256=ixB2d4u7mugx_bCBycvM9OzZ5yD7NmPXFRtKlORZS2Y,91
-imsciences-0.9.5.2.dist-info/top_level.txt,sha256=hsENS-AlDVRh8tQJ6-426iUQlla9bPcGc0-UlFF0_iU,11
-imsciences-0.9.5.2.dist-info/RECORD,,
+imsciences-0.9.5.4.dist-info/LICENSE.txt,sha256=lVq2QwcExPX4Kl2DHeEkRrikuItcDB1Pr7yF7FQ8_z8,1108
+imsciences-0.9.5.4.dist-info/METADATA,sha256=wCS5rgCM0s3XEwXIPZCwBpaej5PxebfgHB1MaKy-5us,17644
+imsciences-0.9.5.4.dist-info/PKG-INFO-IMS-24Ltp-3,sha256=yqZbigwHjnYoqyI81PGz_AeofRFfOrwH_Vyawyef-mg,854
+imsciences-0.9.5.4.dist-info/WHEEL,sha256=ixB2d4u7mugx_bCBycvM9OzZ5yD7NmPXFRtKlORZS2Y,91
+imsciences-0.9.5.4.dist-info/top_level.txt,sha256=hsENS-AlDVRh8tQJ6-426iUQlla9bPcGc0-UlFF0_iU,11
+imsciences-0.9.5.4.dist-info/RECORD,,