gitlabds 2.1.5__tar.gz → 2.1.7__tar.gz

{gitlabds-2.1.5 → gitlabds-2.1.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: gitlabds
-Version: 2.1.5
+Version: 2.1.7
 Summary: Gitlab Data Science and Modeling Tools
 Home-page: https://gitlab.com/gitlab-data/gitlabds
 Author: Kevin Dietz

{gitlabds-2.1.5 → gitlabds-2.1.7}/gitlabds/dummy.py

@@ -50,6 +50,10 @@ def dummy_code(
         - The transformed DataFrame with dummy-coded columns
         - Dictionary of dummy coding information that can be used with apply_dummy()
     """
+    import pandas as pd
+    import warnings
+    from typing import Dict, List, Any, Union, Optional, Tuple
+    
     # Parameter validation
     if not isinstance(df, pd.DataFrame):
         raise TypeError("df must be a pandas DataFrame")
@@ -103,7 +107,6 @@ def dummy_code(
         # Determine number of levels for each field
         cat_levels = df[var_list].select_dtypes(include="object").nunique(dropna=True, axis=0)
         
-        
         # Filter columns by level count criteria
         eligible_cat_cols = cat_levels[(cat_levels <= categorical_max_levels) & (cat_levels > 1)]
         
@@ -156,7 +159,17 @@ def dummy_code(
         for col in num_columns:
             # Store unique values for later use
             values = sorted(df[col].dropna().unique().tolist())
-            dummy_info[col] = values
+            
+            # create column names with float format for numeric data
+            formatted_values = []
+            for v in values:
+                if isinstance(v, (int, float)):
+                    # Store as float to match get_dummies column naming
+                    formatted_values.append(float(v))
+                else:
+                    formatted_values.append(v)
+            
+            dummy_info[col] = formatted_values
         
         # Perform dummy coding if we have columns to encode
         if num_columns:
@@ -222,6 +235,10 @@ def dummy_top(
         - The transformed DataFrame with dummy-coded columns
         - Dictionary of dummy coding information that can be used with apply_dummy()
     """
+    import pandas as pd
+    import warnings
+    from typing import Dict, List, Any, Union, Optional, Tuple
+    
     # Parameter validation
     if not isinstance(df, pd.DataFrame):
         raise TypeError("df must be a pandas DataFrame")
@@ -279,6 +296,10 @@ def dummy_top(
         else:
             print("No categorical columns found")
 
+    # Collect all dummy columns to create at once
+    all_dummy_cols = {}
+    columns_to_drop = []
+
     # Create dummy codes for categorical fields that exceed min_threshold
     for col in cat_columns:
         # Calculate value frequencies
@@ -300,18 +321,32 @@ def dummy_top(
                 print(f"No levels exceed threshold for column '{col}'")
             continue
         
-        # Store information for this column
-        dummy_info[col] = high_freq_list
+        # For numeric columns, ensure float format to match what get_dummies creates
+        if pd.api.types.is_numeric_dtype(df[col]):
+            formatted_values = [float(v) if isinstance(v, (int, float)) else v for v in high_freq_list]
+        else:
+            formatted_values = high_freq_list
+            
+        dummy_info[col] = formatted_values
         
         # Create dummies for values that exceed threshold
         for value in high_freq_list:
             dummy_name = f"{col}{prefix_sep}{value}"
             # Create dummy column (1 if equal to value, 0 otherwise)
-            work_df[dummy_name] = (work_df[col] == value).astype(int)
+            all_dummy_cols[dummy_name] = (work_df[col] == value).astype(int)
         
-        # Drop categorical field if requested
+        # Mark for dropping if requested
         if drop_categorical:
-            work_df = work_df.drop(columns=[col])
+            columns_to_drop.append(col)
+
+    # Create all dummy columns at once using concat (prevents fragmentation)
+    if all_dummy_cols:
+        dummy_df = pd.DataFrame(all_dummy_cols, index=work_df.index)
+        work_df = pd.concat([work_df, dummy_df], axis=1)
+
+    # Drop categorical fields if requested
+    if columns_to_drop:
+        work_df = work_df.drop(columns=columns_to_drop)
 
     # Store the prefix_sep in the dummy_info to use with apply_dummy
     dummy_info["_config"] = {"prefix_sep": prefix_sep}
@@ -319,11 +354,12 @@ def dummy_top(
     return work_df, dummy_info
 
 
+
 def apply_dummy(
     df: pd.DataFrame, 
     dummy_info: Dict[str, List[Any]],
     drop_original: bool = False,
-
+    dummy_na: bool = False,
 ) -> pd.DataFrame:
     """
     Apply dummy coding to a new dataframe using the information from a previous
@@ -338,12 +374,18 @@ def apply_dummy(
         dummy_code() or dummy_top().
     drop_original : bool, default=False
         Whether to drop the original columns after dummy coding.
+    dummy_na : bool, default=False
+        Whether to create dummy variables for NA values.
     
     Returns
     -------
     pd.DataFrame
         Transformed dataframe with dummy-coded columns.
     """
+    import pandas as pd
+    import warnings
+    from typing import Dict, List, Any
+    
     # Validate input
     if not isinstance(df, pd.DataFrame):
         raise TypeError("df must be a pandas DataFrame")
@@ -353,43 +395,78 @@ def apply_dummy(
         
     if df.empty:
         raise ValueError("DataFrame is empty")
-
+    
     # Get the prefix_sep from dummy_info or use default
     prefix_sep = "_dummy_"
+    processing_dummy_info = dummy_info.copy()
+    
     if "_config" in dummy_info and "prefix_sep" in dummy_info["_config"]:
         prefix_sep = dummy_info["_config"]["prefix_sep"]
         # Remove _config from processing
-        dummy_info = {k: v for k, v in dummy_info.items() if k != "_config"}
+        processing_dummy_info = {k: v for k, v in dummy_info.items() if k != "_config"}
     
     # Create a copy to avoid modifying the original
     work_df = df.copy(deep=True)
     
+    # Collect all new dummy columns in a dictionary
+    new_dummy_cols = {}
+    columns_to_drop = []
+    
     # Process each column in the dummy_info dictionary
-    for col, values in dummy_info.items():
+    for col, values in processing_dummy_info.items():
         if col not in work_df.columns:
             warnings.warn(f"Column '{col}' not found in dataframe and will be skipped")
             continue
-            
+        
         # Create dummies for each value
         for value in values:
             dummy_name = f"{col}{prefix_sep}{value}"
             
-            # Handle numeric vs string values appropriately
-            if pd.api.types.is_numeric_dtype(type(value)):
-                work_df[dummy_name] = (work_df[col] == value).astype(int)
+            # Handle type mismatches between config and actual data
+            # Try both the original value and converted versions
+            if pd.api.types.is_numeric_dtype(work_df[col]):
+                # For numeric columns, try both int and float versions
+                try:
+                    # First try exact match
+                    mask = (work_df[col] == value)
+                    
+                    # If no matches and value types differ, try conversion
+                    if not mask.any():
+                        if isinstance(value, float) and work_df[col].dtype in ['int64', 'int32', 'int8']:
+                            # Config expects float (1.0) but data is int (1)
+                            mask = (work_df[col] == int(value))
+                        elif isinstance(value, int) and work_df[col].dtype in ['float64', 'float32']:
+                            # Config expects int (1) but data is float (1.0)
+                            mask = (work_df[col] == float(value))
+                    
+                    new_dummy_cols[dummy_name] = mask.astype(int)
+                    
+                except (ValueError, TypeError):
+                    # Fallback to string comparison if numeric conversion fails
+                    new_dummy_cols[dummy_name] = (work_df[col].astype(str) == str(value)).astype(int)
             else:
-                work_df[dummy_name] = (work_df[col] == value).astype(int)
+                # For non-numeric columns, use direct comparison
+                new_dummy_cols[dummy_name] = (work_df[col] == value).astype(int)
             
         # Create dummy for NA values if requested
         if dummy_na:
-            dummy_name = f"{col}_dummy_nan"
-            work_df[dummy_name] = work_df[col].isna().astype(int)
+            dummy_name = f"{col}{prefix_sep}nan"
+            new_dummy_cols[dummy_name] = work_df[col].isna().astype(int)
             
-        # Drop the original column if requested
+        # Mark column for dropping if requested
         if drop_original:
             # For categorical columns (from dummy_code) or any column (from dummy_top)
             # if its object type or explicitly requested
             if pd.api.types.is_object_dtype(work_df[col]) or drop_original:
-                work_df = work_df.drop(columns=[col])
+                columns_to_drop.append(col)
+    
+    # Create all dummy columns at once using concat
+    if new_dummy_cols:
+        dummy_df = pd.DataFrame(new_dummy_cols, index=work_df.index)
+        work_df = pd.concat([work_df, dummy_df], axis=1)
+    
+    # Drop original columns if requested
+    if columns_to_drop:
+        work_df = work_df.drop(columns=columns_to_drop)
     
     return work_df

{gitlabds-2.1.5 → gitlabds-2.1.7}/gitlabds/model_evaluator.py

@@ -965,7 +965,7 @@ class ModelEvaluator:
                     # Calculate metrics efficiently
                     class_lift[f"class_{class_label}_total"] = grouped.size()
                     class_lift[f"class_{class_label}_count"] = grouped.apply(
-                        lambda x: sum(x["actual"] == class_label)
+                        lambda x: sum(x["actual"] == class_label), include_groups=False
                     )
                     class_lift[f"class_{class_label}_accuracy"] = (
                         class_lift[f"class_{class_label}_count"] / 
@@ -1938,7 +1938,6 @@ class ModelEvaluator:
                     ax.set_yticks([])
                     ax.set_title(f"Class {class_idx}")
 
-
     def plot_shap_beeswarm(
         self,
         n_features: int = 20,
@@ -1954,6 +1953,13 @@ class ModelEvaluator:
     ) -> None:
         """
         Create a SHAP beeswarm plot for feature importance.
+        
+        Parameters
+        ----------
+        class_index : int, optional
+            For multi-class classification, specify which class to plot.
+            If None, plots global importance (averaged across classes).
+            Only applicable for multi-class models.
         """
         # Set defaults if not provided
         save_plot = self.save_plots if save_plot is None else save_plot
@@ -1986,33 +1992,62 @@ class ModelEvaluator:
             # Get SHAP values
             shap_values = explainer(X)
             
-            # For multi-class classification - create a global view
+            # For multi-class classification
             if self.classification and self.n_classes > 2:
-                # For multi-class, we need to create a 2D representation for beeswarm
                 if hasattr(shap_values, 'values') and len(shap_values.values.shape) == 3:
-                    # Create global explanation by taking the mean across the class dimension
-                    global_values = shap_values.values.mean(axis=2)
-                    global_exp = shap.Explanation(
-                        values=global_values,
-                        data=X.values,
-                        feature_names=X.columns.tolist()
-                    )
-                    
-                    # Create a figure first
-                    plt.figure(figsize=(12, 8))
-                    
-                    # Use the global explanation for plotting
-                    if plot_type == "beeswarm":
-                        shap.plots.beeswarm(global_exp, max_display=n_features, show=False)
+                    # Validate class_index if provided
+                    if class_index is not None:
+                        if not isinstance(class_index, int) or class_index < 0 or class_index >= self.n_classes:
+                            logger.warning(f"Invalid class_index {class_index}. Must be integer between 0 and {self.n_classes-1}. Using global view.")
+                            class_index = None
+                    
+                    if class_index is not None:
+                        # Plot for specific class
+                        class_values = shap_values.values[:, :, class_index]
+                        class_exp = shap.Explanation(
+                            values=class_values,
+                            data=X.values,
+                            feature_names=X.columns.tolist()
+                        )
+                        
+                        # Create a figure first
+                        plt.figure(figsize=(12, 8))
+                        
+                        # Use the class-specific explanation for plotting
+                        if plot_type == "beeswarm":
+                            shap.plots.beeswarm(class_exp, max_display=n_features, show=False)
+                        else:
+                            shap.plots.bar(class_exp, max_display=n_features, show=False)
+                        
+                        plt.title(f"SHAP Values for Class {class_index} ({dataset.title()} Data)")
+                        
                     else:
-                        shap.plots.bar(global_exp, max_display=n_features, show=False)
-                    
-                    plt.title(f"SHAP Values ({dataset.title()} Data)")
+                        # Create global explanation by taking the mean across the class dimension
+                        global_values = shap_values.values.mean(axis=2)
+                        global_exp = shap.Explanation(
+                            values=global_values,
+                            data=X.values,
+                            feature_names=X.columns.tolist()
+                        )
+                        
+                        # Create a figure first
+                        plt.figure(figsize=(12, 8))
+                        
+                        # Use the global explanation for plotting
+                        if plot_type == "beeswarm":
+                            shap.plots.beeswarm(global_exp, max_display=n_features, show=False)
+                        else:
+                            shap.plots.bar(global_exp, max_display=n_features, show=False)
+                        
+                        plt.title(f"SHAP Values - Global View ({dataset.title()} Data)")
                     
-                    # Handle saving
+                    # Handle saving and showing
                     if save_plot:
                         os.makedirs(plot_dir, exist_ok=True)
-                        save_path = os.path.join(plot_dir, f"{plot_name}.{plot_save_format}")
+                        if class_index is not None:
+                            save_path = os.path.join(plot_dir, f"{plot_name}_class_{class_index}.{plot_save_format}")
+                        else:
+                            save_path = os.path.join(plot_dir, f"{plot_name}.{plot_save_format}")
                         plt.savefig(save_path, format=plot_save_format, dpi=plot_save_dpi, bbox_inches="tight")
                     
                     # Show or close
@@ -2024,10 +2059,15 @@ class ModelEvaluator:
                     return
                 else:
                     # If the SHAP values don't have the expected format
+                    if class_index is not None:
+                        logger.warning("Multi-class SHAP values don't have expected format for class-specific plotting. Using global view.")
                     logger.warning("Multi-class SHAP values don't have expected format. Unable to create beeswarm plot.")
                     return
             
             # Binary classification or regression - use the original behavior
+            if class_index is not None:
+                logger.warning(f"class_index parameter ignored for {('binary classification' if self.classification else 'regression')} model.")
+            
             # Create a figure first
             plt.figure(figsize=(12, 8))
             
@@ -2056,6 +2096,8 @@ class ModelEvaluator:
             print(f"Error: {str(e)}")
 
 
+
+
     def plot_score_distribution(
         self,
         bins=None,

{gitlabds-2.1.5 → gitlabds-2.1.7}/gitlabds.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: gitlabds
-Version: 2.1.5
+Version: 2.1.7
 Summary: Gitlab Data Science and Modeling Tools
 Home-page: https://gitlab.com/gitlab-data/gitlabds
 Author: Kevin Dietz