gitlabds 2.1.5__tar.gz → 2.1.6__tar.gz

{gitlabds-2.1.5 → gitlabds-2.1.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: gitlabds
-Version: 2.1.5
+Version: 2.1.6
 Summary: Gitlab Data Science and Modeling Tools
 Home-page: https://gitlab.com/gitlab-data/gitlabds
 Author: Kevin Dietz

{gitlabds-2.1.5 → gitlabds-2.1.6}/gitlabds/dummy.py

@@ -50,6 +50,10 @@ def dummy_code(
         - The transformed DataFrame with dummy-coded columns
         - Dictionary of dummy coding information that can be used with apply_dummy()
     """
+    import pandas as pd
+    import warnings
+    from typing import Dict, List, Any, Union, Optional, Tuple
+    
     # Parameter validation
     if not isinstance(df, pd.DataFrame):
         raise TypeError("df must be a pandas DataFrame")
@@ -103,7 +107,6 @@ def dummy_code(
         # Determine number of levels for each field
         cat_levels = df[var_list].select_dtypes(include="object").nunique(dropna=True, axis=0)
         
-        
         # Filter columns by level count criteria
         eligible_cat_cols = cat_levels[(cat_levels <= categorical_max_levels) & (cat_levels > 1)]
         
@@ -156,7 +159,17 @@ def dummy_code(
         for col in num_columns:
             # Store unique values for later use
             values = sorted(df[col].dropna().unique().tolist())
-            dummy_info[col] = values
+            
+            # create column names with float format for numeric data
+            formatted_values = []
+            for v in values:
+                if isinstance(v, (int, float)):
+                    # Store as float to match get_dummies column naming
+                    formatted_values.append(float(v))
+                else:
+                    formatted_values.append(v)
+            
+            dummy_info[col] = formatted_values
         
         # Perform dummy coding if we have columns to encode
         if num_columns:
@@ -222,6 +235,10 @@ def dummy_top(
         - The transformed DataFrame with dummy-coded columns
         - Dictionary of dummy coding information that can be used with apply_dummy()
     """
+    import pandas as pd
+    import warnings
+    from typing import Dict, List, Any, Union, Optional, Tuple
+    
     # Parameter validation
     if not isinstance(df, pd.DataFrame):
         raise TypeError("df must be a pandas DataFrame")
@@ -279,6 +296,10 @@ def dummy_top(
         else:
             print("No categorical columns found")
 
+    # Collect all dummy columns to create at once
+    all_dummy_cols = {}
+    columns_to_drop = []
+
     # Create dummy codes for categorical fields that exceed min_threshold
     for col in cat_columns:
         # Calculate value frequencies
@@ -300,18 +321,32 @@ def dummy_top(
                 print(f"No levels exceed threshold for column '{col}'")
             continue
         
-        # Store information for this column
-        dummy_info[col] = high_freq_list
+        # For numeric columns, ensure float format to match what get_dummies creates
+        if pd.api.types.is_numeric_dtype(df[col]):
+            formatted_values = [float(v) if isinstance(v, (int, float)) else v for v in high_freq_list]
+        else:
+            formatted_values = high_freq_list
+            
+        dummy_info[col] = formatted_values
         
         # Create dummies for values that exceed threshold
         for value in high_freq_list:
             dummy_name = f"{col}{prefix_sep}{value}"
             # Create dummy column (1 if equal to value, 0 otherwise)
-            work_df[dummy_name] = (work_df[col] == value).astype(int)
+            all_dummy_cols[dummy_name] = (work_df[col] == value).astype(int)
         
-        # Drop categorical field if requested
+        # Mark for dropping if requested
         if drop_categorical:
-            work_df = work_df.drop(columns=[col])
+            columns_to_drop.append(col)
+
+    # Create all dummy columns at once using concat (prevents fragmentation)
+    if all_dummy_cols:
+        dummy_df = pd.DataFrame(all_dummy_cols, index=work_df.index)
+        work_df = pd.concat([work_df, dummy_df], axis=1)
+
+    # Drop categorical fields if requested
+    if columns_to_drop:
+        work_df = work_df.drop(columns=columns_to_drop)
 
     # Store the prefix_sep in the dummy_info to use with apply_dummy
     dummy_info["_config"] = {"prefix_sep": prefix_sep}
@@ -319,11 +354,12 @@ def dummy_top(
     return work_df, dummy_info
 
 
+
 def apply_dummy(
     df: pd.DataFrame, 
     dummy_info: Dict[str, List[Any]],
     drop_original: bool = False,
-
+    dummy_na: bool = False,
 ) -> pd.DataFrame:
     """
     Apply dummy coding to a new dataframe using the information from a previous
@@ -338,12 +374,18 @@ def apply_dummy(
         dummy_code() or dummy_top().
     drop_original : bool, default=False
         Whether to drop the original columns after dummy coding.
+    dummy_na : bool, default=False
+        Whether to create dummy variables for NA values.
     
     Returns
     -------
     pd.DataFrame
         Transformed dataframe with dummy-coded columns.
     """
+    import pandas as pd
+    import warnings
+    from typing import Dict, List, Any
+    
     # Validate input
     if not isinstance(df, pd.DataFrame):
         raise TypeError("df must be a pandas DataFrame")
@@ -353,43 +395,78 @@ def apply_dummy(
         
     if df.empty:
         raise ValueError("DataFrame is empty")
-
+    
     # Get the prefix_sep from dummy_info or use default
     prefix_sep = "_dummy_"
+    processing_dummy_info = dummy_info.copy()
+    
     if "_config" in dummy_info and "prefix_sep" in dummy_info["_config"]:
         prefix_sep = dummy_info["_config"]["prefix_sep"]
         # Remove _config from processing
-        dummy_info = {k: v for k, v in dummy_info.items() if k != "_config"}
+        processing_dummy_info = {k: v for k, v in dummy_info.items() if k != "_config"}
     
     # Create a copy to avoid modifying the original
     work_df = df.copy(deep=True)
     
+    # Collect all new dummy columns in a dictionary
+    new_dummy_cols = {}
+    columns_to_drop = []
+    
     # Process each column in the dummy_info dictionary
-    for col, values in dummy_info.items():
+    for col, values in processing_dummy_info.items():
         if col not in work_df.columns:
             warnings.warn(f"Column '{col}' not found in dataframe and will be skipped")
             continue
-            
+        
         # Create dummies for each value
         for value in values:
             dummy_name = f"{col}{prefix_sep}{value}"
             
-            # Handle numeric vs string values appropriately
-            if pd.api.types.is_numeric_dtype(type(value)):
-                work_df[dummy_name] = (work_df[col] == value).astype(int)
+            # Handle type mismatches between config and actual data
+            # Try both the original value and converted versions
+            if pd.api.types.is_numeric_dtype(work_df[col]):
+                # For numeric columns, try both int and float versions
+                try:
+                    # First try exact match
+                    mask = (work_df[col] == value)
+                    
+                    # If no matches and value types differ, try conversion
+                    if not mask.any():
+                        if isinstance(value, float) and work_df[col].dtype in ['int64', 'int32', 'int8']:
+                            # Config expects float (1.0) but data is int (1)
+                            mask = (work_df[col] == int(value))
+                        elif isinstance(value, int) and work_df[col].dtype in ['float64', 'float32']:
+                            # Config expects int (1) but data is float (1.0)
+                            mask = (work_df[col] == float(value))
+                    
+                    new_dummy_cols[dummy_name] = mask.astype(int)
+                    
+                except (ValueError, TypeError):
+                    # Fallback to string comparison if numeric conversion fails
+                    new_dummy_cols[dummy_name] = (work_df[col].astype(str) == str(value)).astype(int)
             else:
-                work_df[dummy_name] = (work_df[col] == value).astype(int)
+                # For non-numeric columns, use direct comparison
+                new_dummy_cols[dummy_name] = (work_df[col] == value).astype(int)
             
         # Create dummy for NA values if requested
         if dummy_na:
-            dummy_name = f"{col}_dummy_nan"
-            work_df[dummy_name] = work_df[col].isna().astype(int)
+            dummy_name = f"{col}{prefix_sep}nan"
+            new_dummy_cols[dummy_name] = work_df[col].isna().astype(int)
             
-        # Drop the original column if requested
+        # Mark column for dropping if requested
         if drop_original:
             # For categorical columns (from dummy_code) or any column (from dummy_top)
             # if its object type or explicitly requested
             if pd.api.types.is_object_dtype(work_df[col]) or drop_original:
-                work_df = work_df.drop(columns=[col])
+                columns_to_drop.append(col)
+    
+    # Create all dummy columns at once using concat
+    if new_dummy_cols:
+        dummy_df = pd.DataFrame(new_dummy_cols, index=work_df.index)
+        work_df = pd.concat([work_df, dummy_df], axis=1)
+    
+    # Drop original columns if requested
+    if columns_to_drop:
+        work_df = work_df.drop(columns=columns_to_drop)
     
     return work_df

{gitlabds-2.1.5 → gitlabds-2.1.6}/gitlabds/model_evaluator.py

@@ -965,7 +965,7 @@ class ModelEvaluator:
                     # Calculate metrics efficiently
                     class_lift[f"class_{class_label}_total"] = grouped.size()
                     class_lift[f"class_{class_label}_count"] = grouped.apply(
-                        lambda x: sum(x["actual"] == class_label)
+                        lambda x: sum(x["actual"] == class_label), include_groups=False
                     )
                     class_lift[f"class_{class_label}_accuracy"] = (
                         class_lift[f"class_{class_label}_count"] / 

{gitlabds-2.1.5 → gitlabds-2.1.6}/gitlabds.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: gitlabds
-Version: 2.1.5
+Version: 2.1.6
 Summary: Gitlab Data Science and Modeling Tools
 Home-page: https://gitlab.com/gitlab-data/gitlabds
 Author: Kevin Dietz