workbench 0.8.213__py3-none-any.whl → 0.8.219__py3-none-any.whl

workbench/utils/metrics_utils.py

@@ -18,10 +18,32 @@ from sklearn.metrics import (
 log = logging.getLogger("workbench")
 
 
+def validate_proba_columns(predictions_df: pd.DataFrame, class_labels: List[str], guessing: bool = False) -> bool:
+    """Validate that probability columns match class labels.
+
+    Args:
+        predictions_df: DataFrame with prediction results
+        class_labels: List of class labels
+        guessing: Whether class labels were guessed from data
+
+    Returns:
+        True if validation passes
+
+    Raises:
+        ValueError: If probability columns don't match class labels
+    """
+    proba_columns = [col.replace("_proba", "") for col in predictions_df.columns if col.endswith("_proba")]
+
+    if sorted(class_labels) != sorted(proba_columns):
+        label_type = "GUESSED class_labels" if guessing else "class_labels"
+        raise ValueError(f"_proba columns {proba_columns} != {label_type} {class_labels}!")
+    return True
+
+
 def compute_classification_metrics(
     predictions_df: pd.DataFrame,
     target_col: str,
-    class_labels: List[str],
+    class_labels: Optional[List[str]] = None,
     prediction_col: str = "prediction",
 ) -> pd.DataFrame:
     """Compute classification metrics from a predictions DataFrame.
@@ -29,26 +51,62 @@ def compute_classification_metrics(
     Args:
         predictions_df: DataFrame with target and prediction columns
         target_col: Name of the target column
-        class_labels: List of class labels in order
+        class_labels: List of class labels in order (if None, inferred from target column)
         prediction_col: Name of the prediction column (default: "prediction")
 
     Returns:
         DataFrame with per-class metrics (precision, recall, f1, roc_auc, support)
-        plus a weighted 'all' row
+        plus a weighted 'all' row. Returns empty DataFrame if validation fails.
     """
-    y_true = predictions_df[target_col]
-    y_pred = predictions_df[prediction_col]
+    # Validate inputs
+    if predictions_df.empty:
+        log.warning("Empty DataFrame provided. Returning empty metrics.")
+        return pd.DataFrame()
+
+    if prediction_col not in predictions_df.columns:
+        log.warning(f"Prediction column '{prediction_col}' not found in DataFrame. Returning empty metrics.")
+        return pd.DataFrame()
+
+    if target_col not in predictions_df.columns:
+        log.warning(f"Target column '{target_col}' not found in DataFrame. Returning empty metrics.")
+        return pd.DataFrame()
+
+    # Handle NaN predictions
+    df = predictions_df.copy()
+    nan_pred = df[prediction_col].isnull().sum()
+    if nan_pred > 0:
+        log.warning(f"Dropping {nan_pred} rows with NaN predictions.")
+        df = df[~df[prediction_col].isnull()]
+
+    if df.empty:
+        log.warning("No valid rows after dropping NaNs. Returning empty metrics.")
+        return pd.DataFrame()
+
+    # Handle class labels
+    guessing = False
+    if class_labels is None:
+        log.warning("Class labels not provided. Inferring from target column.")
+        class_labels = df[target_col].unique().tolist()
+        guessing = True
+
+    # Validate probability columns if present
+    proba_cols = [col for col in df.columns if col.endswith("_proba")]
+    if proba_cols:
+        validate_proba_columns(df, class_labels, guessing=guessing)
+
+    y_true = df[target_col]
+    y_pred = df[prediction_col]
 
     # Precision, recall, f1, support per class
     prec, rec, f1, support = precision_recall_fscore_support(y_true, y_pred, labels=class_labels, zero_division=0)
 
     # ROC AUC per class (requires probability columns and sorted labels)
-    proba_cols = [f"{label}_proba" for label in class_labels]
-    if all(col in predictions_df.columns for col in proba_cols):
+    proba_col_names = [f"{label}_proba" for label in class_labels]
+    if all(col in df.columns for col in proba_col_names):
         # roc_auc_score requires labels to be sorted, so we sort and reorder results back
         sorted_labels = sorted(class_labels)
         sorted_proba_cols = [f"{label}_proba" for label in sorted_labels]
-        y_score_sorted = predictions_df[sorted_proba_cols].values
+        y_score_sorted = df[sorted_proba_cols].values
         roc_auc_sorted = roc_auc_score(y_true, y_score_sorted, labels=sorted_labels, multi_class="ovr", average=None)
         # Map back to original class_labels order
         label_to_auc = dict(zip(sorted_labels, roc_auc_sorted))
@@ -97,9 +155,35 @@ def compute_regression_metrics(
 
     Returns:
         DataFrame with regression metrics (rmse, mae, medae, r2, spearmanr, support)
+        Returns empty DataFrame if validation fails or no valid data.
     """
-    y_true = predictions_df[target_col].values
-    y_pred = predictions_df[prediction_col].values
+    # Validate inputs
+    if predictions_df.empty:
+        log.warning("Empty DataFrame provided. Returning empty metrics.")
+        return pd.DataFrame()
+
+    if prediction_col not in predictions_df.columns:
+        log.warning(f"Prediction column '{prediction_col}' not found in DataFrame. Returning empty metrics.")
+        return pd.DataFrame()
+
+    if target_col not in predictions_df.columns:
+        log.warning(f"Target column '{target_col}' not found in DataFrame. Returning empty metrics.")
+        return pd.DataFrame()
+
+    # Handle NaN values
+    df = predictions_df[[target_col, prediction_col]].copy()
+    nan_target = df[target_col].isnull().sum()
+    nan_pred = df[prediction_col].isnull().sum()
+    if nan_target > 0 or nan_pred > 0:
+        log.warning(f"NaNs found: {target_col}={nan_target}, {prediction_col}={nan_pred}. Dropping NaN rows.")
+        df = df.dropna()
+
+    if df.empty:
+        log.warning("No valid rows after dropping NaNs. Returning empty metrics.")
+        return pd.DataFrame()
+
+    y_true = df[target_col].values
+    y_pred = df[prediction_col].values
 
     return pd.DataFrame(
         [

workbench/utils/model_utils.py

@@ -93,16 +93,17 @@ def get_custom_script_path(package: str, script_name: str) -> Path:
     return script_path
 
 
-def proximity_model_local(model: "Model"):
-    """Create a Proximity Model for this Model
+def proximity_model_local(model: "Model", include_all_columns: bool = False):
+    """Create a FeatureSpaceProximity Model for this Model
 
     Args:
         model (Model): The Model/FeatureSet used to create the proximity model
+        include_all_columns (bool): Include all DataFrame columns in neighbor results (default: False)
 
     Returns:
-        Proximity: The proximity model
+        FeatureSpaceProximity: The proximity model
     """
-    from workbench.algorithms.dataframe.proximity import Proximity  # noqa: F401 (avoid circular import)
+    from workbench.algorithms.dataframe.feature_space_proximity import FeatureSpaceProximity  # noqa: F401
     from workbench.api import Model, FeatureSet  # noqa: F401 (avoid circular import)
 
     # Get Feature and Target Columns from the existing given Model
@@ -121,8 +122,59 @@ def proximity_model_local(model: "Model"):
     model_ids = set(model_df[id_column])
     full_df["in_model"] = full_df[id_column].isin(model_ids)
 
-    # Create and return the Proximity Model
-    return Proximity(full_df, id_column, features, target, track_columns=features)
+    # Create and return the FeatureSpaceProximity Model
+    return FeatureSpaceProximity(
+        full_df, id_column=id_column, features=features, target=target, include_all_columns=include_all_columns
+    )
+
+
+def fingerprint_prox_model_local(
+    model: "Model",
+    include_all_columns: bool = False,
+    radius: int = 2,
+    n_bits: int = 1024,
+    counts: bool = False,
+):
+    """Create a FingerprintProximity Model for this Model
+
+    Args:
+        model (Model): The Model used to create the fingerprint proximity model
+        include_all_columns (bool): Include all DataFrame columns in neighbor results (default: False)
+        radius (int): Morgan fingerprint radius (default: 2)
+        n_bits (int): Number of bits for the fingerprint (default: 1024)
+        counts (bool): Use count fingerprints instead of binary (default: False)
+
+    Returns:
+        FingerprintProximity: The fingerprint proximity model
+    """
+    from workbench.algorithms.dataframe.fingerprint_proximity import FingerprintProximity  # noqa: F401
+    from workbench.api import Model, FeatureSet  # noqa: F401 (avoid circular import)
+
+    # Get Target Column from the existing given Model
+    target = model.target()
+
+    # Backtrack our FeatureSet to get the ID column
+    fs = FeatureSet(model.get_input())
+    id_column = fs.id_column
+
+    # Create the Proximity Model from both the full FeatureSet and the Model training data
+    full_df = fs.pull_dataframe()
+    model_df = model.training_view().pull_dataframe()
+
+    # Mark rows that are in the model
+    model_ids = set(model_df[id_column])
+    full_df["in_model"] = full_df[id_column].isin(model_ids)
+
+    # Create and return the FingerprintProximity Model
+    return FingerprintProximity(
+        full_df,
+        id_column=id_column,
+        target=target,
+        include_all_columns=include_all_columns,
+        radius=radius,
+        n_bits=n_bits,
+        counts=counts,
+    )
 
 
 def noise_model_local(model: "Model"):
@@ -157,13 +209,43 @@ def noise_model_local(model: "Model"):
     return NoiseModel(full_df, id_column, features, target)
 
 
-def published_proximity_model(model: "Model", prox_model_name: str, track_columns: list = None) -> "Model":
+def cleanlab_model_local(model: "Model"):
+    """Create a CleanlabModels instance for detecting data quality issues in a Model's training data.
+
+    Args:
+        model (Model): The Model used to create the cleanlab models
+
+    Returns:
+        CleanlabModels: Factory providing access to CleanLearning and Datalab models.
+            - clean_learning(): CleanLearning model with enhanced get_label_issues()
+            - datalab(): Datalab instance with report(), get_issues()
+    """
+    from workbench.algorithms.models.cleanlab_model import create_cleanlab_model  # noqa: F401 (avoid circular import)
+    from workbench.api import Model, FeatureSet  # noqa: F401 (avoid circular import)
+
+    # Get Feature and Target Columns from the existing given Model
+    features = model.features()
+    target = model.target()
+    model_type = model.model_type
+
+    # Backtrack our FeatureSet to get the ID column
+    fs = FeatureSet(model.get_input())
+    id_column = fs.id_column
+
+    # Get the full FeatureSet data
+    full_df = fs.pull_dataframe()
+
+    # Create and return the CleanLearning model
+    return create_cleanlab_model(full_df, id_column, features, target, model_type=model_type)
+
+
+def published_proximity_model(model: "Model", prox_model_name: str, include_all_columns: bool = False) -> "Model":
     """Create a published proximity model based on the given model
 
     Args:
         model (Model): The model to create the proximity model from
         prox_model_name (str): The name of the proximity model to create
-        track_columns (list, optional): List of columns to track in the proximity model
+        include_all_columns (bool): Include all DataFrame columns in results (default: False)
     Returns:
         Model: The proximity model
     """
@@ -186,7 +268,7 @@ def published_proximity_model(model: "Model", prox_model_name: str, track_column
         description=f"Proximity Model for {model.name}",
         tags=["proximity", model.name],
         custom_script=script_path,
-        custom_args={"track_columns": track_columns},
+        custom_args={"include_all_columns": include_all_columns},
     )
     return prox_model
 

workbench/utils/pytorch_utils.py

@@ -75,7 +75,7 @@ if __name__ == "__main__":
     from workbench.api import Model
 
     # Test pulling CV results
-    model_name = "aqsol-pytorch-reg"
+    model_name = "aqsol-reg-pytorch"
     print(f"Loading Workbench model: {model_name}")
     model = Model(model_name)
     print(f"Model Framework: {model.model_framework}")

workbench/utils/shap_utils.py

@@ -9,6 +9,7 @@ from typing import Optional, List, Tuple, Dict, Union
 from workbench.utils.xgboost_model_utils import xgboost_model_from_s3
 from workbench.utils.model_utils import load_category_mappings_from_s3
 from workbench.utils.pandas_utils import convert_categorical_types
+from workbench.model_script_utils.model_script_utils import decompress_features
 
 # Set up the log
 log = logging.getLogger("workbench")
@@ -111,61 +112,6 @@ def shap_values_data(
         return result_df, feature_df
 
 
-def decompress_features(
-    df: pd.DataFrame, features: List[str], compressed_features: List[str]
-) -> Tuple[pd.DataFrame, List[str]]:
-    """Prepare features for the XGBoost model
-
-    Args:
-        df (pd.DataFrame): The features DataFrame
-        features (List[str]): Full list of feature names
-        compressed_features (List[str]): List of feature names to decompress (bitstrings)
-
-    Returns:
-        pd.DataFrame: DataFrame with the decompressed features
-        List[str]: Updated list of feature names after decompression
-
-    Raises:
-        ValueError: If any missing values are found in the specified features
-    """
-
-    # Check for any missing values in the required features
-    missing_counts = df[features].isna().sum()
-    if missing_counts.any():
-        missing_features = missing_counts[missing_counts > 0]
-        print(
-            f"WARNING: Found missing values in features: {missing_features.to_dict()}. "
-            "WARNING: You might want to remove/replace all NaN values before processing."
-        )
-
-    # Decompress the specified compressed features
-    decompressed_features = features
-    for feature in compressed_features:
-        if (feature not in df.columns) or (feature not in features):
-            print(f"Feature '{feature}' not in the features list, skipping decompression.")
-            continue
-
-        # Remove the feature from the list of features to avoid duplication
-        decompressed_features.remove(feature)
-
-        # Handle all compressed features as bitstrings
-        bit_matrix = np.array([list(bitstring) for bitstring in df[feature]], dtype=np.uint8)
-        prefix = feature[:3]
-
-        # Create all new columns at once - avoids fragmentation
-        new_col_names = [f"{prefix}_{i}" for i in range(bit_matrix.shape[1])]
-        new_df = pd.DataFrame(bit_matrix, columns=new_col_names, index=df.index)
-
-        # Add to features list
-        decompressed_features.extend(new_col_names)
-
-        # Drop original column and concatenate new ones
-        df = df.drop(columns=[feature])
-        df = pd.concat([df, new_df], axis=1)
-
-    return df, decompressed_features
-
-
 def _calculate_shap_values(workbench_model, sample_df: pd.DataFrame = None):
     """
     Internal function to calculate SHAP values for Workbench Models.

workbench/web_interface/components/plugins/scatter_plot.py

@@ -420,21 +420,17 @@ if __name__ == "__main__":
     df = pd.DataFrame(data)
 
     # Get a UQ regressor model
-    # from workbench.api import Endpoint, DFStore
-    # end = Endpoint("aqsol-uq")
-    # df = end.auto_inference()
-    # DFStore().upsert("/workbench/models/aqsol-uq/auto_inference", df)
+    from workbench.api import Model
 
-    from workbench.api import DFStore
-
-    df = DFStore().get("/workbench/models/aqsol-uq-100/full_cross_fold_inference")
+    model = Model("logd-reg-xgb")
+    df = model.get_inference_predictions("full_cross_fold")
 
     # Run the Unit Test on the Plugin
     PluginUnitTest(
         ScatterPlot,
         input_data=df,
         theme="midnight_blue",
-        x="solubility",
+        x="logd",
         y="prediction",
         color="prediction_std",
         suppress_hover_display=True,

{workbench-0.8.213.dist-info → workbench-0.8.219.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: workbench
-Version: 0.8.213
+Version: 0.8.219
 Summary: Workbench: A Dashboard and Python API for creating and deploying AWS SageMaker Model Pipelines
 Author-email: SuperCowPowers LLC <support@supercowpowers.com>
 License: MIT License
@@ -47,6 +47,7 @@ Requires-Dist: cryptography>=44.0.2
 Requires-Dist: ipython>=8.37.0
 Requires-Dist: pyreadline3; sys_platform == "win32"
 Requires-Dist: scikit-learn>=1.5.2
+Requires-Dist: umap-learn>=0.5.8
 Requires-Dist: xgboost>=3.0.3
 Requires-Dist: joblib>=1.3.2
 Requires-Dist: requests>=2.26.0