workbench 0.8.202__py3-none-any.whl → 0.8.220__py3-none-any.whl

workbench/algorithms/dataframe/projection_2d.py

@@ -22,7 +22,14 @@ class Projection2D:
         self.log = logging.getLogger("workbench")
         self.projection_model = None
 
-    def fit_transform(self, input_df: pd.DataFrame, features: list = None, projection: str = "UMAP") -> pd.DataFrame:
+    def fit_transform(
+        self,
+        input_df: pd.DataFrame,
+        features: list = None,
+        feature_matrix: np.ndarray = None,
+        metric: str = "euclidean",
+        projection: str = "UMAP",
+    ) -> pd.DataFrame:
         """Fit and transform a DataFrame using the selected dimensionality reduction method.
 
         This method creates a copy of the input DataFrame, processes the specified features
@@ -32,6 +39,9 @@ class Projection2D:
         Args:
             input_df (pd.DataFrame): The DataFrame containing features to project.
             features (list, optional): List of feature column names. If None, numeric columns are auto-selected.
+            feature_matrix (np.ndarray, optional): Pre-computed feature matrix. If provided, features is ignored
+                and no scaling is applied (caller is responsible for appropriate preprocessing).
+            metric (str, optional): Distance metric for UMAP (e.g., 'euclidean', 'jaccard'). Default 'euclidean'.
             projection (str, optional): The projection to use ('UMAP', 'TSNE', 'MDS' or 'PCA'). Default 'UMAP'.
 
         Returns:
@@ -40,36 +50,44 @@ class Projection2D:
         # Create a copy of the input DataFrame
         df = input_df.copy()
 
-        # Auto-identify numeric features if none are provided
-        if features is None:
-            features = [col for col in df.select_dtypes(include="number").columns if not col.endswith("id")]
-            self.log.info(f"Auto-identified numeric features: {features}")
-
-        if len(features) < 2 or df.empty:
-            self.log.critical("At least two numeric features are required, and DataFrame must not be empty.")
-            return df
-
-        # Process a copy of the feature data for projection
-        X = df[features]
-        X = X.apply(lambda col: col.fillna(col.mean()))
-        X_scaled = StandardScaler().fit_transform(X)
+        # If a feature matrix is provided, use it directly (no scaling)
+        if feature_matrix is not None:
+            if len(feature_matrix) != len(df):
+                self.log.critical("feature_matrix length must match DataFrame length.")
+                return df
+            X_processed = feature_matrix
+        else:
+            # Auto-identify numeric features if none are provided
+            if features is None:
+                features = [col for col in df.select_dtypes(include="number").columns if not col.endswith("id")]
+                self.log.info(f"Auto-identified numeric features: {features}")
+
+            if len(features) < 2 or df.empty:
+                self.log.critical("At least two numeric features are required, and DataFrame must not be empty.")
+                return df
+
+            # Process a copy of the feature data for projection
+            X = df[features]
+            X = X.apply(lambda col: col.fillna(col.mean()))
+            X_processed = StandardScaler().fit_transform(X)
 
         # Select the projection method (using df for perplexity calculation)
-        self.projection_model = self._get_projection_model(projection, df)
+        self.projection_model = self._get_projection_model(projection, df, metric=metric)
 
-        # Apply the projection on the normalized data
-        projection_result = self.projection_model.fit_transform(X_scaled)
+        # Apply the projection on the processed data
+        projection_result = self.projection_model.fit_transform(X_processed)
         df[["x", "y"]] = projection_result
 
         # Resolve coincident points and return the new DataFrame
         return self.resolve_coincident_points(df)
 
-    def _get_projection_model(self, projection: str, df: pd.DataFrame):
+    def _get_projection_model(self, projection: str, df: pd.DataFrame, metric: str = "euclidean"):
         """Select and return the appropriate projection model.
 
         Args:
             projection (str): The projection method ('TSNE', 'MDS', 'PCA', or 'UMAP').
             df (pd.DataFrame): The DataFrame being transformed (used for computing perplexity).
+            metric (str): Distance metric for UMAP (default 'euclidean').
 
         Returns:
             A dimensionality reduction model instance.
@@ -88,8 +106,14 @@ class Projection2D:
             return PCA(n_components=2)
 
         if projection == "UMAP" and UMAP_AVAILABLE:
-            self.log.info("Projection: UMAP")
-            return umap.UMAP(n_components=2)
+            # UMAP default n_neighbors=15, adjust if dataset is smaller
+            n_neighbors = min(15, len(df) - 1)
+            if n_neighbors < 15:
+                self.log.warning(
+                    f"Dataset size ({len(df)}) smaller than default n_neighbors, using n_neighbors={n_neighbors}"
+                )
+            self.log.info(f"Projection: UMAP with metric={metric}, n_neighbors={n_neighbors}")
+            return umap.UMAP(n_components=2, metric=metric, n_neighbors=n_neighbors)
 
         self.log.warning(
             f"Projection method '{projection}' not recognized or UMAP not available. Falling back to TSNE."
@@ -118,7 +142,6 @@ class Projection2D:
 
         # Find duplicates
         duplicated = rounded.duplicated(subset=["x_round", "y_round"], keep=False)
-        print("Coincident Points found:", duplicated.sum())
         if not duplicated.any():
             return df
 

workbench/algorithms/dataframe/proximity.py

@@ -1,7 +1,6 @@
 import pandas as pd
 import numpy as np
-from sklearn.preprocessing import StandardScaler
-from sklearn.neighbors import NearestNeighbors
+from abc import ABC, abstractmethod
 from typing import List, Dict, Optional, Union
 import logging
 
@@ -9,14 +8,16 @@ import logging
 log = logging.getLogger("workbench")
 
 
-class Proximity:
+class Proximity(ABC):
+    """Abstract base class for proximity/neighbor computations."""
+
     def __init__(
         self,
         df: pd.DataFrame,
         id_column: str,
         features: List[str],
         target: Optional[str] = None,
-        track_columns: Optional[List[str]] = None,
+        include_all_columns: bool = False,
     ):
         """
         Initialize the Proximity class.
@@ -26,29 +27,61 @@ class Proximity:
             id_column: Name of the column used as the identifier.
             features: List of feature column names to be used for neighbor computations.
             target: Name of the target column. Defaults to None.
-            track_columns: Additional columns to track in results. Defaults to None.
+            include_all_columns: Include all DataFrame columns in neighbor results. Defaults to False.
         """
         self.id_column = id_column
+        self.features = features
         self.target = target
-        self.track_columns = track_columns or []
+        self.include_all_columns = include_all_columns
 
-        # Filter out non-numeric features
-        self.features = self._validate_features(df, features)
+        # Store the DataFrame (subclasses may filter/modify in _prepare_data)
+        self.df = df.copy()
 
-        # Drop NaN rows and set up DataFrame
-        self.df = df.dropna(subset=self.features).copy()
+        # Prepare data (subclasses can override)
+        self._prepare_data()
 
         # Compute target range if target is provided
         self.target_range = None
         if self.target and self.target in self.df.columns:
             self.target_range = self.df[self.target].max() - self.df[self.target].min()
 
-        # Build the proximity model
+        # Build the proximity model (subclass-specific)
         self._build_model()
 
         # Precompute landscape metrics
         self._precompute_metrics()
 
+        # Define core columns for output (subclasses can override)
+        self._set_core_columns()
+
+        # Project the data to 2D (subclass-specific)
+        self._project_2d()
+
+    def _prepare_data(self) -> None:
+        """Prepare the data before building the model. Subclasses can override."""
+        pass
+
+    def _set_core_columns(self) -> None:
+        """Set the core columns for output. Subclasses can override."""
+        self.core_columns = [self.id_column, "nn_distance", "nn_id"]
+        if self.target:
+            self.core_columns.extend([self.target, "nn_target", "nn_target_diff"])
+
+    @abstractmethod
+    def _build_model(self) -> None:
+        """Build the proximity model. Must set self.nn (NearestNeighbors instance)."""
+        pass
+
+    @abstractmethod
+    def _transform_features(self, df: pd.DataFrame) -> np.ndarray:
+        """Transform features for querying. Returns feature matrix for nearest neighbor lookup."""
+        pass
+
+    @abstractmethod
+    def _project_2d(self) -> None:
+        """Project the data to 2D for visualization. Updates self.df with 'x' and 'y' columns."""
+        pass
+
     def isolated(self, top_percent: float = 1.0) -> pd.DataFrame:
         """
         Find isolated data points based on distance to nearest neighbor.
@@ -62,7 +95,19 @@ class Proximity:
         percentile = 100 - top_percent
         threshold = np.percentile(self.df["nn_distance"], percentile)
         isolated = self.df[self.df["nn_distance"] >= threshold].copy()
-        return isolated.sort_values("nn_distance", ascending=False).reset_index(drop=True)
+        isolated = isolated.sort_values("nn_distance", ascending=False).reset_index(drop=True)
+        return isolated if self.include_all_columns else isolated[self.core_columns]
+
+    def proximity_stats(self) -> pd.DataFrame:
+        """
+        Return distribution statistics for nearest neighbor distances.
+
+        Returns:
+            DataFrame with proximity distribution statistics (count, mean, std, percentiles)
+        """
+        return (
+            self.df["nn_distance"].describe(percentiles=[0.01, 0.05, 0.1, 0.25, 0.5, 0.75, 0.9, 0.95, 0.99]).to_frame()
+        )
 
     def target_gradients(
         self,
@@ -90,7 +135,7 @@ class Proximity:
         if self.target is None:
             raise ValueError("Target column must be specified")
 
-        epsilon = 1e-5
+        epsilon = 1e-6
 
         # Phase 1: Quick filter using precomputed nearest neighbor
         candidates = self.df.copy()
@@ -111,13 +156,13 @@ class Proximity:
             threshold = np.percentile(candidates["gradient"], percentile)
             candidates = candidates[candidates["gradient"] >= threshold].copy()
 
-        # Phase 2: Verify with k-neighbor median to filter out cases where nearest neighbor is the outlier
+        # Phase 2: Verify with K-neighbor median to filter out cases where nearest neighbor is the outlier
         results = []
         for _, row in candidates.iterrows():
             cmpd_id = row[self.id_column]
             cmpd_target = row[self.target]
 
-            # Get k nearest neighbors (excluding self)
+            # Get K nearest neighbors (excluding self)
             nbrs = self.neighbors(cmpd_id, n_neighbors=k_neighbors, include_self=False)
 
             # Calculate median target of k neighbors, excluding the nearest neighbor (index 0)
@@ -146,10 +191,12 @@ class Proximity:
                 columns=[
                     self.id_column,
                     self.target,
+                    "nn_target",
+                    "nn_target_diff",
+                    "nn_distance",
+                    "gradient",
                     "neighbor_median",
                     "neighbor_median_diff",
-                    "mean_distance",
-                    "gradient",
                 ]
             )
 
@@ -188,8 +235,8 @@ class Proximity:
         query_df = self.df[self.df[self.id_column].isin(ids)]
         query_df = query_df.set_index(self.id_column).loc[ids].reset_index()
 
-        # Transform query features
-        X_query = self.scaler.transform(query_df[self.features])
+        # Transform query features (subclass-specific)
+        X_query = self._transform_features(query_df)
 
         # Get neighbors
         if radius is not None:
@@ -216,20 +263,7 @@ class Proximity:
         df_results = df_results.sort_values([self.id_column, "is_self", "distance"], ascending=[True, False, True])
         return df_results.drop("is_self", axis=1).reset_index(drop=True)
 
-    def _validate_features(self, df: pd.DataFrame, features: List[str]) -> List[str]:
-        """Remove non-numeric features and log warnings."""
-        non_numeric = [f for f in features if f not in df.select_dtypes(include=["number"]).columns]
-        if non_numeric:
-            log.warning(f"Non-numeric features {non_numeric} aren't currently supported, excluding them")
-        return [f for f in features if f not in non_numeric]
-
-    def _build_model(self) -> None:
-        """Standardize features and fit Nearest Neighbors model."""
-        self.scaler = StandardScaler()
-        X = self.scaler.fit_transform(self.df[self.features])
-        self.nn = NearestNeighbors().fit(X)
-
-    def _precompute_metrics(self, n_neighbors: int = 10) -> None:
+    def _precompute_metrics(self) -> None:
         """
         Precompute landscape metrics for all compounds.
 
@@ -243,12 +277,9 @@ class Proximity:
         """
         log.info("Precomputing proximity metrics...")
 
-        # Make sure n_neighbors isn't greater than dataset size
-        n_neighbors = min(n_neighbors, len(self.df) - 1)
-
-        # Get nearest neighbors for all points (including self)
-        X = self.scaler.transform(self.df[self.features])
-        distances, indices = self.nn.kneighbors(X, n_neighbors=2)  # Just need nearest neighbor
+        # Get nearest neighbors for all points (n=2 because index 0 is self)
+        X = self._transform_features(self.df)
+        distances, indices = self.nn.kneighbors(X, n_neighbors=2)
 
         # Extract nearest neighbor (index 1, since index 0 is self)
         self.df["nn_distance"] = distances[:, 1]
@@ -285,126 +316,23 @@ class Proximity:
         result = {
             self.id_column: query_id,
             "neighbor_id": neighbor_id,
-            "distance": 0.0 if distance < 1e-5 else distance,
+            "distance": 0.0 if distance < 1e-6 else distance,
         }
 
         # Add target if present
         if self.target and self.target in self.df.columns:
             result[self.target] = neighbor_row[self.target]
 
-        # Add tracked columns
-        for col in self.track_columns:
-            if col in self.df.columns:
-                result[col] = neighbor_row[col]
-
         # Add prediction/probability columns if they exist
         for col in self.df.columns:
             if col == "prediction" or "_proba" in col or "residual" in col or col == "in_model":
                 result[col] = neighbor_row[col]
 
-        return result
-
+        # Include all columns if requested
+        if self.include_all_columns:
+            result.update(neighbor_row.to_dict())
+            # Restore query_id after update (neighbor_row may have overwritten id column)
+            result[self.id_column] = query_id
+            result["neighbor_id"] = neighbor_id
 
-# Testing the Proximity class
-if __name__ == "__main__":
-
-    pd.set_option("display.max_columns", None)
-    pd.set_option("display.width", 1000)
-
-    # Create a sample DataFrame
-    data = {
-        "ID": [1, 2, 3, 4, 5],
-        "Feature1": [0.1, 0.2, 0.3, 0.4, 0.5],
-        "Feature2": [0.5, 0.4, 0.3, 0.2, 0.1],
-        "Feature3": [2.5, 2.4, 2.3, 2.3, np.nan],
-    }
-    df = pd.DataFrame(data)
-
-    # Test the Proximity class
-    features = ["Feature1", "Feature2", "Feature3"]
-    prox = Proximity(df, id_column="ID", features=features)
-    print(prox.neighbors(1, n_neighbors=2))
-
-    # Test the neighbors method with radius
-    print(prox.neighbors(1, radius=2.0))
-
-    # Test with Features list
-    prox = Proximity(df, id_column="ID", features=["Feature1"])
-    print(prox.neighbors(1))
-
-    # Create a sample DataFrame
-    data = {
-        "foo_id": ["a", "b", "c", "d", "e"],  # Testing string IDs
-        "Feature1": [0.1, 0.2, 0.3, 0.4, 0.5],
-        "Feature2": [0.5, 0.4, 0.3, 0.2, 0.1],
-        "target": [1, 0, 1, 0, 5],
-    }
-    df = pd.DataFrame(data)
-
-    # Test with String Ids
-    prox = Proximity(
-        df,
-        id_column="foo_id",
-        features=["Feature1", "Feature2"],
-        target="target",
-        track_columns=["Feature1", "Feature2"],
-    )
-    print(prox.neighbors(["a", "b"]))
-
-    # Test duplicate IDs
-    data = {
-        "foo_id": ["a", "b", "c", "d", "d"],  # Duplicate ID (d)
-        "Feature1": [0.1, 0.2, 0.3, 0.4, 0.5],
-        "Feature2": [0.5, 0.4, 0.3, 0.2, 0.1],
-        "target": [1, 0, 1, 0, 5],
-    }
-    df = pd.DataFrame(data)
-    prox = Proximity(df, id_column="foo_id", features=["Feature1", "Feature2"], target="target")
-    print(df.equals(prox.df))
-
-    # Test with a categorical feature
-    from workbench.api import FeatureSet, Model
-
-    fs = FeatureSet("aqsol_features")
-    model = Model("aqsol-regression")
-    features = model.features()
-    df = fs.pull_dataframe()
-    prox = Proximity(
-        df, id_column=fs.id_column, features=model.features(), target=model.target(), track_columns=features
-    )
-    print(prox.neighbors(df[fs.id_column].tolist()[:3]))
-
-    print("\n" + "=" * 80)
-    print("Testing isolated_compounds...")
-    print("=" * 80)
-
-    # Test isolated data in the top 1%
-    isolated_1pct = prox.isolated(top_percent=1.0)
-    print(f"\nTop 1% most isolated compounds (n={len(isolated_1pct)}):")
-    print(isolated_1pct[[fs.id_column, "nn_distance", "nn_id"]].head(10))
-
-    # Test isolated data in the top 5%
-    isolated_5pct = prox.isolated(top_percent=5.0)
-    print(f"\nTop 5% most isolated compounds (n={len(isolated_5pct)}):")
-    print(isolated_5pct[[fs.id_column, "nn_distance", "nn_id"]].head(10))
-
-    print("\n" + "=" * 80)
-    print("Testing target_gradients...")
-    print("=" * 80)
-
-    # Test with different parameters
-    gradients_1pct = prox.target_gradients(top_percent=1.0, min_delta=1.0)
-    print(f"\nTop 1% target gradients (min_delta=5.0) (n={len(gradients_1pct)}):")
-    print(
-        gradients_1pct[
-            [fs.id_column, model.target(), "neighbor_median", "neighbor_median_diff", "mean_distance", "gradient"]
-        ].head(10)
-    )
-
-    gradients_5pct = prox.target_gradients(top_percent=5.0, min_delta=5.0)
-    print(f"\nTop 5% target gradients (min_delta=5.0) (n={len(gradients_5pct)}):")
-    print(
-        gradients_5pct[
-            [fs.id_column, model.target(), "neighbor_median", "neighbor_median_diff", "mean_distance", "gradient"]
-        ].head(10)
-    )
+        return result

workbench/algorithms/graph/light/proximity_graph.py

@@ -4,7 +4,7 @@ from typing import Union
 import logging
 
 # Workbench Imports
-from workbench.algorithms.dataframe import Proximity
+from workbench.algorithms.dataframe.proximity import Proximity
 from workbench.api.graph_store import GraphStore
 
 # Set up logging
@@ -132,7 +132,7 @@ class ProximityGraph:
 
 
 if __name__ == "__main__":
-    from workbench.algorithms.dataframe.proximity import Proximity
+    from workbench.algorithms.dataframe.feature_space_proximity import FeatureSpaceProximity
     from workbench.algorithms.dataframe.fingerprint_proximity import FingerprintProximity
     from workbench.web_interface.components.plugins.graph_plot import GraphPlot
     from workbench.api import DFStore
@@ -157,9 +157,9 @@ if __name__ == "__main__":
     }
     feature_df = pd.DataFrame(feature_data)
 
-    # Build a graph using the base Proximity class
-    print("\n--- Proximity Class ---")
-    prox = Proximity(feature_df, id_column="id", features=["Feature1", "Feature2"], target="target")
+    # Build a graph using FeatureSpaceProximity
+    print("\n--- FeatureSpaceProximity Class ---")
+    prox = FeatureSpaceProximity(feature_df, id_column="id", features=["Feature1", "Feature2"], target="target")
     feature_graph = ProximityGraph()
     feature_graph.build_graph(prox)
     nx_graph = feature_graph.nx_graph