likelihood 1.5.4__py3-none-any.whl → 1.5.6__py3-none-any.whl

likelihood/graph/nn.py

@@ -5,6 +5,7 @@ os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
 logging.getLogger("tensorflow").setLevel(logging.ERROR)
 
 import warnings
+from multiprocessing import Pool, cpu_count
 from typing import Any, List, Tuple
 
 import numpy as np
@@ -13,65 +14,79 @@ import tensorflow as tf
 from IPython.display import clear_output
 from pandas.core.frame import DataFrame
 from sklearn.metrics import f1_score
-from sklearn.model_selection import train_test_split
 
 tf.get_logger().setLevel("ERROR")
 
 from likelihood.tools import LoRALayer
 
 
-def compare_similarity(arr1: List[Any], arr2: List[Any], threshold: float = 0.05) -> int:
-    """Calculate the similarity between two arrays considering numeric values near to 1 in ratio."""
+def compare_similarity_np(arr1: np.ndarray, arr2: np.ndarray, threshold: float = 0.05) -> int:
+    """Vectorized similarity comparison between two numeric/categorical arrays."""
+    arr1 = np.asarray(arr1)
+    arr2 = np.asarray(arr2)
 
-    def is_similar(a: Any, b: Any) -> bool:
-        if isinstance(a, (int, float)) and isinstance(b, (int, float)):
-            if a == 0 and b == 0:
-                return True
-            if a == 0 or b == 0:
-                return False
-            # For numeric values, check if their ratio is within the threshold range
-            ratio = max(a, b) / min(a, b)
-            return 1 - threshold <= ratio <= 1 + threshold
-        else:
-            return a == b
+    is_numeric = np.vectorize(
+        lambda a, b: isinstance(a, (int, float)) and isinstance(b, (int, float))
+    )(arr1, arr2)
+
+    similarity = np.zeros_like(arr1, dtype=bool)
+
+    if np.any(is_numeric):
+        a_num = arr1[is_numeric].astype(float)
+        b_num = arr2[is_numeric].astype(float)
+
+        both_zero = (a_num == 0) & (b_num == 0)
+        nonzero = ~both_zero & (a_num != 0) & (b_num != 0)
+        ratio = np.zeros_like(a_num)
+        ratio[nonzero] = np.maximum(a_num[nonzero], b_num[nonzero]) / np.minimum(
+            a_num[nonzero], b_num[nonzero]
+        )
+        numeric_similar = both_zero | ((1 - threshold <= ratio) & (ratio <= 1 + threshold))
+
+        similarity[is_numeric] = numeric_similar
+
+    similarity[~is_numeric] = arr1[~is_numeric] == arr2[~is_numeric]
+
+    return np.count_nonzero(similarity)
 
-    return sum(is_similar(a, b) for a, b in zip(arr1, arr2))
+
+def compare_pair(pair, data, similarity, threshold):
+    i, j = pair
+    sim = compare_similarity_np(data[i], data[j], threshold=threshold)
+    return (i, j, 1 if sim >= similarity else 0)
 
 
 def cal_adjacency_matrix(
-    df: DataFrame, exclude_subset: List[str] = [], sparse: bool = True, **kwargs
+    df: pd.DataFrame, exclude_subset: List[str] = [], sparse: bool = True, **kwargs
 ) -> Tuple[dict, np.ndarray]:
-    """Calculates the adjacency matrix for a given DataFrame.
-    The adjacency matrix is a matrix that represents the similarity between each pair of features.
-    The similarity is calculated using the `compare_similarity` function.
-    The resulting matrix is a square matrix with the same number of rows and columns as the rows of the input DataFrame.
+    """
+    Calculates the adjacency matrix for a given DataFrame using parallel processing.
 
     Parameters
     ----------
     df : `DataFrame`
         The input DataFrame containing the features.
-    exclude_subset : `List[str]`, optional
+    exclude_subset : `List[str]`, `optional`
         A list of features to exclude from the calculation of the adjacency matrix.
-    sparse : `bool`, optional
+    sparse : `bool`, `optional`
         Whether to return a sparse matrix or a dense matrix.
     **kwargs : `dict`
         Additional keyword arguments to pass to the `compare_similarity` function.
 
-    Keyword Arguments:
-    ----------
-    similarity: `int`
-        The minimum number of features that must be the same in both arrays to be considered similar.
-    threshold : `float`
-        The threshold value used in the `compare_similarity` function. Default is 0.05.
-
     Returns
     -------
     adj_dict : `dict`
         A dictionary containing the features.
     adjacency_matrix : `ndarray`
         The adjacency matrix.
-    """
 
+    Keyword Arguments:
+    ----------
+    similarity: `int`
+        The minimum number of features that must be the same in both arrays to be considered similar.
+    threshold : `float`
+        The threshold value used in the `compare_similarity` function. Default is 0.0
+    """
     if len(exclude_subset) > 0:
         columns = [col for col in df.columns if col not in exclude_subset]
         df_ = df[columns].copy()
@@ -84,14 +99,26 @@ def cal_adjacency_matrix(
     threshold = kwargs.get("threshold", 0.05)
     assert similarity <= df_.shape[1]
 
-    adj_dict = {index: row.tolist() for index, row in df_.iterrows()}
+    data = df_.to_numpy()
+    n = len(data)
 
-    adjacency_matrix = np.zeros((len(df_), len(df_)))
+    adj_dict = {i: data[i].tolist() for i in range(n)}
 
-    for i in range(len(df_)):
-        for j in range(len(df_)):
-            if compare_similarity(adj_dict[i], adj_dict[j], threshold=threshold) >= similarity:
-                adjacency_matrix[i][j] = 1
+    def pair_generator():
+        for i in range(n):
+            for j in range(i, n):
+                yield (i, j)
+
+    with Pool(cpu_count()) as pool:
+        results = pool.starmap(
+            compare_pair, ((pair, data, similarity, threshold) for pair in pair_generator())
+        )
+
+    adjacency_matrix = np.zeros((n, n), dtype=np.uint8)
+    for i, j, val in results:
+        if val:
+            adjacency_matrix[i, j] = 1
+            adjacency_matrix[j, i] = 1
 
     if sparse:
         num_nodes = adjacency_matrix.shape[0]
@@ -103,9 +130,7 @@ def cal_adjacency_matrix(
             indices=indices, values=values, dense_shape=(num_nodes, num_nodes)
         )
 
-        return adj_dict, adjacency_matrix
-    else:
-        return adj_dict, adjacency_matrix
+    return adj_dict, adjacency_matrix
 
 
 class Data:
@@ -260,12 +285,17 @@ class VanillaGNN(tf.keras.Model):
         val_losses = []
         val_f1_scores = []
 
-        X_train, X_test, y_train, y_test = train_test_split(
-            data.x, data.y, test_size=test_size, shuffle=False
-        )
-        adjacency_train = tf.sparse.slice(data.adjacency, [0, 0], [len(X_train), len(X_train)])
+        num_nodes = len(data.x)
+        split_index = int((1 - test_size) * num_nodes)
+
+        X_train, X_test = data.x[:split_index], data.x[split_index:]
+        y_train, y_test = data.y[:split_index], data.y[split_index:]
+
+        adjacency_train = tf.sparse.slice(data.adjacency, [0, 0], [split_index, split_index])
         adjacency_test = tf.sparse.slice(
-            data.adjacency, [len(X_train), 0], [len(X_test), len(X_test)]
+            data.adjacency,
+            [split_index, split_index],
+            [num_nodes - split_index, num_nodes - split_index],
         )
 
         batch_starts = np.arange(0, len(X_train), batch_size)
@@ -286,10 +316,6 @@ class VanillaGNN(tf.keras.Model):
 
             if epoch % 5 == 0:
                 clear_output(wait=True)
-                warnings.warn(
-                    "It is normal for validation metrics to underperform during training. Use the test method to validate after training.",
-                    UserWarning,
-                )
                 val_loss, val_f1 = self.evaluate(X_test, adjacency_test, y_test)
                 val_losses.append(val_loss)
                 val_f1_scores.append(val_f1)

likelihood/models/deep/__init__.py

@@ -1,2 +1,3 @@
 from .autoencoders import *
 from .gan import *
+from .predictor import GetInsights

likelihood/models/deep/autoencoders.py

@@ -1,27 +1,17 @@
 import logging
 import os
-import random
-import warnings
 from functools import partial
 from shutil import rmtree
 
-import matplotlib
-import matplotlib.colors as mcolors
-import matplotlib.pyplot as plt
 import numpy as np
 import pandas as pd
-from pandas.plotting import radviz
 
 os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
 logging.getLogger("tensorflow").setLevel(logging.ERROR)
 
-
-from typing import List
-
 import keras_tuner
 import tensorflow as tf
 from pandas.core.frame import DataFrame
-from sklearn.manifold import TSNE
 from tensorflow.keras.layers import InputLayer
 from tensorflow.keras.regularizers import l2
 
@@ -889,448 +879,3 @@ def setup_model(
 
     best_hps = tuner.get_best_hyperparameters(1)[0].values
     return best_model, pd.DataFrame(best_hps, index=["Value"])
-
-
-class GetInsights:
-    """
-    A class to analyze the output of a neural network model, including visualizations
-    of the weights, t-SNE representation, and feature statistics.
-
-    Parameters
-    ----------
-    model : `AutoClassifier`
-        The trained model to analyze.
-    inputs : `np.ndarray`
-        The input data for analysis.
-    """
-
-    def __init__(self, model: AutoClassifier, inputs: np.ndarray) -> None:
-        """
-        Initializes the GetInsights class.
-
-        Parameters
-        ----------
-        model : `AutoClassifier`
-            The trained model to analyze.
-        inputs : `np.ndarray`
-            The input data for analysis.
-        """
-        self.inputs = inputs
-        self.model = model
-
-        self.encoder_layer = (
-            self.model.encoder.layers[1]
-            if isinstance(self.model.encoder.layers[0], InputLayer)
-            else self.model.encoder.layers[0]
-        )
-        self.decoder_layer = self.model.decoder.layers[0]
-
-        self.encoder_weights = self.encoder_layer.get_weights()[0]
-        self.decoder_weights = self.decoder_layer.get_weights()[0]
-
-        self.sorted_names = self._generate_sorted_color_names()
-
-    def _generate_sorted_color_names(self) -> list:
-        """
-        Generate sorted color names based on their HSV values.
-
-        Parameters
-        ----------
-        `None`
-
-        Returns
-        -------
-        `list` : Sorted color names.
-        """
-        colors = dict(mcolors.BASE_COLORS, **mcolors.CSS4_COLORS)
-        by_hsv = sorted(
-            (tuple(mcolors.rgb_to_hsv(mcolors.to_rgba(color)[:3])), name)
-            for name, color in colors.items()
-        )
-        sorted_names = [name for hsv, name in by_hsv if hsv[1] > 0.4 and hsv[2] >= 0.4]
-        random.shuffle(sorted_names)
-        return sorted_names
-
-    def predictor_analyzer(
-        self,
-        frac: float = None,
-        cmap: str = "viridis",
-        aspect: str = "auto",
-        highlight: bool = True,
-        **kwargs,
-    ) -> None:
-        """
-        Analyze the model's predictions and visualize data.
-
-        Parameters
-        ----------
-        frac : `float`, optional
-            Fraction of data to use for analysis (default is `None`).
-        cmap : `str`, optional
-            The colormap for visualization (default is `"viridis"`).
-        aspect : `str`, optional
-            Aspect ratio for the visualization (default is `"auto"`).
-        highlight : `bool`, optional
-            Whether to highlight the maximum weights (default is `True`).
-        **kwargs : `dict`, optional
-            Additional keyword arguments for customization.
-
-        Returns
-        -------
-        `DataFrame` : The statistical summary of the input data.
-        """
-        self._viz_weights(cmap=cmap, aspect=aspect, highlight=highlight, **kwargs)
-        inputs = self.inputs.copy()
-        inputs = self._prepare_inputs(inputs, frac)
-        y_labels = kwargs.get("y_labels", None)
-        encoded, reconstructed = self._encode_decode(inputs)
-        self._visualize_data(inputs, reconstructed, cmap, aspect)
-        self._prepare_data_for_analysis(inputs, reconstructed, encoded, y_labels)
-
-        try:
-            self._get_tsne_repr(inputs, frac)
-            self._viz_tsne_repr(c=self.classification)
-
-            self._viz_radviz(self.data, "class", "Radviz Visualization of Latent Space")
-            self._viz_radviz(self.data_input, "class", "Radviz Visualization of Input Data")
-        except ValueError:
-            warnings.warn(
-                "Some functions or processes will not be executed for regression problems.",
-                UserWarning,
-            )
-
-        return self._statistics(self.data_input)
-
-    def _prepare_inputs(self, inputs: np.ndarray, frac: float) -> np.ndarray:
-        """
-        Prepare the input data, possibly selecting a fraction of it.
-
-        Parameters
-        ----------
-        inputs : `np.ndarray`
-            The input data.
-        frac : `float`
-            Fraction of data to use.
-
-        Returns
-        -------
-        `np.ndarray` : The prepared input data.
-        """
-        if frac:
-            n = int(frac * self.inputs.shape[0])
-            indexes = np.random.choice(np.arange(inputs.shape[0]), n, replace=False)
-            inputs = inputs[indexes]
-        inputs[np.isnan(inputs)] = 0.0
-        return inputs
-
-    def _encode_decode(self, inputs: np.ndarray) -> tuple:
-        """
-        Perform encoding and decoding on the input data.
-
-        Parameters
-        ----------
-        inputs : `np.ndarray`
-            The input data.
-
-        Returns
-        -------
-        `tuple` : The encoded and reconstructed data.
-        """
-        try:
-            mean, log_var = self.model.encoder(inputs)
-            encoded = sampling(mean, log_var)
-        except:
-            encoded = self.model.encoder(inputs)
-        reconstructed = self.model.decoder(encoded)
-        return encoded, reconstructed
-
-    def _visualize_data(
-        self, inputs: np.ndarray, reconstructed: np.ndarray, cmap: str, aspect: str
-    ) -> None:
-        """
-        Visualize the original data and the reconstructed data.
-
-        Parameters
-        ----------
-        inputs : `np.ndarray`
-            The input data.
-        reconstructed : `np.ndarray`
-            The reconstructed data.
-        cmap : `str`
-            The colormap for visualization.
-        aspect : `str`
-            Aspect ratio for the visualization.
-
-        Returns
-        -------
-        `None`
-        """
-        ax = plt.subplot(1, 2, 1)
-        plt.imshow(inputs, cmap=cmap, aspect=aspect)
-        plt.colorbar()
-        plt.title("Original Data")
-
-        plt.subplot(1, 2, 2, sharex=ax, sharey=ax)
-        plt.imshow(reconstructed, cmap=cmap, aspect=aspect)
-        plt.colorbar()
-        plt.title("Decoder Layer Reconstruction")
-        plt.show()
-
-    def _prepare_data_for_analysis(
-        self,
-        inputs: np.ndarray,
-        reconstructed: np.ndarray,
-        encoded: np.ndarray,
-        y_labels: List[str],
-    ) -> None:
-        """
-        Prepare data for statistical analysis.
-
-        Parameters
-        ----------
-        inputs : `np.ndarray`
-            The input data.
-        reconstructed : `np.ndarray`
-            The reconstructed data.
-        encoded : `np.ndarray`
-            The encoded data.
-        y_labels : `List[str]`
-            The labels of features.
-
-        Returns
-        -------
-        `None`
-        """
-        self.classification = (
-            self.model.classifier(tf.concat([reconstructed, encoded], axis=1))
-            .numpy()
-            .argmax(axis=1)
-        )
-
-        self.data = pd.DataFrame(encoded, columns=[f"Feature {i}" for i in range(encoded.shape[1])])
-        self.data_input = pd.DataFrame(
-            inputs,
-            columns=(
-                [f"Feature {i}" for i in range(inputs.shape[1])] if y_labels is None else y_labels
-            ),
-        )
-
-        self.data["class"] = self.classification
-        self.data_input["class"] = self.classification
-
-    def _get_tsne_repr(self, inputs: np.ndarray = None, frac: float = None) -> None:
-        """
-        Perform t-SNE dimensionality reduction on the input data.
-
-        Parameters
-        ----------
-        inputs : `np.ndarray`
-            The input data.
-        frac : `float`
-            Fraction of data to use.
-
-        Returns
-        -------
-        `None`
-        """
-        if inputs is None:
-            inputs = self.inputs.copy()
-            if frac:
-                n = int(frac * self.inputs.shape[0])
-                indexes = np.random.choice(np.arange(inputs.shape[0]), n, replace=False)
-                inputs = inputs[indexes]
-            inputs[np.isnan(inputs)] = 0.0
-        self.latent_representations = inputs @ self.encoder_weights
-
-        tsne = TSNE(n_components=2)
-        self.reduced_data_tsne = tsne.fit_transform(self.latent_representations)
-
-    def _viz_tsne_repr(self, **kwargs) -> None:
-        """
-        Visualize the t-SNE representation of the latent space.
-
-        Parameters
-        ----------
-        **kwargs : `dict`
-            Additional keyword arguments for customization.
-
-        Returns
-        -------
-        `None`
-        """
-        c = kwargs.get("c", None)
-        self.colors = (
-            kwargs.get("colors", self.sorted_names[: len(np.unique(c))]) if c is not None else None
-        )
-
-        plt.scatter(
-            self.reduced_data_tsne[:, 0],
-            self.reduced_data_tsne[:, 1],
-            cmap=matplotlib.colors.ListedColormap(self.colors) if c is not None else None,
-            c=c,
-        )
-
-        if c is not None:
-            cb = plt.colorbar()
-            loc = np.arange(0, max(c), max(c) / float(len(self.colors)))
-            cb.set_ticks(loc)
-            cb.set_ticklabels(np.unique(c))
-
-        plt.title("t-SNE Visualization of Latent Space")
-        plt.xlabel("t-SNE 1")
-        plt.ylabel("t-SNE 2")
-        plt.show()
-
-    def _viz_radviz(self, data: pd.DataFrame, color_column: str, title: str) -> None:
-        """
-        Visualize the data using RadViz.
-
-        Parameters
-        ----------
-        data : `pd.DataFrame`
-            The data to visualize.
-        color_column : `str`
-            The column to use for coloring.
-        title : `str`
-            The title of the plot.
-
-        Returns
-        -------
-        `None`
-        """
-        data_normalized = data.copy(deep=True)
-        data_normalized.iloc[:, :-1] = (
-            2.0
-            * (data_normalized.iloc[:, :-1] - data_normalized.iloc[:, :-1].min())
-            / (data_normalized.iloc[:, :-1].max() - data_normalized.iloc[:, :-1].min())
-            - 1
-        )
-        radviz(data_normalized, color_column, color=self.colors)
-        plt.title(title)
-        plt.show()
-
-    def _viz_weights(
-        self, cmap: str = "viridis", aspect: str = "auto", highlight: bool = True, **kwargs
-    ) -> None:
-        """
-        Visualize the encoder layer weights of the model.
-
-        Parameters
-        ----------
-        cmap : `str`, optional
-            The colormap for visualization (default is `"viridis"`).
-        aspect : `str`, optional
-            Aspect ratio for the visualization (default is `"auto"`).
-        highlight : `bool`, optional
-            Whether to highlight the maximum weights (default is `True`).
-        **kwargs : `dict`, optional
-            Additional keyword arguments for customization.
-
-        Returns
-        -------
-        `None`
-        """
-        title = kwargs.get("title", "Encoder Layer Weights (Dense Layer)")
-        y_labels = kwargs.get("y_labels", None)
-        cmap_highlight = kwargs.get("cmap_highlight", "Pastel1")
-        highlight_mask = np.zeros_like(self.encoder_weights, dtype=bool)
-
-        plt.imshow(self.encoder_weights, cmap=cmap, aspect=aspect)
-        plt.colorbar()
-        plt.title(title)
-        if y_labels is not None:
-            plt.yticks(ticks=np.arange(self.encoder_weights.shape[0]), labels=y_labels)
-        if highlight:
-            for i, j in enumerate(self.encoder_weights.argmax(axis=1)):
-                highlight_mask[i, j] = True
-            plt.imshow(
-                np.ma.masked_where(~highlight_mask, self.encoder_weights),
-                cmap=cmap_highlight,
-                alpha=0.5,
-                aspect=aspect,
-            )
-        plt.show()
-
-    def _statistics(self, data_input: DataFrame) -> DataFrame:
-        """
-        Compute statistical summaries of the input data.
-
-        Parameters
-        ----------
-        data_input : `DataFrame`
-            The data to compute statistics for.
-
-        Returns
-        -------
-        `DataFrame` : The statistical summary of the input data.
-        """
-        data = data_input.copy(deep=True)
-
-        if not pd.api.types.is_string_dtype(data["class"]):
-            data["class"] = data["class"].astype(str)
-
-        data.ffill(inplace=True)
-        grouped_data = data.groupby("class")
-
-        numerical_stats = grouped_data.agg(["mean", "min", "max", "std", "median"])
-        numerical_stats.columns = ["_".join(col).strip() for col in numerical_stats.columns.values]
-
-        def get_mode(x):
-            mode_series = x.mode()
-            return mode_series.iloc[0] if not mode_series.empty else None
-
-        mode_stats = grouped_data.apply(get_mode, include_groups=False)
-        mode_stats.columns = [f"{col}_mode" for col in mode_stats.columns]
-        combined_stats = pd.concat([numerical_stats, mode_stats], axis=1)
-
-        return combined_stats.T
-
-
-########################################################################################
-
-if __name__ == "__main__":
-    # Example usage
-    import pandas as pd
-    from sklearn.datasets import load_iris
-    from sklearn.preprocessing import OneHotEncoder
-
-    # Load the dataset
-    iris = load_iris()
-
-    # Convert to a DataFrame for easy exploration
-    iris_df = pd.DataFrame(data=iris.data, columns=iris.feature_names)
-    iris_df["species"] = iris.target
-
-    X = iris_df.drop(columns="species")
-    y_labels = X.columns
-    X = X.values
-    y = iris_df["species"].values
-
-    X = np.asarray(X).astype(np.float32)
-
-    encoder = OneHotEncoder()
-    y = encoder.fit_transform(y.reshape(-1, 1)).toarray()
-    y = np.asarray(y).astype(np.float32)
-
-    model = AutoClassifier(
-        input_shape_parm=X.shape[1],
-        num_classes=3,
-        units=27,
-        activation="tanh",
-        num_layers=2,
-        dropout=0.2,
-    )
-    model.compile(
-        optimizer="adam",
-        loss=tf.keras.losses.CategoricalCrossentropy(),
-        metrics=[tf.keras.metrics.F1Score(threshold=0.5)],
-    )
-    model.fit(X, y, epochs=50, validation_split=0.2)
-
-    insights = GetInsights(model, X)
-    summary = insights.predictor_analyzer(frac=1.0, y_labels=y_labels)
-    insights._get_tsne_repr()
-    insights._viz_tsne_repr()
-    insights._viz_tsne_repr(c=iris_df["species"])
-    insights._viz_weights()
-    print(summary)