recnexteval 0.1.0__py3-none-any.whl

recnexteval/matrix/prediction_matrix.py

@@ -0,0 +1,153 @@
+import logging
+from warnings import warn
+
+import numpy as np
+import pandas as pd
+
+from .interaction_matrix import InteractionMatrix
+
+
+logger = logging.getLogger(__name__)
+
+
+class PredictionMatrix(InteractionMatrix):
+    @classmethod
+    def from_interaction_matrix(cls, im: InteractionMatrix) -> "PredictionMatrix":
+        """Create a PredictionMatrix from an InteractionMatrix.
+
+        :param im: The InteractionMatrix to convert.
+        :type im: InteractionMatrix
+        :return: A new PredictionMatrix with the same data.
+        :rtype: PredictionMatrix
+        """
+        return cls(
+            df=im._df,
+            item_ix=im.ITEM_IX,
+            user_ix=im.USER_IX,
+            timestamp_ix=im.TIMESTAMP_IX,
+            shape=getattr(im, 'shape', None),
+            skip_df_processing=True,
+        )
+
+    def mask_user_item_shape(
+        self,
+        shape: None | tuple[int, int] = None,
+        drop_unknown_user: bool = False,
+        drop_unknown_item: bool = False,
+        inherit_max_id: bool = False,
+    ) -> None:
+        """Masks global user and item ID.
+
+        To ensure released matrix released to the models only contains data
+        that is intended to be released. This addresses the data leakage issue.
+        It is recommended that the programmer defines the shape of the matrix
+        such that the model only sees the data that is intended to be seen.
+
+        =======
+        Example
+        =======
+
+        Given the following case where the data is as follows::
+
+            > uid: [0, 1, 2, 3, 4, 5]
+            > iid: [0, 1, 2, 3, -1, -1]
+            > ts : [0, 1, 2, 3, 4, 6]
+
+        Where user 4, 5 is the user to be predicted. Assuming that user 4, 5 is an
+        unknown user, that is, the model has never seen user 4, 5 before. The shape
+        of the matrix should be (4, 4). This should be defined when calling the
+        function in :param:`shape`.
+
+        If the shape is defined, and it contains ID of unknown user/item, a warning
+        will be raised if :attr:`drop_unknown` is set to False. If :attr:`drop_unknown`
+        is set to True, the unknown user/item will be dropped from the data. All
+        user/item ID greater than `shape[0]` will be dropped. This follows from
+        the initial assumption that the user/item ID starts from 0 as defined in
+        the dataset class.
+
+        Else, in the event that :param:`shape` is not defined, the shape will be
+        inferred from the data. The shape will be determined by the number of
+        unique users/items. In this case the shape will be (5, 4). Note that the
+        shape may not be as intended by the programmer if the data contains
+        unknown users/items or if the dataframe does not contain all historical
+        users/items.
+
+        :param shape: Shape of the known user and item base. This value is
+            usually set by the evaluator during the evaluation run. This value
+            can also be set manually but the programmer if there is a need to
+            alter the known user/item base. Defaults to None
+        :type shape: Optional[tuple[int, int]], optional
+        :param drop_unknown_user: To drop unknown users in the dataset,
+            defaults to False
+        :type drop_unknown_user: bool, optional
+        :param drop_unknown_item: To drop unknown items in the dataset,
+            defaults to False
+        :type drop_unknown_item: bool, optional
+        :param inherit_max_id: To inherit the maximum user and item ID from the
+            given shape and the dataframe. This is useful when the shape is
+            defined and the dataframe contains unknown users/items. Defaults to False
+        :type inherit_max_id: bool, optional
+        """
+
+        if not shape:
+            # infer shape from the data
+            known_user = np.nan_to_num(self._df[self._df != -1][InteractionMatrix.USER_IX].max(), nan=-1)
+            known_item = np.nan_to_num(self._df[self._df != -1][InteractionMatrix.ITEM_IX].max(), nan=-1)
+            self.user_item_shape = (known_user, known_item)
+            logger.debug(f"(user x item) shape inferred is {self.user_item_shape}")
+            if known_user == -1 or known_item == -1:
+                warn(
+                    "One of the dimensions of the shape cannot be inferred from the data. "
+                    "Call mask_shape() with shape parameter.",
+                    stacklevel=2,
+                )
+            return
+
+        logger.debug(
+            f"(user x item) shape defined is {shape}. "
+            f"Shape of dataframe stored in matrix was {self._df.shape} before masking"
+        )
+        if drop_unknown_user:
+            logger.debug("Dropping unknown users from interaction matrix based on defined shape")
+            self._df = pd.DataFrame(self._df[self._df[InteractionMatrix.USER_IX] < shape[0]])
+        if drop_unknown_item:
+            logger.debug("Dropping unknown items from interaction matrix based on defined shape")
+            self._df = pd.DataFrame(self._df[self._df[InteractionMatrix.ITEM_IX] < shape[1]])
+        logger.debug(f"Shape of dataframe stored in matrix is now {self._df.shape} after masking")
+
+        if inherit_max_id:
+            # we are only concerned about the absolute maximum id in the data regardless if its unknown
+            known_user = int(self._df[InteractionMatrix.USER_IX].max())
+            known_item = int(self._df[InteractionMatrix.ITEM_IX].max())
+            # + 1 as id starts from 0
+            self.user_item_shape = (max(shape[0], known_user + 1), max(shape[1], known_item + 1))
+        else:
+            self.user_item_shape = shape
+        logger.debug(f"Final (user x item) shape defined is {self.user_item_shape}")
+        self._check_user_item_shape()
+
+    def _check_user_item_shape(self) -> None:
+        if not hasattr(self, "user_item_shape"):
+            raise AttributeError("InteractionMatrix has no `user_item_shape` attribute. Please call mask_shape() first.")
+        if self.user_item_shape[0] is None or self.user_item_shape[1] is None:
+            raise ValueError("Shape must be defined.")
+
+        valid_df = self._df[self._df != -1]
+        req_rows = valid_df[InteractionMatrix.USER_IX].max()
+        req_cols = np.nan_to_num(valid_df[InteractionMatrix.ITEM_IX].max(), nan=-1)
+
+        if self.user_item_shape[0] < req_rows or self.user_item_shape[1] < req_cols:
+            logger.warning(
+                "InteractionMatrix shape mismatch detected. "
+                "Current shape: %s. Required minimum: (%s, %s). "
+                "Data loss may occur.",
+                self.user_item_shape,
+                req_rows,
+                req_cols,
+            )
+            warn(
+                "Provided shape does not match known id; there are id that are out of bounds. "
+                "Call mask_shape(drop_unknown=True) to drop unknown users and items.",
+                category=UserWarning,
+                stacklevel=2,
+            )

recnexteval/matrix/util.py

@@ -0,0 +1,24 @@
+from scipy.sparse import csr_matrix
+
+from recnexteval.matrix.interaction_matrix import InteractionMatrix
+from recnexteval.utils.util import to_binary
+
+
+def to_csr_matrix(X: InteractionMatrix | csr_matrix, binary: bool = False) -> csr_matrix:
+    """Convert a matrix-like object to a scipy csr_matrix.
+
+    :param X: Matrix-like object or tuple of objects to convert.
+    :type X: csr_matrix
+    :param binary: If true, ensure matrix is binary by setting non-zero values to 1.
+    :type binary: bool, optional
+    :return: Matrices as csr_matrix.
+    :rtype: Union[csr_matrix, Tuple[csr_matrix, ...]]
+    """
+
+    if isinstance(X, csr_matrix):
+        res = X
+    elif isinstance(X, InteractionMatrix):
+        res = X.values
+    else:
+        raise AttributeError("Not supported Matrix conversion")
+    return to_binary(res) if binary else res

recnexteval/metrics/__init__.py

@@ -0,0 +1,57 @@
+"""Metrics module for evaluating recommender system performance.
+
+This module provides a collection of metrics for evaluating the performance of
+recommender systems in streaming environments. Metrics are implemented as classes
+that inherit from the `Metric` base class, allowing for easy extension and customization.
+
+## Available Metrics
+
+The following metrics are currently available:
+
+- `PrecisionK`: Precision at K
+- `RecallK`: Recall at K
+- `DCGK`: Discounted Cumulative Gain at K
+- `NDCGK`: Normalized Discounted Cumulative Gain at K
+- `HitK`: Hit Rate at K
+
+## Using Metrics
+
+To use a metric, simply instantiate the corresponding class and call the `evaluate` method
+with the predicted and ground truth rankings:
+
+```python
+from recnexteval.metrics import PrecisionK
+
+metric = PrecisionK(k=10)
+score = metric.evaluate(
+    predicted_ranking, ground_truth_ranking
+)
+```
+
+The `evaluate` method returns a single float value representing the metric score.
+
+## Extending the Framework
+
+To add custom metrics, inherit from the `Metric` base class and implement the `evaluate` method.
+Refer to the base class documentation for implementation details.
+
+# Related Modules
+
+- recnexteval.evaluators: Evaluator classes for running metrics over data streams
+"""
+
+from .binary import HitK
+from .core import ListwiseMetricK, Metric, MetricTopK
+from .ranking import DCGK, NDCGK, PrecisionK, RecallK
+
+
+__all__ = [
+    "Metric",
+    "PrecisionK",
+    "RecallK",
+    "DCGK",
+    "NDCGK",
+    "HitK",
+    "ListwiseMetricK",
+    "MetricTopK",
+]

recnexteval/metrics/binary/__init__.py

@@ -0,0 +1,4 @@
+from .hit import HitK
+
+
+__all__ = ["HitK"]

recnexteval/metrics/binary/hit.py

@@ -0,0 +1,49 @@
+# Adopted from RecPack, An Experimentation Toolkit for Top-N Recommendation
+# Copyright (C) 2020  Froomle N.V.
+# License: GNU AGPLv3 - https://gitlab.com/recpack-maintainers/recpack/-/blob/master/LICENSE
+# Author:
+#   Lien Michiels
+#   Robin Verachtert
+
+import logging
+
+from scipy.sparse import csr_matrix, lil_matrix
+
+from ..core.elementwise_top_k import ElementwiseMetricK
+
+
+logger = logging.getLogger(__name__)
+
+
+class HitK(ElementwiseMetricK):
+    """Computes the number of hits in a list of Top-K recommendations.
+
+    A hit is counted when a recommended item in the top K for this user was interacted with.
+
+    Detailed :attr:`results` show which of the items in the list of Top-K recommended items
+    were hits and which were not.
+
+
+    :param K: Size of the recommendation list consisting of the Top-K item predictions.
+    :type K: int
+
+    This code is adapted from RecPack :cite:`recpack`
+    """
+    IS_BASE: bool = False
+
+    def _calculate(self, y_true: csr_matrix, y_pred: csr_matrix) -> None:
+        # log number of users and ground truth interactions
+        logger.debug(f"HitK compute started - {self.name}")
+        logger.debug(f"Number of users: {y_true.shape[0]}")
+        logger.debug(f"Number of ground truth interactions: {y_true.nnz}")
+
+        scores = lil_matrix(y_pred.shape)
+
+        # Elementwise multiplication of top K predicts and true interactions
+        scores[y_pred.multiply(y_true).astype(bool)] = 1
+
+        scores = scores.tocsr()
+        binary_score = (scores.sum(axis=1) >= 1).astype(int)
+        self._scores = csr_matrix(binary_score)
+
+        logger.debug(f"HitK compute complete - {self.name}")

recnexteval/metrics/core/__init__.py

@@ -0,0 +1,10 @@
+from .base import Metric
+from .listwise_top_k import ListwiseMetricK
+from .top_k import MetricTopK
+
+
+__all__ = [
+    "Metric",
+    "MetricTopK",
+    "ListwiseMetricK",
+]

recnexteval/metrics/core/base.py

@@ -0,0 +1,126 @@
+import logging
+from abc import abstractmethod
+from warnings import warn
+
+import numpy as np
+from scipy.sparse import csr_matrix
+
+from ...algorithms.utils import get_top_K_ranks
+from ...models import BaseModel, ParamMixin
+
+
+logger = logging.getLogger(__name__)
+
+
+class Metric(BaseModel, ParamMixin):
+    """Base class for all metrics.
+
+    A Metric object is stateful, i.e. after `calculate`
+    the results can be retrieved in one of two ways:
+      - Detailed results are stored in :attr:`results`,
+      - Aggregated result value can be retrieved using :attr:`value`
+    """
+
+    _scores: None | csr_matrix
+    _user_id_map: np.ndarray
+    _y_true: csr_matrix
+    _y_pred: csr_matrix
+    _user_id_sequence_array: np.ndarray
+    """Sequence of user IDs in the evaluation data."""
+    _num_users: int
+    _true_positive: int
+    """Number of true positives computed. Used for caching to obtain macro results."""
+    _false_negative: int
+    """Number of false negatives computed. Used for caching to obtain macro results."""
+    _false_positive: int
+    """Number of false positives computed. Used for caching to obtain macro results."""
+
+    def __init__(
+        self,
+        user_id_sequence_array: np.ndarray,
+        user_item_shape: tuple[int, int],
+        timestamp_limit: None | int = None,
+    ) -> None:
+        self._user_id_sequence_array = user_id_sequence_array
+        self._num_users, self._num_items = user_item_shape
+        self._timestamp_limit: None | int = timestamp_limit
+
+    @property
+    def _is_computed(self) -> bool:
+        """Whether the metric has been computed."""
+        return hasattr(self, "_scores")
+
+    def get_params(self) -> dict[str, int | None]:
+        """Get the parameters of the metric."""
+        if not self.is_time_aware:
+            return {}
+        return {"timestamp_limit": self._timestamp_limit}
+
+    @property
+    def micro_result(self) -> dict[str, np.ndarray]:
+        """Micro results for the metric.
+
+        :return: Detailed results for the metric.
+        :rtype: dict[str, np.ndarray]
+        """
+        return {"score": np.array(self.macro_result)}
+
+    @property
+    def macro_result(self) -> None | float:
+        """The global metric value."""
+        raise NotImplementedError()
+
+    @property
+    def is_time_aware(self) -> bool:
+        """Whether the metric is time-aware."""
+        return self._timestamp_limit is not None
+
+    @property
+    def timestamp_limit(self) -> int:
+        """The timestamp limit for the metric."""
+        if not self.is_time_aware or self._timestamp_limit is None:
+            raise ValueError("This metric is not time-aware.")
+        return self._timestamp_limit
+
+    @property
+    def num_items(self) -> int:
+        """Dimension of the item-space in both `y_true` and `y_pred`"""
+        return self._num_items
+
+    @property
+    def num_users(self) -> int:
+        """Dimension of the user-space in both `y_true` and `y_pred`
+        after elimination of users without interactions in `y_true`.
+        """
+        return self._num_users
+
+    def _prepare_matrix(
+        self, y_true: csr_matrix, y_pred: csr_matrix
+    ) -> tuple[csr_matrix, csr_matrix]:
+        """Prepare the matrices for the metric calculation.
+
+        This method is used to prepare the matrices for the metric calculation.
+        It is used to eliminate empty users and to set the shape of the matrices.
+        """
+        if not y_true.shape == y_pred.shape:
+            raise AssertionError(
+                f"Shape mismatch between y_true: {y_true.shape} and y_pred: {y_pred.shape}"
+            )
+        self._set_shape(y_true=y_true)
+        return y_true, y_pred
+
+    @abstractmethod
+    def _calculate(self, y_true: csr_matrix, y_pred: csr_matrix) -> None:
+        raise NotImplementedError()
+
+    def calculate(self, y_true: csr_matrix, y_pred: csr_matrix) -> None:
+        """Calculates this metric for all nonzero users in `y_true`,
+        given true labels and predicted scores.
+        """
+        y_true, y_pred = self._prepare_matrix(y_true, y_pred)
+        self._calculate(y_true, y_pred)
+
+    def _set_shape(self, y_true: csr_matrix) -> None:
+        """Set the number of users and items based on the shape of y_true.
+        """
+        self._num_users, self._num_items = y_true.shape  # type: ignore

recnexteval/metrics/core/elementwise_top_k.py

@@ -0,0 +1,75 @@
+import logging
+from warnings import warn
+
+import numpy as np
+
+from .top_k import MetricTopK
+
+
+logger = logging.getLogger(__name__)
+
+
+class ElementwiseMetricK(MetricTopK):
+    """Base class for all elementwise metrics that can be calculated for
+    each user-item pair in the Top-K recommendations.
+
+    :attr:`results` contains an entry for each user-item pair.
+
+    Examples are: HitK
+
+    This code is adapted from RecPack :cite:`recpack`
+
+    :param K: Size of the recommendation list consisting of the Top-K item predictions.
+    :type K: int
+    """
+
+    # TODO to fix this function
+    @property
+    def micro_result(self) -> dict[str, np.ndarray]:
+        if not self._is_computed:
+            raise ValueError("Metric has not been calculated yet.")
+        elif self._scores is None:
+            warn(UserWarning("No scores were computed. Returning empty dict."))
+            return dict(zip(self.col_names, (np.array([]), np.array([]))))
+
+        scores = self._scores.toarray().reshape(-1)
+        unique_users, inv = np.unique(self._user_id_sequence_array, return_inverse=True)
+
+        # Sum hits per user
+        sum_ones = np.zeros(len(unique_users))
+        np.add.at(sum_ones, inv, scores)
+
+        # Count recommendations per user
+        count_all = np.zeros(len(unique_users))
+        np.add.at(count_all, inv, 1)
+
+        # aggregated score per user
+        agg_score = sum_ones / count_all
+
+        return dict(zip(self.col_names, (unique_users, agg_score)))
+
+
+    @property
+    def macro_result(self) -> None | float:
+        if not self._is_computed:
+            raise ValueError("Metric has not been calculated yet.")
+        elif self._scores is None:
+            logger.warning(UserWarning("No scores were computed. Returning Null value."))
+            return None
+        elif self._scores.size == 0:
+            logger.warning(
+                UserWarning(
+                    f"All predictions were off or the ground truth matrix was empty during compute of {self.identifier}."
+                )
+            )
+            return 0
+
+        scores = self._scores.toarray().reshape(-1)
+        unique_users, inv = np.unique(self._user_id_sequence_array, return_inverse=True)
+        # get all users that was recommended at least a relevant item
+        sum_ones = np.zeros(len(unique_users))
+        np.add.at(sum_ones, inv, scores)
+        # Convert to binary: 1 if at least 1 hit, 0 otherwise
+        binary_hits = (sum_ones > 0).astype(int)
+        # Fraction of users with at least 1 hit
+        return binary_hits.mean().item()

recnexteval/metrics/core/listwise_top_k.py

@@ -0,0 +1,72 @@
+import logging
+
+import numpy as np
+
+from .top_k import MetricTopK
+
+
+logger = logging.getLogger(__name__)
+
+
+class ListwiseMetricK(MetricTopK):
+    """Base class for all listwise metrics that can be calculated for every Top-K recommendation list,
+    i.e. one value for each user.
+    Examples are: PrecisionK, RecallK, DCGK, NDCGK.
+
+    :param K: Size of the recommendation list consisting of the Top-K item predictions.
+    :type K: int
+    """
+
+    @property
+    def micro_result(self) -> dict[str, np.ndarray]:
+        """User level results for the metric.
+
+        Contains an entry for every user.
+
+        :return: The results DataFrame with columns: user_id, score
+        :rtype: pd.DataFrame
+        """
+        if not self._is_computed:
+            raise ValueError("Metric has not been calculated yet.")
+        elif self._scores is None:
+            logger.warning(UserWarning("No scores were computed. Returning empty dict."))
+            return dict(zip(self.col_names, (np.array([]), np.array([]))))
+
+        scores = self._scores.toarray().reshape(-1)
+
+        unique_users, inv = np.unique(self._user_id_sequence_array, return_inverse=True)
+
+        # sum of scores per user
+        sum_ones = np.zeros(len(unique_users))
+        np.add.at(sum_ones, inv, scores)
+
+        # count per user
+        count_all = np.zeros(len(unique_users))
+        np.add.at(count_all, inv, 1)
+
+        # aggregated score per user
+        agg_score = sum_ones / count_all
+
+        return dict(zip(self.col_names, (unique_users, agg_score)))
+
+    @property
+    def macro_result(self) -> None | float:
+        """Global metric value obtained by taking the average over all users.
+
+        :raises ValueError: If the metric has not been calculated yet.
+        :return: The global metric value.
+        :rtype: float, optional
+        """
+        if not self._is_computed:
+            raise ValueError("Metric has not been calculated yet.")
+        elif self._scores is None:
+            logger.warning(UserWarning("No scores were computed. Returning Null value."))
+            return None
+        elif self._scores.size == 0:
+            logger.warning(
+                UserWarning(
+                    f"All predictions were off or the ground truth matrix was empty during compute of {self.identifier}."
+                )
+            )
+            return 0
+        return self._scores.mean().item()

recnexteval/metrics/core/top_k.py

@@ -0,0 +1,60 @@
+import logging
+from warnings import warn
+
+import numpy as np
+from scipy.sparse import csr_matrix
+
+from ...algorithms.utils import get_top_K_ranks
+from .base import Metric
+
+
+logger = logging.getLogger(__name__)
+
+
+class MetricTopK(Metric):
+    """Base class for all metrics computed based on the Top-K recommendations for every user.
+
+    A MetricTopK object is stateful, i.e. after `calculate`
+    the results can be retrieved in one of two ways:
+      - Detailed results are stored in :attr:`results`,
+      - Aggregated result value can be retrieved using :attr:`value`
+
+    :param K: Size of the recommendation list consisting of the Top-K item predictions.
+    :type K: int
+    """
+
+    def __init__(
+        self,
+        user_id_sequence_array: np.ndarray,
+        user_item_shape: tuple[int, int],
+        timestamp_limit: None | int = None,
+        K: int = 10,
+    ) -> None:
+        super().__init__(
+            user_id_sequence_array=user_id_sequence_array,
+            user_item_shape=user_item_shape,
+            timestamp_limit=timestamp_limit,
+        )
+        if K is None:
+            warn(f"K not specified, using default value {K}.")
+        self.K = K
+
+    @property
+    def name(self) -> str:
+        """Name of the metric."""
+        return f"{super().name}_{self.K}"
+
+    @property
+    def params(self) -> dict[str, int | None]:
+        """Parameters of the metric."""
+        return super().params | {"K": self.K}
+
+    @property
+    def col_names(self) -> list[str]:
+        """The names of the columns in the results DataFrame."""
+        return ["user_id", "score"]
+
+    def prepare_matrix(self, y_true: csr_matrix, y_pred: csr_matrix) -> tuple[csr_matrix, csr_matrix]:
+        y_true, y_pred = super()._prepare_matrix(y_true, y_pred)
+        y_pred = get_top_K_ranks(y_pred, self.K)
+        return y_true, y_pred

recnexteval/metrics/core/util.py

@@ -0,0 +1,29 @@
+from scipy.sparse import csr_matrix
+
+
+def sparse_inverse_nonzero(a: csr_matrix) -> csr_matrix:
+    """Invert nonzero elements of a `scipy.sparse.csr_matrix`.
+
+    :param a: Matrix to invert.
+    :type a: csr_matrix
+    :return: Matrix with nonzero elements inverted.
+    :rtype: csr_matrix
+    """
+    inv_a = a.copy()
+    inv_a.data = 1 / inv_a.data
+    return inv_a
+
+
+def sparse_divide_nonzero(a: csr_matrix, b: csr_matrix) -> csr_matrix:
+    """Elementwise divide of nonzero elements of a by nonzero elements of b.
+
+    Elements that were zero in either a or b are zero in the resulting matrix.
+
+    :param a: Numerator.
+    :type a: csr_matrix
+    :param b: Denominator.
+    :type b: csr_matrix
+    :return: Result of the elementwise division of matrix a by matrix b.
+    :rtype: csr_matrix
+    """
+    return a.multiply(sparse_inverse_nonzero(b))

recnexteval/metrics/ranking/__init__.py

@@ -0,0 +1,6 @@
+from .ndcg import NDCGK
+from .dcg import DCGK
+from .precision import PrecisionK
+from .recall import RecallK
+
+__all__ = ["NDCGK", "DCGK"]