recnexteval 0.1.0__py3-none-any.whl

recnexteval/evaluators/accumulator.py

@@ -0,0 +1,167 @@
+import logging
+from collections import defaultdict
+from typing import Optional
+
+import pandas as pd
+
+from recnexteval.metrics import Metric
+from .util import MetricLevelEnum
+
+
+logger = logging.getLogger(__name__)
+
+
+class MetricAccumulator:
+    def __init__(self) -> None:
+        self.acc: defaultdict[str, dict[str, Metric]] = defaultdict(dict)
+
+    def __getitem__(self, key) -> dict[str, Metric]:
+        return self.acc[key]
+
+    def add(self, metric: Metric, algorithm_name: str) -> None:
+        """Add a metric to the accumulator
+
+        Takes a :class:`Metric` object and adds it under the algorithm name. If
+        the specified metric already exists for the algorithm, it will be
+        overwritten with the new metric.
+
+        :param metric: Metric to store
+        :type metric: Metric
+        :param algorithm_name: Name of the algorithm
+        :type algorithm_name: str
+        """
+        if metric.identifier in self.acc[algorithm_name]:
+            logger.warning(
+                f"Metric {metric.identifier} already exists for algorithm {algorithm_name}. Overwriting..."
+            )
+
+        logger.debug(f"Metric {metric.identifier} created for algorithm {algorithm_name}")
+
+        self.acc[algorithm_name][metric.identifier] = metric
+
+    @property
+    def user_level_metrics(self) -> defaultdict:
+        results = defaultdict()
+        for algo_name in self.acc:
+            for metric_identifier in self.acc[algo_name]:
+                metric = self.acc[algo_name][metric_identifier]
+                results[(algo_name, f"t={metric.timestamp_limit}", metric.name)] = (
+                    metric.micro_result
+                )
+        return results
+
+    @property
+    def window_level_metrics(self) -> defaultdict:
+        results = defaultdict(dict)
+        for algo_name in self.acc:
+            for metric_identifier in self.acc[algo_name]:
+                metric = self.acc[algo_name][metric_identifier]
+                score = metric.macro_result
+                num_user = metric.num_users
+                if score == 0 and num_user == 0:
+                    logger.info(
+                        f"Metric {metric.name} for algorithm {algo_name} "
+                        f"at t={metric.timestamp_limit} has 0 score and 0 users. "
+                        "The ground truth may be empty due to no interactions occurring in that window."
+                    )
+                elif score == 0 and num_user != 0:
+                    logger.info(
+                        f"Metric {metric.name} for algorithm {algo_name} "
+                        f"at t={metric.timestamp_limit} has 0 score but there are interactions. "
+                        f"{algo_name} did not have any correct predictions."
+                    )
+                results[(algo_name, f"t={metric.timestamp_limit}", metric.name)]["score"] = score
+                results[(algo_name, f"t={metric.timestamp_limit}", metric.name)]["num_user"] = (
+                    num_user
+                )
+        return results
+
+    def df_user_level_metric(self) -> pd.DataFrame:
+        """User metric across all timestamps
+
+        Computation of metrics evaluated on the user level
+
+        :return: _description_
+        :rtype: pd.DataFrame
+        """
+        df = pd.DataFrame.from_dict(self.user_level_metrics, orient="index").explode(
+            ["user_id", "score"]
+        )
+        df = df.rename_axis(["algorithm", "timestamp", "metric"])
+        df.rename(columns={"score": "user_score"}, inplace=True)
+        return df
+
+    def df_window_level_metric(self) -> pd.DataFrame:
+        df = pd.DataFrame.from_dict(self.window_level_metrics, orient="index").explode(
+            ["score", "num_user"]
+        )
+        df = df.rename_axis(["algorithm", "timestamp", "metric"])
+        df.rename(columns={"score": "window_score"}, inplace=True)
+        return df
+
+    def df_macro_level_metric(self) -> pd.DataFrame:
+        """Macro metric across all timestamps
+
+        :return: _description_
+        :rtype: pd.DataFrame
+        """
+        df = pd.DataFrame.from_dict(self.window_level_metrics, orient="index").explode(
+            ["score", "num_user"]
+        )
+        df = df.rename_axis(["algorithm", "timestamp", "metric"])
+        result = df.groupby(["algorithm", "metric"]).mean()["score"].to_frame()
+        result["num_window"] = df.groupby(["algorithm", "metric"]).count()["score"]
+        result = result.rename(columns={"score": "macro_score"})
+        return result
+
+    def df_micro_level_metric(self) -> pd.DataFrame:
+        """Micro metric across all timestamps
+
+        :return: _description_
+        :rtype: pd.DataFrame
+        """
+        df = pd.DataFrame.from_dict(self.user_level_metrics, orient="index").explode(
+            ["user_id", "score"]
+        )
+        df = df.rename_axis(["algorithm", "timestamp", "metric"])
+        result = df.groupby(["algorithm", "metric"])["score"].mean().to_frame()
+        result["num_user"] = df.groupby(["algorithm", "metric"])["score"].count()
+        result = result.rename(columns={"score": "micro_score"})
+        return result
+
+    def df_metric(
+        self,
+        filter_timestamp: Optional[int] = None,
+        filter_algo: Optional[str] = None,
+        level: MetricLevelEnum = MetricLevelEnum.MACRO,
+    ) -> pd.DataFrame:
+        """Dataframe representation of the metric
+
+        Returns a dataframe representation of the metric. The dataframe can be
+        filtered based on the algorithm name and the timestamp.
+
+        :param filter_timestamp: Timestamp value to filter on, defaults to None
+        :type filter_timestamp: Optional[int], optional
+        :param filter_algo: Algorithm name to filter on, defaults to None
+        :type filter_algo: Optional[str], optional
+        :param level: Level of the metric to compute, defaults to MetricLevelEnum.MACRO
+        :type level: MetricLevelEnum, optional
+        :return: Dataframe representation of the metric
+        :rtype: pd.DataFrame
+        """
+        if level == MetricLevelEnum.MACRO:
+            df = self.df_macro_level_metric()
+        elif level == MetricLevelEnum.MICRO:
+            df = self.df_micro_level_metric()
+        elif level == MetricLevelEnum.WINDOW:
+            df = self.df_window_level_metric()
+        elif level == MetricLevelEnum.USER:
+            df = self.df_user_level_metric()
+        else:
+            raise ValueError("Invalid level specified")
+
+        if filter_algo:
+            df = df.filter(like=filter_algo, axis=0)
+        if filter_timestamp:
+            df = df.filter(like=f"t={filter_timestamp}", axis=0)
+        return df

recnexteval/evaluators/base.py

@@ -0,0 +1,216 @@
+import logging
+from typing import Literal
+
+import pandas as pd
+from scipy.sparse import csr_matrix
+
+from ..matrix import PredictionMatrix
+from ..registries import MetricEntry
+from ..settings import EOWSettingError, Setting
+from .accumulator import MetricAccumulator
+from .util import MetricLevelEnum, UserItemBaseStatus
+
+
+logger = logging.getLogger(__name__)
+
+
+class EvaluatorBase(object):
+    """Base class for evaluator.
+
+    Provides the common methods and attributes for the evaluator classes. Should
+    there be a need to create a new evaluator, it should inherit from this class.
+
+    Args:
+        metric_entries: List of metric entries to compute.
+        setting: Setting object.
+        ignore_unknown_user: Ignore unknown users, defaults to False.
+        ignore_unknown_item: Ignore unknown items, defaults to False.
+    """
+
+    def __init__(
+        self,
+        metric_entries: list[MetricEntry],
+        setting: Setting,
+        metric_k: int,
+        ignore_unknown_user: bool = False,
+        ignore_unknown_item: bool = False,
+        seed: int = 42,
+    ) -> None:
+        self.metric_entries = metric_entries
+        self.setting = setting
+        """Setting to evaluate the algorithms on."""
+        self.metric_k = metric_k
+        """Value of K for the metrics."""
+        self.ignore_unknown_user = ignore_unknown_user
+        """To ignore unknown users during evaluation."""
+        self.ignore_unknown_item = ignore_unknown_item
+        """To ignore unknown items during evaluation."""
+
+        self.user_item_base = UserItemBaseStatus()
+        self.seed = seed
+        self._run_step = 0
+        self._acc: MetricAccumulator
+        self._current_timestamp: int
+
+    def _get_evaluation_data(self) -> tuple[PredictionMatrix, PredictionMatrix, int]:
+        """Get the evaluation data for the current step.
+
+        Internal method to get the evaluation data for the current step. The
+        evaluation data consists of the unlabeled data, ground truth data, and
+        the current timestamp which will be returned as a tuple. The shapes
+        are masked based through `user_item_base`. The unknown users in
+        the ground truth data are also updated in `user_item_base`.
+
+        Note:
+            `_current_timestamp` is updated with the current timestamp.
+
+        Returns:
+            Tuple of unlabeled data, ground truth data, and current timestamp.
+
+        Raises:
+            EOWSettingError: If there is no more data to be processed.
+        """
+        try:
+            split = self.setting.get_split_at(self._run_step)
+            unlabeled_data = split.unlabeled
+            ground_truth_data = split.ground_truth
+            if split.t_window is None:
+                raise ValueError("Timestamp of the current split cannot be None")
+            self._current_timestamp = split.t_window
+
+            unlabeled_data = PredictionMatrix.from_interaction_matrix(unlabeled_data)
+            ground_truth_data = PredictionMatrix.from_interaction_matrix(ground_truth_data)
+            self._run_step += 1
+        except EOWSettingError:
+            raise EOWSettingError("There is no more data to be processed, EOW reached")
+
+        self.user_item_base.update_unknown_user_item_base(ground_truth_data)
+
+        mask_shape = (self.user_item_base.known_shape[0], self.user_item_base.known_shape[1])
+        if not self.ignore_unknown_user:
+            mask_shape = (self.user_item_base.global_shape[0], mask_shape[1])
+
+        unlabeled_data.mask_user_item_shape(
+            shape=mask_shape
+        )
+        ground_truth_data.mask_user_item_shape(
+            shape=mask_shape,
+            drop_unknown_item=self.ignore_unknown_item,
+            inherit_max_id=True,  # Ensures that shape of ground truth contains all user id that appears globally
+        )
+        # get the index of ground_truth_data._df
+        if self.ignore_unknown_item:
+            unlabeled_data._df = unlabeled_data._df.loc[ground_truth_data._df.index]
+        return unlabeled_data, ground_truth_data, self._current_timestamp
+
+    def _prediction_shape_handler(
+        self, y_true: csr_matrix, y_pred: csr_matrix
+    ) -> csr_matrix:
+        """Handle shape difference of the prediction matrix.
+
+        If there is a difference in the shape of the prediction matrix and the
+        ground truth matrix, this function will handle the difference based on
+        `ignore_unknown_user` and `ignore_unknown_item`.
+
+        Args:
+            X_true: Ground truth matrix.
+            X_pred: Prediction matrix.
+        """
+        X_true_shape = y_true.shape
+        if y_pred.shape != X_true_shape:
+            logger.warning("Prediction matrix shape %s is different from ground truth matrix shape %s.", y_pred.shape, X_true_shape)
+            # We cannot expect the algorithm to predict an unknown item, so we
+            # only check user dimension
+            if y_pred.shape[0] < X_true_shape[0] and not self.ignore_unknown_user:  # type: ignore
+                raise ValueError(
+                    "Prediction matrix shape, user dimension, is less than the ground truth matrix shape."
+                )
+
+            if not self.ignore_unknown_item:
+                # prediction matrix would not contain unknown item ID
+                # update the shape of the prediction matrix to include the ID
+                y_pred = csr_matrix(
+                    (y_pred.data, y_pred.indices, y_pred.indptr),
+                    shape=(y_pred.shape[0], X_true_shape[1]),  # type: ignore
+                )
+
+            # shapes might not be the same in the case of dropping unknowns
+            # from the ground truth data. We ensure that the same unknowns
+            # are dropped from the predictions
+            if self.ignore_unknown_user:
+                y_pred = y_pred[: X_true_shape[0], :]  # type: ignore
+            if self.ignore_unknown_item:
+                y_pred = y_pred[:, : X_true_shape[1]]  # type: ignore
+
+        return y_pred
+
+    def metric_results(
+        self,
+        level: MetricLevelEnum | Literal["macro", "micro", "window", "user"] = MetricLevelEnum.MACRO,
+        only_current_timestamp: None | bool = False,
+        filter_timestamp: None | int = None,
+        filter_algo: None | str = None,
+    ) -> pd.DataFrame:
+        """Results of the metrics computed.
+
+        Computes the metrics of all algorithms based on the level specified and
+        return the results in a pandas DataFrame. The results can be filtered
+        based on the algorithm name and the current timestamp.
+
+        Specifics
+        ---------
+        - User level: User level metrics computed across all timestamps.
+        - Window level: Window level metrics computed across all timestamps. This can
+            be viewed as a macro level metric in the context of a single window, where
+            the scores of each user is averaged within the window.
+        - Macro level: Macro level metrics computed for entire timeline. This
+            score is computed by averaging the scores of all windows, treating each
+            window equally.
+        - Micro level: Micro level metrics computed for entire timeline. This
+            score is computed by averaging the scores of all users, treating each
+            user and the timestamp the user is in as unique contribution to the
+            overall score.
+
+        Args:
+            level: Level of the metric to compute, defaults to "macro".
+            only_current_timestamp: Filter only the current timestamp, defaults to False.
+            filter_timestamp: Timestamp value to filter on, defaults to None.
+                If both `only_current_timestamp` and `filter_timestamp` are provided,
+                `filter_timestamp` will be used.
+            filter_algo: Algorithm name to filter on, defaults to None.
+
+        Returns:
+            Dataframe representation of the metric.
+        """
+        if isinstance(level, str) and not MetricLevelEnum.has_value(level):
+            raise ValueError("Invalid level specified")
+        level = MetricLevelEnum(level)
+
+        if only_current_timestamp and filter_timestamp:
+            raise ValueError("Cannot specify both only_current_timestamp and filter_timestamp.")
+
+        timestamp = None
+        if only_current_timestamp:
+            timestamp = self._current_timestamp
+
+        if filter_timestamp:
+            timestamp = filter_timestamp
+
+        return self._acc.df_metric(filter_algo=filter_algo, filter_timestamp=timestamp, level=level)
+
+    def restore(self) -> None:
+        """Restore the generators before pickling.
+
+        This method is used to restore the generators after loading the object
+        from a pickle file.
+        """
+        self.setting.restore(self._run_step)
+        logger.debug("Generators restored")
+
+    def current_step(self) -> int:
+        """Return the current step of the evaluator.
+
+        Returns:
+            Current step of the evaluator.
+        """
+        return self._run_step

recnexteval/evaluators/builder/__init__.py

@@ -0,0 +1,125 @@
+"""Builder module for constructing evaluator objects.
+
+This module provides builder classes for constructing evaluator objects in the
+RecNextEval library. Builders follow the builder pattern to facilitate the
+construction of evaluators with proper validation and error checking.
+
+## Builder Overview
+
+The builder pattern is used to construct complex evaluator objects step by step.
+Builders ensure that all necessary components (settings, metrics, algorithms)
+are properly configured before building the evaluator, preventing runtime errors.
+
+## Available Builders
+
+- `Builder`: Abstract base class for all builder implementations
+- `EvaluatorPipelineBuilder`: Builder for pipeline evaluators that evaluate
+  multiple algorithms on static data
+- `EvaluatorStreamerBuilder`: Builder for streaming evaluators that evaluate
+  algorithms on streaming data
+
+## Using Builders
+
+### Basic Pipeline Evaluation
+
+To evaluate multiple algorithms on a static dataset using a pipeline evaluator:
+
+```python
+from recnexteval.evaluators.builder import EvaluatorPipelineBuilder
+from recnexteval.settings import Setting
+from recnexteval.datasets import AmazonMusicDataset
+
+# Load dataset
+dataset = AmazonMusicDataset()
+data = dataset.load()
+
+# Create setting
+setting = Setting(data=data, top_K=10)
+setting.split()
+
+# Build evaluator
+builder = EvaluatorPipelineBuilder(seed=42)
+builder.add_setting(setting)
+builder.set_metric_K(10)
+builder.add_metric("PrecisionK")
+builder.add_metric("RecallK")
+builder.add_algorithm("MostPopular")
+builder.add_algorithm("RecentPop", params={"K": 10})
+
+evaluator = builder.build()
+results = evaluator.evaluate()
+```
+
+### Streaming Evaluation
+
+To evaluate algorithms on streaming data:
+
+```python
+from recnexteval.evaluators.builder import EvaluatorStreamerBuilder
+from recnexteval.settings import StreamingSetting
+from recnexteval.datasets import AmazonMusicDataset
+
+# Load dataset
+dataset = AmazonMusicDataset()
+data = dataset.load()
+
+# Create streaming setting
+setting = StreamingSetting(data=data, top_K=10, window_size=1000)
+setting.split()
+
+# Build streaming evaluator
+builder = EvaluatorStreamerBuilder(seed=42)
+builder.add_setting(setting)
+builder.set_metric_K(10)
+builder.add_metric("HitK")
+builder.add_metric("NDCGK")
+
+evaluator = builder.build()
+# The evaluator can now process streaming data
+```
+
+### Advanced Configuration
+
+Builders support advanced configuration options:
+
+```python
+from recnexteval.evaluators.builder import EvaluatorPipelineBuilder
+
+builder = EvaluatorPipelineBuilder(
+    ignore_unknown_user=False,  # Don't ignore unknown users
+    ignore_unknown_item=True,   # Ignore unknown items
+    seed=123
+)
+
+builder.add_setting(setting)
+builder.set_metric_K(20)
+
+# Add multiple metrics
+metrics = ["PrecisionK", "RecallK", "DCGK", "NDCGK", "HitK"]
+for metric in metrics:
+    builder.add_metric(metric)
+
+# Add algorithms with custom parameters
+builder.add_algorithm("ItemKNN", params={"K": 50, "similarity": "cosine"})
+builder.add_algorithm("DecayPop", params={"decay_factor": 0.9})
+
+evaluator = builder.build()
+```
+
+## Extending the Framework
+
+To create custom builders, inherit from the `Builder` base class and implement
+the `build()` method. Ensure to call `super().__init__()` and implement proper
+validation in `_check_ready()`.
+"""
+
+from .base import Builder
+from .pipeline import EvaluatorPipelineBuilder
+from .stream import EvaluatorStreamerBuilder
+
+
+__all__ = [
+    "Builder",
+    "EvaluatorPipelineBuilder",
+    "EvaluatorStreamerBuilder",
+]

recnexteval/evaluators/builder/base.py

@@ -0,0 +1,166 @@
+import logging
+from abc import ABC, abstractmethod
+from warnings import warn
+
+from recnexteval.registries import (
+    METRIC_REGISTRY,
+    MetricEntry,
+)
+from recnexteval.settings import Setting
+from recnexteval.utils import arg_to_str
+from ..base import EvaluatorBase
+
+
+logger = logging.getLogger(__name__)
+
+
+class Builder(ABC):
+    """Base class for Builder objects.
+
+    Provides methods to set specific values for the builder and enforce checks
+    such that the builder can be constructed correctly and to avoid possible
+    errors when the builder is executed.
+    """
+
+    def __init__(
+        self,
+        ignore_unknown_user: bool = True,
+        ignore_unknown_item: bool = True,
+        seed: int = 42,
+    ) -> None:
+        """Initialize the Builder.
+
+        Args:
+            ignore_unknown_user: Ignore unknown user in the evaluation.
+            ignore_unknown_item: Ignore unknown item in the evaluation.
+            seed: Random seed for reproducibility.
+        """
+        self.metric_entries: dict[str, MetricEntry] = dict()
+        """dict of metrics to evaluate algorithm on.
+        Using dict instead of list for fast lookup"""
+        self.setting: Setting
+        """Setting to evaluate the algorithms on"""
+        self.ignore_unknown_user = ignore_unknown_user
+        """Ignore unknown user in the evaluation"""
+        self.ignore_unknown_item = ignore_unknown_item
+        """Ignore unknown item in the evaluation"""
+        self.metric_k: int
+        self.seed: int = seed
+
+    def _check_setting_exist(self) -> bool:
+        """Check if setting is already set.
+
+        Returns:
+            True if setting is set, False otherwise.
+        """
+        return not (not hasattr(self, "setting") or self.setting is None)
+
+    def set_metric_K(self, K: int) -> None:
+        """Set K value for all metrics.
+
+        Args:
+            K: K value to set for all metrics.
+        """
+        self.metric_k = K
+
+    def add_metric(self, metric: str | type) -> None:
+        """Add metric to evaluate algorithm on.
+
+        Metric will be added to the metric_entries dict where it will later be
+        converted to a list when the evaluator is constructed.
+
+        Note:
+            If K is not yet specified, the setting's top_K value will be used. This
+            requires the setting to be set before adding the metric.
+
+        Args:
+            metric: Metric to evaluate algorithm on.
+
+        Raises:
+            ValueError: If metric is not found in METRIC_REGISTRY.
+            RuntimeError: If setting is not set.
+        """
+        if not self._check_setting_exist():
+            raise RuntimeError(
+                "Setting has not been set. To ensure conformity, of the addition of"
+                " other components please set the setting first. Call add_setting() method."
+            )
+
+        metric = arg_to_str(metric)
+
+        if metric not in METRIC_REGISTRY:
+            raise ValueError(f"Metric {metric} could not be resolved.")
+
+        if not hasattr(self, "metric_k"):
+            self.metric_k = self.setting.top_K
+            warn(
+                "K value not yet specified before setting metric, using setting's top_K value."
+                " We recommend specifying K value for metric. If you want to change the K value,"
+                " you can clear all metric entry and set the K value before adding metrics."
+            )
+
+        metric_name = f"{metric}_{self.metric_k}"
+        if metric_name in self.metric_entries:
+            logger.warning(f"Metric {metric_name} already exists. Skipping adding metric.")
+            return
+
+        self.metric_entries[metric_name] = MetricEntry(metric, self.metric_k)
+
+    def add_setting(self, setting: Setting) -> None:
+        """Add setting to the evaluator builder.
+
+        Note:
+            The setting should be set before adding metrics or algorithms
+            to the evaluator.
+
+        Args:
+            setting: Setting to evaluate the algorithms on.
+
+        Raises:
+            ValueError: If setting is not of instance Setting.
+        """
+        if not isinstance(setting, Setting):
+            raise ValueError(f"setting should be of type Setting, got {type(setting)}")
+        if hasattr(self, "setting") and self.setting is not None:
+            warn("Setting is already set. Continuing will overwrite the setting.")
+
+        self.setting = setting
+
+    def clear_metrics(self) -> None:
+        """Clear all metrics from the builder."""
+        self.metric_entries.clear()
+
+    def _check_ready(self) -> None:
+        """Check if the builder is ready to construct Evaluator.
+
+        Raises:
+            RuntimeError: If there are invalid configurations.
+        """
+        if not hasattr(self, "metric_k"):
+            self.metric_k = self.setting.top_K
+            warn(
+                "K value not yet specified before setting metric, using setting's top_K value."
+                " We recommend specifying K value for metric. If you want to change the K value,"
+                " you can clear all metric entry and set the K value before adding metrics."
+            )
+
+        if len(self.metric_entries) == 0:
+            raise RuntimeError("No metrics specified, can't construct Evaluator")
+
+        # Check for settings #
+        if self.setting is None:
+            raise RuntimeError("No settings specified, can't construct Evaluator")
+        if not self.setting.is_ready:
+            raise RuntimeError(
+                "Setting is not ready, can't construct Evaluator. "
+                "Call split() on the setting first."
+            )
+
+    @abstractmethod
+    def build(self) -> EvaluatorBase:
+        """Build object.
+
+        Raises:
+            NotImplementedError: If the method is not implemented.
+        """
+        raise NotImplementedError