replay-rec 0.20.2__py3-none-any.whl → 0.20.3rc0__py3-none-any.whl

replay/experimental/metrics/coverage.py

@@ -0,0 +1,97 @@
+from typing import Optional, Union
+
+from replay.utils import PYSPARK_AVAILABLE, DataFrameLike, IntOrList, NumType, SparkDataFrame
+from replay.utils.spark_utils import convert2spark
+
+from .base_metric import RecOnlyMetric, process_k
+
+if PYSPARK_AVAILABLE:
+    from pyspark.sql import (
+        Window,
+        functions as sf,
+    )
+
+
+class Coverage(RecOnlyMetric):
+    """
+    Metric calculation is as follows:
+
+    * take ``K`` recommendations with the biggest ``relevance`` for each ``user_id``
+    * count the number of distinct ``item_id`` in these recommendations
+    * divide it by the number of items in the whole data set
+
+    """
+
+    def __init__(self, log: DataFrameLike):
+        """
+        :param log: pandas or Spark DataFrame
+                    It is important for ``log`` to contain all available items.
+        """
+        self.items = convert2spark(log).select("item_idx").distinct()
+        self.item_count = self.items.count()
+
+    @staticmethod
+    def _get_metric_value_by_user(k, *args):
+        # not averaged by users
+        pass
+
+    def _get_enriched_recommendations(
+        self,
+        recommendations: DataFrameLike,
+        ground_truth: DataFrameLike,  # noqa: ARG002
+        max_k: int,  # noqa: ARG002
+        ground_truth_users: Optional[DataFrameLike] = None,
+    ) -> SparkDataFrame:
+        recommendations = convert2spark(recommendations)
+        if ground_truth_users is not None:
+            ground_truth_users = convert2spark(ground_truth_users)
+            return recommendations.join(ground_truth_users, on="user_idx", how="inner")
+        return recommendations
+
+    def _conf_interval(
+        self,
+        recs: DataFrameLike,  # noqa: ARG002
+        k_list: IntOrList,
+        alpha: float = 0.95,  # noqa: ARG002
+    ) -> Union[dict[int, float], float]:
+        if isinstance(k_list, int):
+            return 0.0
+        return dict.fromkeys(k_list, 0.0)
+
+    def _median(
+        self,
+        recs: DataFrameLike,
+        k_list: IntOrList,
+    ) -> Union[dict[int, NumType], NumType]:
+        return self._mean(recs, k_list)
+
+    @process_k
+    def _mean(
+        self,
+        recs: SparkDataFrame,
+        k_list: list,
+    ) -> Union[dict[int, NumType], NumType]:
+        unknown_item_count = recs.select("item_idx").distinct().exceptAll(self.items).count()
+        if unknown_item_count > 0:
+            self.logger.warning(
+                "Recommendations contain items that were not present in the log. "
+                r"The resulting metric value can be more than 1.0 ¯\_(ツ)_/¯"
+            )
+
+        best_positions = (
+            recs.withColumn(
+                "row_num",
+                sf.row_number().over(Window.partitionBy("user_idx").orderBy(sf.desc("relevance"))),
+            )
+            .select("item_idx", "row_num")
+            .groupBy("item_idx")
+            .agg(sf.min("row_num").alias("best_position"))
+            .cache()
+        )
+
+        res = {}
+        for current_k in k_list:
+            res[current_k] = best_positions.filter(sf.col("best_position") <= current_k).count() / self.item_count
+
+        best_positions.unpersist()
+        return res

replay/experimental/metrics/experiment.py

@@ -0,0 +1,175 @@
+from typing import Any, Optional
+
+from replay.utils import DataFrameLike, IntOrList, NumType, PandasDataFrame
+from replay.utils.spark_utils import convert2spark
+
+from .base_metric import Metric, NCISMetric, RecOnlyMetric, get_enriched_recommendations
+
+
+class Experiment:
+    """
+    This class calculates and stores metric values.
+    Initialize it with test data and a dictionary mapping metrics to their depth cut-offs.
+
+    Results are available with ``pandas_df`` attribute.
+
+    Example:
+
+    >>> import pandas as pd
+    >>> from replay.experimental.metrics import Coverage, NDCG, Precision, Surprisal
+    >>> log = pd.DataFrame({"user_idx": [2, 2, 2, 1], "item_idx": [1, 2, 3, 3], "relevance": [5, 5, 5, 5]})
+    >>> test = pd.DataFrame({"user_idx": [1, 1, 1], "item_idx": [1, 2, 3], "relevance": [5, 3, 4]})
+    >>> pred = pd.DataFrame({"user_idx": [1, 1, 1], "item_idx": [4, 1, 3], "relevance": [5, 4, 5]})
+    >>> recs = pd.DataFrame({"user_idx": [1, 1, 1], "item_idx": [1, 4, 5], "relevance": [5, 4, 5]})
+    >>> ex = Experiment(test, {NDCG(): [2, 3], Surprisal(log): 3})
+    >>> ex.add_result("baseline", recs)
+    >>> ex.add_result("baseline_gt_users", recs, ground_truth_users=pd.DataFrame({"user_idx": [1, 3]}))
+    >>> ex.add_result("model", pred)
+    >>> ex.results
+                         NDCG@2    NDCG@3  Surprisal@3
+    baseline           0.386853  0.296082     1.000000
+    baseline_gt_users  0.193426  0.148041     0.500000
+    model              0.386853  0.530721     0.666667
+    >>> ex.compare("baseline")
+                       NDCG@2  NDCG@3 Surprisal@3
+    baseline                –       –           –
+    baseline_gt_users  -50.0%  -50.0%      -50.0%
+    model                0.0%  79.25%     -33.33%
+    >>> ex = Experiment(test, {Precision(): [3]}, calc_median=True, calc_conf_interval=0.95)
+    >>> ex.add_result("baseline", recs)
+    >>> ex.add_result("model", pred)
+    >>> ex.results
+              Precision@3  Precision@3_median  Precision@3_0.95_conf_interval
+    baseline     0.333333            0.333333                             0.0
+    model        0.666667            0.666667                             0.0
+    >>> ex = Experiment(test, {Coverage(log): 3}, calc_median=True, calc_conf_interval=0.95)
+    >>> ex.add_result("baseline", recs)
+    >>> ex.add_result("model", pred)
+    >>> ex.results
+              Coverage@3  Coverage@3_median  Coverage@3_0.95_conf_interval
+    baseline         1.0                1.0                            0.0
+    model            1.0                1.0                            0.0
+    """
+
+    def __init__(
+        self,
+        test: Any,
+        metrics: dict[Metric, IntOrList],
+        calc_median: bool = False,
+        calc_conf_interval: Optional[float] = None,
+    ):
+        """
+        :param test: test DataFrame
+        :param metrics: Dictionary of metrics to calculate.
+            Key -- metric, value -- ``int`` or a list of ints.
+        :param calc_median: flag to calculate median value across users
+        :param calc_conf_interval: quantile value for the calculation of the confidence interval.
+            Resulting value is the half of confidence interval.
+        """
+        self.test = convert2spark(test)
+        self.results = PandasDataFrame()
+        self.metrics = metrics
+        self.calc_median = calc_median
+        self.calc_conf_interval = calc_conf_interval
+
+    def add_result(
+        self,
+        name: str,
+        pred: DataFrameLike,
+        ground_truth_users: Optional[DataFrameLike] = None,
+    ) -> None:
+        """
+        Calculate metrics for predictions
+
+        :param name: name of the run to store in the resulting DataFrame
+        :param pred: model recommendations
+        :param ground_truth_users: list of users to consider in metric calculation.
+            if None, only the users from ground_truth are considered.
+        """
+        max_k = 0
+        for current_k in self.metrics.values():
+            max_k = max((*current_k, max_k) if isinstance(current_k, list) else (current_k, max_k))
+
+        recs = get_enriched_recommendations(pred, self.test, max_k, ground_truth_users).cache()
+        for metric, k_list in sorted(self.metrics.items(), key=lambda x: str(x[0])):
+            enriched = None
+            if isinstance(metric, (RecOnlyMetric, NCISMetric)):
+                enriched = metric._get_enriched_recommendations(pred, self.test, max_k, ground_truth_users)
+            values, median, conf_interval = self._calculate(metric, enriched or recs, k_list)
+
+            if isinstance(k_list, int):
+                self._add_metric(
+                    name,
+                    metric,
+                    k_list,
+                    values,
+                    median,
+                    conf_interval,
+                )
+            else:
+                for k, val in sorted(values.items(), key=lambda x: x[0]):
+                    self._add_metric(
+                        name,
+                        metric,
+                        k,
+                        val,
+                        None if median is None else median[k],
+                        None if conf_interval is None else conf_interval[k],
+                    )
+        recs.unpersist()
+
+    def _calculate(self, metric, enriched, k_list):
+        median = None
+        conf_interval = None
+        values = metric._mean(enriched, k_list)
+        if self.calc_median:
+            median = metric._median(enriched, k_list)
+        if self.calc_conf_interval is not None:
+            conf_interval = metric._conf_interval(enriched, k_list, self.calc_conf_interval)
+        return values, median, conf_interval
+
+    def _add_metric(
+        self,
+        name: str,
+        metric: Metric,
+        k: int,
+        value: NumType,
+        median: Optional[NumType],
+        conf_interval: Optional[NumType],
+    ):
+        """
+        Add metric for a specific k
+
+        :param name: name to save results
+        :param metric: metric object
+        :param k: length of the recommendation list
+        :param value: metric value
+        :param median: median value
+        :param conf_interval: confidence interval value
+        """
+        self.results.at[name, f"{metric}@{k}"] = value
+        if median is not None:
+            self.results.at[name, f"{metric}@{k}_median"] = median
+        if conf_interval is not None:
+            self.results.at[name, f"{metric}@{k}_{self.calc_conf_interval}_conf_interval"] = conf_interval
+
+    def compare(self, name: str) -> PandasDataFrame:
+        """
+        Show results as a percentage difference to record ``name``.
+
+        :param name: name of the baseline record
+        :return: results table in a percentage format
+        """
+        if name not in self.results.index:
+            msg = f"No results for model {name}"
+            raise ValueError(msg)
+        columns = [column for column in self.results.columns if column[-1].isdigit()]
+        data_frame = self.results[columns].copy()
+        baseline = data_frame.loc[name]
+        for idx in data_frame.index:
+            if idx != name:
+                diff = data_frame.loc[idx] / baseline - 1
+                data_frame.loc[idx] = [str(round(v * 100, 2)) + "%" for v in diff]
+            else:
+                data_frame.loc[name] = ["–"] * len(baseline)
+        return data_frame

replay/experimental/metrics/hitrate.py

@@ -0,0 +1,26 @@
+from .base_metric import Metric
+
+
+class HitRate(Metric):
+    """
+    Percentage of users that have at least one
+        correctly recommended item among top-k.
+
+    .. math::
+        HitRate@K(i) = \\max_{j \\in [1..K]}\\mathbb{1}_{r_{ij}}
+
+    .. math::
+        HitRate@K = \\frac {\\sum_{i=1}^{N}HitRate@K(i)}{N}
+
+    :math:`\\mathbb{1}_{r_{ij}}` -- indicator function stating that user :math:`i` interacted with item :math:`j`
+
+    """
+
+    _scala_udf_name = "getHitRateMetricValue"
+
+    @staticmethod
+    def _get_metric_value_by_user(k, pred, ground_truth) -> float:
+        for i in pred[:k]:
+            if i in ground_truth:
+                return 1
+        return 0

replay/experimental/metrics/map.py

@@ -0,0 +1,30 @@
+from .base_metric import Metric
+
+
+class MAP(Metric):
+    """
+    Mean Average Precision -- average the ``Precision`` at relevant positions for each user,
+        and then calculate the mean across all users.
+
+    .. math::
+        &AP@K(i) = \\frac 1K \\sum_{j=1}^{K}\\mathbb{1}_{r_{ij}}Precision@j(i)
+
+        &MAP@K = \\frac {\\sum_{i=1}^{N}AP@K(i)}{N}
+
+    :math:`\\mathbb{1}_{r_{ij}}` -- indicator function showing if user :math:`i` interacted with item :math:`j`
+    """
+
+    _scala_udf_name = "getMAPMetricValue"
+
+    @staticmethod
+    def _get_metric_value_by_user(k, pred, ground_truth) -> float:
+        length = min(k, len(pred))
+        if len(ground_truth) == 0 or len(pred) == 0:
+            return 0
+        tp_cum = 0
+        result = 0
+        for i in range(length):
+            if pred[i] in ground_truth:
+                tp_cum += 1
+                result += tp_cum / (i + 1)
+        return result / k

replay/experimental/metrics/mrr.py

@@ -0,0 +1,18 @@
+from .base_metric import Metric
+
+
+class MRR(Metric):
+    """
+    Mean Reciprocal Rank --
+    Reciprocal Rank is the inverse position of the first relevant item among top-k recommendations,
+    :math:`\\frac {1}{rank_i}`. This value is averaged by all users.
+    """
+
+    _scala_udf_name = "getMRRMetricValue"
+
+    @staticmethod
+    def _get_metric_value_by_user(k, pred, ground_truth) -> float:
+        for i in range(min(k, len(pred))):
+            if pred[i] in ground_truth:
+                return 1 / (1 + i)
+        return 0

replay/experimental/metrics/ncis_precision.py

@@ -0,0 +1,31 @@
+import numpy as np
+
+from .base_metric import NCISMetric
+
+
+class NCISPrecision(NCISMetric):
+    """
+    Share of relevant items among top ``K`` recommendations with NCIS weighting.
+
+    .. math::
+        Precision@K(i) = \\frac {\\sum_{j=1}^{K}\\mathbb{1}_{r_{ij} w_{ij}}}{\\sum_{j=1}^{K} w_{ij}}
+
+    .. math::
+        Precision@K = \\frac {\\sum_{i=1}^{N}Precision@K(i)}{N}
+
+    :math:`\\mathbb{1}_{r_{ij}}` -- indicator function
+        showing that user :math:`i` interacted with item :math:`j`
+    :math:`w_{ij}` -- NCIS weight, calculated as ratio of current policy score on previous
+        policy score with clipping and optional activation over policy scores (relevance).
+        Source: arxiv.org/abs/1801.07030
+    """
+
+    _scala_udf_name = "getNCISPrecisionMetricValue"
+
+    @staticmethod
+    def _get_metric_value_by_user(k, *args):
+        pred, ground_truth, pred_weights = args
+        if len(pred) == 0 or len(ground_truth) == 0:
+            return 0
+        mask = np.isin(pred[:k], ground_truth)
+        return sum(np.array(pred_weights)[mask]) / sum(pred_weights[:k])

replay/experimental/metrics/ndcg.py

@@ -0,0 +1,49 @@
+import math
+
+from .base_metric import Metric
+
+
+class NDCG(Metric):
+    """
+    Normalized Discounted Cumulative Gain is a metric
+    that takes into account positions of relevant items.
+
+    This is the binary version, it takes into account
+    whether the item was consumed or not, relevance value is ignored.
+
+    .. math::
+        DCG@K(i) = \\sum_{j=1}^{K}\\frac{\\mathbb{1}_{r_{ij}}}{\\log_2 (j+1)}
+
+
+    :math:`\\mathbb{1}_{r_{ij}}` -- indicator function showing that user :math:`i` interacted with item :math:`j`
+
+    To get from :math:`DCG` to :math:`nDCG` we calculate the biggest possible value of `DCG`
+    for user :math:`i` and recommendation length :math:`K`.
+
+    .. math::
+        IDCG@K(i) = max(DCG@K(i)) = \\sum_{j=1}^{K}\\frac{\\mathbb{1}_{j\\le|Rel_i|}}{\\log_2 (j+1)}
+
+    .. math::
+        nDCG@K(i) = \\frac {DCG@K(i)}{IDCG@K(i)}
+
+    :math:`|Rel_i|` -- number of relevant items for user :math:`i`
+
+    Metric is averaged by users.
+
+    .. math::
+        nDCG@K = \\frac {\\sum_{i=1}^{N}nDCG@K(i)}{N}
+    """
+
+    _scala_udf_name = "getNDCGMetricValue"
+
+    @staticmethod
+    def _get_metric_value_by_user(k, pred, ground_truth) -> float:
+        if len(pred) == 0 or len(ground_truth) == 0:
+            return 0.0
+        pred_len = min(k, len(pred))
+        ground_truth_len = min(k, len(ground_truth))
+        denom = [1 / math.log2(i + 2) for i in range(k)]
+        dcg = sum(denom[i] for i in range(pred_len) if pred[i] in ground_truth)
+        idcg = sum(denom[:ground_truth_len])
+
+        return dcg / idcg

replay/experimental/metrics/precision.py

@@ -0,0 +1,22 @@
+from .base_metric import Metric
+
+
+class Precision(Metric):
+    """
+    Mean percentage of relevant items among top ``K`` recommendations.
+
+    .. math::
+        Precision@K(i) = \\frac {\\sum_{j=1}^{K}\\mathbb{1}_{r_{ij}}}{K}
+
+    .. math::
+        Precision@K = \\frac {\\sum_{i=1}^{N}Precision@K(i)}{N}
+
+    :math:`\\mathbb{1}_{r_{ij}}` -- indicator function showing that user :math:`i` interacted with item :math:`j`"""
+
+    _scala_udf_name = "getPrecisionMetricValue"
+
+    @staticmethod
+    def _get_metric_value_by_user(k, pred, ground_truth) -> float:
+        if len(pred) == 0:
+            return 0
+        return len(set(pred[:k]) & set(ground_truth)) / k

replay/experimental/metrics/recall.py

@@ -0,0 +1,25 @@
+from .base_metric import Metric
+
+
+class Recall(Metric):
+    """
+    Mean percentage of relevant items, that was shown among top ``K`` recommendations.
+
+    .. math::
+        Recall@K(i) = \\frac {\\sum_{j=1}^{K}\\mathbb{1}_{r_{ij}}}{|Rel_i|}
+
+    .. math::
+        Recall@K = \\frac {\\sum_{i=1}^{N}Recall@K(i)}{N}
+
+    :math:`\\mathbb{1}_{r_{ij}}` -- indicator function showing that user :math:`i` interacted with item :math:`j`
+
+    :math:`|Rel_i|` -- the number of relevant items for user :math:`i`
+    """
+
+    _scala_udf_name = "getRecallMetricValue"
+
+    @staticmethod
+    def _get_metric_value_by_user(k, pred, ground_truth) -> float:
+        if len(ground_truth) == 0:
+            return 0.0
+        return len(set(pred[:k]) & set(ground_truth)) / len(ground_truth)

replay/experimental/metrics/rocauc.py

@@ -0,0 +1,49 @@
+from .base_metric import Metric
+
+
+class RocAuc(Metric):
+    """
+    Receiver Operating Characteristic/Area Under the Curve is the aggregated performance measure,
+    that depends only on the order of recommended items.
+    It can be interpreted as the fraction of object pairs (object of class 1, object of class 0)
+    that were correctly ordered by the model.
+    The bigger the value of AUC, the better the classification model.
+
+    .. math::
+        ROCAUC@K(i) = \\frac {\\sum_{s=1}^{K}\\sum_{t=1}^{K}
+        \\mathbb{1}_{r_{si}<r_{ti}}
+        \\mathbb{1}_{gt_{si}<gt_{ti}}}
+        {\\sum_{s=1}^{K}\\sum_{t=1}^{K} \\mathbb{1}_{gt_{si}<gt_{tj}}}
+
+    :math:`\\mathbb{1}_{r_{si}<r_{ti}}` -- indicator function showing that recommendation score for
+    user :math:`i` for item :math:`s` is bigger than for item :math:`t`
+
+    :math:`\\mathbb{1}_{gt_{si}<gt_{ti}}` --  indicator function showing that
+    user :math:`i` values item :math:`s` more than item :math:`t`.
+
+    Metric is averaged by all users.
+
+    .. math::
+        ROCAUC@K = \\frac {\\sum_{i=1}^{N}ROCAUC@K(i)}{N}
+    """
+
+    _scala_udf_name = "getRocAucMetricValue"
+
+    @staticmethod
+    def _get_metric_value_by_user(k, pred, ground_truth) -> float:
+        length = min(k, len(pred))
+        if len(ground_truth) == 0 or len(pred) == 0:
+            return 0
+
+        fp_cur = 0
+        fp_cum = 0
+        for item in pred[:length]:
+            if item in ground_truth:
+                fp_cum += fp_cur
+            else:
+                fp_cur += 1
+        if fp_cur == length:
+            return 0
+        if fp_cum == 0:
+            return 1
+        return 1 - fp_cum / (fp_cur * (length - fp_cur))

replay/experimental/metrics/surprisal.py

@@ -0,0 +1,90 @@
+from typing import Optional
+
+import numpy as np
+
+from replay.utils import PYSPARK_AVAILABLE, DataFrameLike, SparkDataFrame
+from replay.utils.spark_utils import convert2spark, get_top_k_recs
+
+from .base_metric import RecOnlyMetric, fill_na_with_empty_array, filter_sort
+
+if PYSPARK_AVAILABLE:
+    from pyspark.sql import (
+        functions as sf,
+        types as st,
+    )
+
+
+class Surprisal(RecOnlyMetric):
+    """
+    Measures how many surprising rare items are present in recommendations.
+
+    .. math::
+        \\textit{Self-Information}(j)= -\\log_2 \\frac {u_j}{N}
+
+    :math:`u_j` -- number of users that interacted with item :math:`j`.
+    Cold items are treated as if they were rated by 1 user.
+    That is, if they appear in recommendations it will be completely unexpected.
+
+    Metric is normalized.
+
+    Surprisal for item :math:`j` is
+
+    .. math::
+        Surprisal(j)= \\frac {\\textit{Self-Information}(j)}{log_2 N}
+
+    Recommendation list surprisal is the average surprisal of items in it.
+
+    .. math::
+        Surprisal@K(i) = \\frac {\\sum_{j=1}^{K}Surprisal(j)} {K}
+
+    Final metric is averaged by users.
+
+    .. math::
+        Surprisal@K = \\frac {\\sum_{i=1}^{N}Surprisal@K(i)}{N}
+    """
+
+    _scala_udf_name = "getSurprisalMetricValue"
+
+    def __init__(self, log: DataFrameLike, use_scala_udf: bool = False):
+        """
+        Here we calculate self-information for each item
+
+        :param log: historical data
+        """
+        self._use_scala_udf = use_scala_udf
+        self.log = convert2spark(log)
+        n_users = self.log.select("user_idx").distinct().count()
+        self.item_weights = self.log.groupby("item_idx").agg(
+            (sf.log2(n_users / sf.countDistinct("user_idx")) / np.log2(n_users)).alias("rec_weight")
+        )
+
+    @staticmethod
+    def _get_metric_value_by_user(k, *args):
+        weigths = args[0]
+        return sum(weigths[:k]) / k
+
+    def _get_enriched_recommendations(
+        self,
+        recommendations: SparkDataFrame,
+        ground_truth: SparkDataFrame,  # noqa: ARG002
+        max_k: int,
+        ground_truth_users: Optional[DataFrameLike] = None,
+    ) -> SparkDataFrame:
+        recommendations = convert2spark(recommendations)
+        ground_truth_users = convert2spark(ground_truth_users)
+        recommendations = get_top_k_recs(recommendations, max_k)
+
+        recommendations = recommendations.join(self.item_weights, on="item_idx", how="left").fillna(1.0)
+        recommendations = filter_sort(recommendations, "rec_weight")
+        recommendations = recommendations.select("user_idx", sf.col("rec_weight")).withColumn(
+            "rec_weight",
+            sf.col("rec_weight").cast(st.ArrayType(st.DoubleType(), True)),
+        )
+        if ground_truth_users is not None:
+            recommendations = fill_na_with_empty_array(
+                recommendations.join(ground_truth_users, on="user_idx", how="right"),
+                "rec_weight",
+                self.item_weights.schema["rec_weight"].dataType,
+            )
+
+        return recommendations