replay-rec 0.19.0__py3-none-any.whl → 0.19.0rc0__py3-none-any.whl

replay/experimental/metrics/unexpectedness.py

@@ -0,0 +1,76 @@
+from typing import Optional
+
+from replay.utils import DataFrameLike, SparkDataFrame
+from replay.utils.spark_utils import convert2spark, get_top_k_recs
+
+from .base_metric import RecOnlyMetric, fill_na_with_empty_array, filter_sort
+
+
+class Unexpectedness(RecOnlyMetric):
+    """
+    Fraction of recommended items that are not present in some baseline recommendations.
+
+    >>> import pandas as pd
+    >>> from replay.utils.session_handler import get_spark_session, State
+    >>> spark = get_spark_session(1, 1)
+    >>> state = State(spark)
+
+    >>> log = pd.DataFrame({
+    ...    "user_idx": [1, 1, 1],
+    ...    "item_idx": [1, 2, 3],
+    ...    "relevance": [5, 5, 5],
+    ...    "timestamp": [1, 1, 1],
+    ... })
+    >>> recs = pd.DataFrame({
+    ...    "user_idx": [1, 1, 1],
+    ...    "item_idx": [0, 0, 1],
+    ...    "relevance": [5, 5, 5],
+    ...    "timestamp": [1, 1, 1],
+    ... })
+    >>> metric = Unexpectedness(log)
+    >>> round(metric(recs, 3), 2)
+    0.67
+    """
+
+    _scala_udf_name = "getUnexpectednessMetricValue"
+
+    def __init__(self, pred: DataFrameLike, use_scala_udf: bool = False):
+        """
+        :param pred: model predictions
+        """
+        self._use_scala_udf = use_scala_udf
+        self.pred = convert2spark(pred)
+
+    @staticmethod
+    def _get_metric_value_by_user(k, *args) -> float:
+        pred = args[0]
+        base_pred = args[1]
+        if len(pred) == 0:
+            return 0
+        return 1.0 - len(set(pred[:k]) & set(base_pred[:k])) / k
+
+    def _get_enriched_recommendations(
+        self,
+        recommendations: SparkDataFrame,
+        ground_truth: SparkDataFrame,  # noqa: ARG002
+        max_k: int,
+        ground_truth_users: Optional[DataFrameLike] = None,
+    ) -> SparkDataFrame:
+        recommendations = convert2spark(recommendations)
+        ground_truth_users = convert2spark(ground_truth_users)
+        base_pred = self.pred
+
+        # TO DO: preprocess base_recs once in __init__
+
+        base_recs = filter_sort(base_pred).withColumnRenamed("pred", "base_pred")
+
+        # if there are duplicates in recommendations,
+        # we will leave fewer than k recommendations after sort_udf
+        recommendations = get_top_k_recs(recommendations, k=max_k)
+        recommendations = filter_sort(recommendations)
+        recommendations = recommendations.join(base_recs, how="right", on=["user_idx"])
+
+        if ground_truth_users is not None:
+            recommendations = recommendations.join(ground_truth_users, on="user_idx", how="right")
+
+        return fill_na_with_empty_array(recommendations, "pred", base_pred.schema["item_idx"].dataType)

replay/experimental/models/__init__.py

@@ -0,0 +1,13 @@
+from replay.experimental.models.admm_slim import ADMMSLIM
+from replay.experimental.models.base_torch_rec import TorchRecommender
+from replay.experimental.models.cql import CQL
+from replay.experimental.models.ddpg import DDPG
+from replay.experimental.models.dt4rec.dt4rec import DT4Rec
+from replay.experimental.models.hierarchical_recommender import HierarchicalRecommender
+from replay.experimental.models.implicit_wrap import ImplicitWrap
+from replay.experimental.models.lightfm_wrap import LightFMWrap
+from replay.experimental.models.mult_vae import MultVAE
+from replay.experimental.models.neural_ts import NeuralTS
+from replay.experimental.models.neuromf import NeuroMF
+from replay.experimental.models.scala_als import ScalaALSWrap
+from replay.experimental.models.u_lin_ucb import ULinUCB

replay/experimental/models/admm_slim.py

@@ -0,0 +1,205 @@
+from typing import Any, Dict, Optional, Tuple
+
+import numba as nb
+import numpy as np
+import pandas as pd
+from scipy.sparse import coo_matrix, csr_matrix
+
+from replay.experimental.models.base_neighbour_rec import NeighbourRec
+from replay.experimental.utils.session_handler import State
+from replay.models.extensions.ann.index_builders.base_index_builder import IndexBuilder
+from replay.utils import SparkDataFrame
+
+
+@nb.njit(parallel=True)
+def _main_iteration(
+    inv_matrix,
+    p_x,
+    mat_b,
+    mat_c,
+    mat_gamma,
+    rho,
+    eps_abs,
+    eps_rel,
+    lambda_1,
+    items_count,
+    threshold,
+    multiplicator,
+):  # pragma: no cover
+    # calculate mat_b
+    mat_b = p_x + np.dot(inv_matrix, rho * mat_c - mat_gamma)
+    vec_gamma = np.diag(mat_b) / np.diag(inv_matrix)
+    mat_b -= inv_matrix * vec_gamma
+
+    # calculate mat_c
+    prev_mat_c = mat_c
+    mat_c = mat_b + mat_gamma / rho
+    coef = lambda_1 / rho
+    mat_c = np.maximum(mat_c - coef, 0.0) - np.maximum(-mat_c - coef, 0.0)
+
+    # calculate mat_gamma
+    mat_gamma += rho * (mat_b - mat_c)
+
+    # calculate residuals
+    r_primal = np.linalg.norm(mat_b - mat_c)
+    r_dual = np.linalg.norm(-rho * (mat_c - prev_mat_c))
+    eps_primal = eps_abs * items_count + eps_rel * max(np.linalg.norm(mat_b), np.linalg.norm(mat_c))
+    eps_dual = eps_abs * items_count + eps_rel * np.linalg.norm(mat_gamma)
+    if r_primal > threshold * r_dual:
+        rho *= multiplicator
+    elif threshold * r_primal < r_dual:
+        rho /= multiplicator
+
+    return (
+        mat_b,
+        mat_c,
+        mat_gamma,
+        rho,
+        r_primal,
+        r_dual,
+        eps_primal,
+        eps_dual,
+    )
+
+
+class ADMMSLIM(NeighbourRec):
+    """`ADMM SLIM: Sparse Recommendations for Many Users
+    <http://www.cs.columbia.edu/~jebara/papers/wsdm20_ADMM.pdf>`_
+
+    This is a modification for the basic SLIM model.
+    Recommendations are improved with Alternating Direction Method of Multipliers.
+    """
+
+    def _get_ann_infer_params(self) -> Dict[str, Any]:
+        return {
+            "features_col": None,
+        }
+
+    rho: float
+    threshold: float = 5
+    multiplicator: float = 2
+    eps_abs: float = 1.0e-3
+    eps_rel: float = 1.0e-3
+    max_iteration: int = 100
+    _mat_c: np.ndarray
+    _mat_b: np.ndarray
+    _mat_gamma: np.ndarray
+    _search_space = {
+        "lambda_1": {"type": "loguniform", "args": [1e-9, 50]},
+        "lambda_2": {"type": "loguniform", "args": [1e-9, 5000]},
+    }
+
+    def __init__(
+        self,
+        lambda_1: float = 5,
+        lambda_2: float = 5000,
+        seed: Optional[int] = None,
+        index_builder: Optional[IndexBuilder] = None,
+    ):
+        """
+        :param lambda_1: l1 regularization term
+        :param lambda_2: l2 regularization term
+        :param seed: random seed
+        :param index_builder: `IndexBuilder` instance that adds ANN functionality.
+            If not set, then ann will not be used.
+        """
+        if lambda_1 < 0 or lambda_2 <= 0:
+            msg = "Invalid regularization parameters"
+            raise ValueError(msg)
+        self.lambda_1 = lambda_1
+        self.lambda_2 = lambda_2
+        self.rho = lambda_2
+        self.seed = seed
+        if isinstance(index_builder, (IndexBuilder, type(None))):
+            self.index_builder = index_builder
+        elif isinstance(index_builder, dict):
+            self.init_builder_from_dict(index_builder)
+
+    @property
+    def _init_args(self):
+        return {
+            "lambda_1": self.lambda_1,
+            "lambda_2": self.lambda_2,
+            "seed": self.seed,
+        }
+
+    def _fit(
+        self,
+        log: SparkDataFrame,
+        user_features: Optional[SparkDataFrame] = None,  # noqa: ARG002
+        item_features: Optional[SparkDataFrame] = None,  # noqa: ARG002
+    ) -> None:
+        self.logger.debug("Fitting ADMM SLIM")
+        pandas_log = log.select("user_idx", "item_idx", "relevance").toPandas()
+        interactions_matrix = csr_matrix(
+            (
+                pandas_log["relevance"],
+                (pandas_log["user_idx"], pandas_log["item_idx"]),
+            ),
+            shape=(self._user_dim, self._item_dim),
+        )
+        self.logger.debug("Gram matrix")
+        xtx = (interactions_matrix.T @ interactions_matrix).toarray()
+        self.logger.debug("Inverse matrix")
+        inv_matrix = np.linalg.inv(xtx + (self.lambda_2 + self.rho) * np.eye(self._item_dim))
+        self.logger.debug("Main calculations")
+        p_x = inv_matrix @ xtx
+        mat_b, mat_c, mat_gamma = self._init_matrix(self._item_dim)
+        r_primal = np.linalg.norm(mat_b - mat_c)
+        r_dual = np.linalg.norm(self.rho * mat_c)
+        eps_primal, eps_dual = 0.0, 0.0
+        iteration = 0
+        while (r_primal > eps_primal or r_dual > eps_dual) and iteration < self.max_iteration:
+            iteration += 1
+            (
+                mat_b,
+                mat_c,
+                mat_gamma,
+                self.rho,
+                r_primal,
+                r_dual,
+                eps_primal,
+                eps_dual,
+            ) = _main_iteration(
+                inv_matrix,
+                p_x,
+                mat_b,
+                mat_c,
+                mat_gamma,
+                self.rho,
+                self.eps_abs,
+                self.eps_rel,
+                self.lambda_1,
+                self._item_dim,
+                self.threshold,
+                self.multiplicator,
+            )
+            result_message = (
+                f"Iteration: {iteration}. primal gap: "
+                f"{r_primal - eps_primal:.5}; dual gap: "
+                f" {r_dual - eps_dual:.5}; rho: {self.rho}"
+            )
+            self.logger.debug(result_message)
+
+        mat_c_sparse = coo_matrix(mat_c)
+        mat_c_pd = pd.DataFrame(
+            {
+                "item_idx_one": mat_c_sparse.row.astype(np.int32),
+                "item_idx_two": mat_c_sparse.col.astype(np.int32),
+                "similarity": mat_c_sparse.data,
+            }
+        )
+        self.similarity = State().session.createDataFrame(
+            mat_c_pd,
+            schema="item_idx_one int, item_idx_two int, similarity double",
+        )
+        self.similarity.cache().count()
+
+    def _init_matrix(self, size: int) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
+        """Matrix initialization"""
+        if self.seed is not None:
+            np.random.seed(self.seed)
+        mat_b = np.random.rand(size, size)
+        mat_c = np.random.rand(size, size)
+        mat_gamma = np.random.rand(size, size)
+        return mat_b, mat_c, mat_gamma

replay/experimental/models/base_neighbour_rec.py

@@ -0,0 +1,204 @@
+"""
+NeighbourRec - base class that requires log at prediction time.
+Part of set of abstract classes (from base_rec.py)
+"""
+
+from abc import ABC
+from typing import Any, Dict, Iterable, Optional, Union
+
+from replay.experimental.models.base_rec import Recommender
+from replay.models.extensions.ann.ann_mixin import ANNMixin
+from replay.utils import PYSPARK_AVAILABLE, SparkDataFrame
+
+if PYSPARK_AVAILABLE:
+    from pyspark.sql import functions as sf
+    from pyspark.sql.column import Column
+
+
+class NeighbourRec(Recommender, ANNMixin, ABC):
+    """Base class that requires log at prediction time"""
+
+    similarity: Optional[SparkDataFrame]
+    can_predict_item_to_item: bool = True
+    can_predict_cold_users: bool = True
+    can_change_metric: bool = False
+    item_to_item_metrics = ["similarity"]
+    _similarity_metric = "similarity"
+
+    @property
+    def _dataframes(self):
+        return {"similarity": self.similarity}
+
+    def _clear_cache(self):
+        if hasattr(self, "similarity"):
+            self.similarity.unpersist()
+
+    @property
+    def similarity_metric(self):
+        return self._similarity_metric
+
+    @similarity_metric.setter
+    def similarity_metric(self, value):
+        if not self.can_change_metric:
+            msg = "This class does not support changing similarity metrics"
+            raise ValueError(msg)
+        if value not in self.item_to_item_metrics:
+            msg = f"Select one of the valid metrics for predict: {self.item_to_item_metrics}"
+            raise ValueError(msg)
+        self._similarity_metric = value
+
+    def _predict_pairs_inner(
+        self,
+        log: SparkDataFrame,
+        filter_df: SparkDataFrame,
+        condition: Column,
+        users: SparkDataFrame,
+    ) -> SparkDataFrame:
+        """
+        Get recommendations for all provided users
+        and filter results with ``filter_df`` by ``condition``.
+        It allows to implement both ``predict_pairs`` and usual ``predict``@k.
+
+        :param log: historical interactions, SparkDataFrame
+            ``[user_idx, item_idx, timestamp, relevance]``.
+        :param filter_df: SparkDataFrame use to filter items:
+            ``[item_idx_filter]`` or ``[user_idx_filter, item_idx_filter]``.
+        :param condition: condition used for inner join with ``filter_df``
+        :param users: users to calculate recommendations for
+        :return: SparkDataFrame ``[user_idx, item_idx, relevance]``
+        """
+        if log is None:
+            msg = "log is not provided, but it is required for prediction"
+            raise ValueError(msg)
+
+        recs = (
+            log.join(users, how="inner", on="user_idx")
+            .join(
+                self.similarity,
+                how="inner",
+                on=sf.col("item_idx") == sf.col("item_idx_one"),
+            )
+            .join(
+                filter_df,
+                how="inner",
+                on=condition,
+            )
+            .groupby("user_idx", "item_idx_two")
+            .agg(sf.sum(self.similarity_metric).alias("relevance"))
+            .withColumnRenamed("item_idx_two", "item_idx")
+        )
+        return recs
+
+    def _predict(
+        self,
+        log: SparkDataFrame,
+        k: int,  # noqa: ARG002
+        users: SparkDataFrame,
+        items: SparkDataFrame,
+        user_features: Optional[SparkDataFrame] = None,  # noqa: ARG002
+        item_features: Optional[SparkDataFrame] = None,  # noqa: ARG002
+        filter_seen_items: bool = True,  # noqa: ARG002
+    ) -> SparkDataFrame:
+        return self._predict_pairs_inner(
+            log=log,
+            filter_df=items.withColumnRenamed("item_idx", "item_idx_filter"),
+            condition=sf.col("item_idx_two") == sf.col("item_idx_filter"),
+            users=users,
+        )
+
+    def _predict_pairs(
+        self,
+        pairs: SparkDataFrame,
+        log: Optional[SparkDataFrame] = None,
+        user_features: Optional[SparkDataFrame] = None,  # noqa: ARG002
+        item_features: Optional[SparkDataFrame] = None,  # noqa: ARG002
+    ) -> SparkDataFrame:
+        if log is None:
+            msg = "log is not provided, but it is required for prediction"
+            raise ValueError(msg)
+
+        return self._predict_pairs_inner(
+            log=log,
+            filter_df=(
+                pairs.withColumnRenamed("user_idx", "user_idx_filter").withColumnRenamed("item_idx", "item_idx_filter")
+            ),
+            condition=(sf.col("user_idx") == sf.col("user_idx_filter"))
+            & (sf.col("item_idx_two") == sf.col("item_idx_filter")),
+            users=pairs.select("user_idx").distinct(),
+        )
+
+    def get_nearest_items(
+        self,
+        items: Union[SparkDataFrame, Iterable],
+        k: int,
+        metric: Optional[str] = None,
+        candidates: Optional[Union[SparkDataFrame, Iterable]] = None,
+    ) -> SparkDataFrame:
+        """
+        Get k most similar items be the `metric` for each of the `items`.
+
+        :param items: spark dataframe or list of item ids to find neighbors
+        :param k: number of neighbors
+        :param metric: metric is not used to find neighbours in NeighbourRec,
+            the parameter is ignored
+        :param candidates: spark dataframe or list of items
+            to consider as similar, e.g. popular/new items. If None,
+            all items presented during model training are used.
+        :return: dataframe with the most similar items an distance,
+            where bigger value means greater similarity.
+            spark-dataframe with columns ``[item_idx, neighbour_item_idx, similarity]``
+        """
+
+        if metric is not None:
+            self.logger.debug(
+                "Metric is not used to determine nearest items in %s model",
+                str(self),
+            )
+
+        return self._get_nearest_items_wrap(
+            items=items,
+            k=k,
+            metric=metric,
+            candidates=candidates,
+        )
+
+    def _get_nearest_items(
+        self,
+        items: SparkDataFrame,
+        metric: Optional[str] = None,
+        candidates: Optional[SparkDataFrame] = None,
+    ) -> SparkDataFrame:
+        similarity_filtered = self.similarity.join(
+            items.withColumnRenamed("item_idx", "item_idx_one"),
+            on="item_idx_one",
+        )
+
+        if candidates is not None:
+            similarity_filtered = similarity_filtered.join(
+                candidates.withColumnRenamed("item_idx", "item_idx_two"),
+                on="item_idx_two",
+            )
+
+        return similarity_filtered.select(
+            "item_idx_one",
+            "item_idx_two",
+            "similarity" if metric is None else metric,
+        )
+
+    def _get_ann_build_params(self, interactions: SparkDataFrame) -> Dict[str, Any]:
+        self.index_builder.index_params.items_count = interactions.select(sf.max("item_idx")).first()[0] + 1
+        return {
+            "features_col": None,
+        }
+
+    def _get_vectors_to_build_ann(self, interactions: SparkDataFrame) -> SparkDataFrame:  # noqa: ARG002
+        similarity_df = self.similarity.select("similarity", "item_idx_one", "item_idx_two")
+        return similarity_df
+
+    def _get_vectors_to_infer_ann_inner(
+        self, interactions: SparkDataFrame, queries: SparkDataFrame  # noqa: ARG002
+    ) -> SparkDataFrame:
+        user_vectors = interactions.groupBy("user_idx").agg(
+            sf.collect_list("item_idx").alias("vector_items"), sf.collect_list("relevance").alias("vector_relevances")
+        )
+        return user_vectors