replay-rec 0.16.0rc0__py3-none-any.whl → 0.17.0__py3-none-any.whl

replay/experimental/metrics/unexpectedness.py

@@ -1,74 +0,0 @@
-from typing import Optional
-
-from replay.utils import DataFrameLike, SparkDataFrame
-from replay.utils.spark_utils import convert2spark, get_top_k_recs
-
-from .base_metric import RecOnlyMetric, fill_na_with_empty_array, filter_sort
-
-
-# pylint: disable=too-few-public-methods
-class Unexpectedness(RecOnlyMetric):
-    """
-    Fraction of recommended items that are not present in some baseline recommendations.
-
-    >>> import pandas as pd
-    >>> from replay.utils.session_handler import get_spark_session, State
-    >>> spark = get_spark_session(1, 1)
-    >>> state = State(spark)
-
-    >>> log = pd.DataFrame({"user_idx": [1, 1, 1], "item_idx": [1, 2, 3], "relevance": [5, 5, 5], "timestamp": [1, 1, 1]})
-    >>> recs = pd.DataFrame({"user_idx": [1, 1, 1], "item_idx": [0, 0, 1], "relevance": [5, 5, 5], "timestamp": [1, 1, 1]})
-    >>> metric = Unexpectedness(log)
-    >>> round(metric(recs, 3), 2)
-    0.67
-    """
-
-    _scala_udf_name = "getUnexpectednessMetricValue"
-
-    def __init__(
-        self, pred: DataFrameLike,
-        use_scala_udf: bool = False
-    ):  # pylint: disable=super-init-not-called
-        """
-        :param pred: model predictions
-        """
-        self._use_scala_udf = use_scala_udf
-        self.pred = convert2spark(pred)
-
-    @staticmethod
-    def _get_metric_value_by_user(k, *args) -> float:
-        pred = args[0]
-        base_pred = args[1]
-        if len(pred) == 0:
-            return 0
-        return 1.0 - len(set(pred[:k]) & set(base_pred[:k])) / k
-
-    def _get_enriched_recommendations(
-        self,
-        recommendations: SparkDataFrame,
-        ground_truth: SparkDataFrame,
-        max_k: int,
-        ground_truth_users: Optional[DataFrameLike] = None,
-    ) -> SparkDataFrame:
-        recommendations = convert2spark(recommendations)
-        ground_truth_users = convert2spark(ground_truth_users)
-        base_pred = self.pred
-
-        # TO DO: preprocess base_recs once in __init__
-
-        base_recs = filter_sort(base_pred).withColumnRenamed("pred", "base_pred")
-
-        # if there are duplicates in recommendations,
-        # we will leave fewer than k recommendations after sort_udf
-        recommendations = get_top_k_recs(recommendations, k=max_k)
-        recommendations = filter_sort(recommendations)
-        recommendations = recommendations.join(base_recs, how="right", on=["user_idx"])
-
-        if ground_truth_users is not None:
-            recommendations = recommendations.join(
-                ground_truth_users, on="user_idx", how="right"
-            )
-
-        return fill_na_with_empty_array(
-            recommendations, "pred", base_pred.schema["item_idx"].dataType
-        )

replay/experimental/models/__init__.py

@@ -1,10 +0,0 @@
-from replay.experimental.models.admm_slim import ADMMSLIM
-from replay.experimental.models.base_torch_rec import TorchRecommender
-from replay.experimental.models.ddpg import DDPG
-from replay.experimental.models.dt4rec.dt4rec import DT4Rec
-from replay.experimental.models.implicit_wrap import ImplicitWrap
-from replay.experimental.models.lightfm_wrap import LightFMWrap
-from replay.experimental.models.mult_vae import MultVAE
-from replay.experimental.models.neuromf import NeuroMF
-from replay.experimental.models.scala_als import ScalaALSWrap
-from replay.experimental.models.cql import CQL

replay/experimental/models/admm_slim.py

@@ -1,216 +0,0 @@
-from typing import Any, Dict, Optional, Tuple
-
-import numba as nb
-import numpy as np
-import pandas as pd
-from scipy.sparse import coo_matrix, csr_matrix
-
-from replay.experimental.models.base_neighbour_rec import NeighbourRec
-from replay.experimental.utils.session_handler import State
-from replay.models.extensions.ann.index_builders.base_index_builder import IndexBuilder
-from replay.utils import SparkDataFrame
-
-
-# pylint: disable=too-many-arguments, too-many-locals
-@nb.njit(parallel=True)
-def _main_iteration(
-    inv_matrix,
-    p_x,
-    mat_b,
-    mat_c,
-    mat_gamma,
-    rho,
-    eps_abs,
-    eps_rel,
-    lambda_1,
-    items_count,
-    threshold,
-    multiplicator,
-):  # pragma: no cover
-
-    # calculate mat_b
-    mat_b = p_x + np.dot(inv_matrix, rho * mat_c - mat_gamma)
-    vec_gamma = np.diag(mat_b) / np.diag(inv_matrix)
-    mat_b -= inv_matrix * vec_gamma
-
-    # calculate mat_c
-    prev_mat_c = mat_c
-    mat_c = mat_b + mat_gamma / rho
-    coef = lambda_1 / rho
-    mat_c = np.maximum(mat_c - coef, 0.0) - np.maximum(-mat_c - coef, 0.0)
-
-    # calculate mat_gamma
-    mat_gamma += rho * (mat_b - mat_c)
-
-    # calculate residuals
-    r_primal = np.linalg.norm(mat_b - mat_c)
-    r_dual = np.linalg.norm(-rho * (mat_c - prev_mat_c))
-    eps_primal = eps_abs * items_count + eps_rel * max(
-        np.linalg.norm(mat_b), np.linalg.norm(mat_c)
-    )
-    eps_dual = eps_abs * items_count + eps_rel * np.linalg.norm(mat_gamma)
-    if r_primal > threshold * r_dual:
-        rho *= multiplicator
-    elif threshold * r_primal < r_dual:
-        rho /= multiplicator
-
-    return (
-        mat_b,
-        mat_c,
-        mat_gamma,
-        rho,
-        r_primal,
-        r_dual,
-        eps_primal,
-        eps_dual,
-    )
-
-
-# pylint: disable=too-many-instance-attributes, too-many-ancestors
-class ADMMSLIM(NeighbourRec):
-    """`ADMM SLIM: Sparse Recommendations for Many Users
-    <http://www.cs.columbia.edu/~jebara/papers/wsdm20_ADMM.pdf>`_
-
-    This is a modification for the basic SLIM model.
-    Recommendations are improved with Alternating Direction Method of Multipliers.
-    """
-
-    def _get_ann_infer_params(self) -> Dict[str, Any]:
-        return {
-            "features_col": None,
-        }
-
-    rho: float
-    threshold: float = 5
-    multiplicator: float = 2
-    eps_abs: float = 1.0e-3
-    eps_rel: float = 1.0e-3
-    max_iteration: int = 100
-    _mat_c: np.ndarray
-    _mat_b: np.ndarray
-    _mat_gamma: np.ndarray
-    _search_space = {
-        "lambda_1": {"type": "loguniform", "args": [1e-9, 50]},
-        "lambda_2": {"type": "loguniform", "args": [1e-9, 5000]},
-    }
-
-    def __init__(
-        self,
-        lambda_1: float = 5,
-        lambda_2: float = 5000,
-        seed: Optional[int] = None,
-        index_builder: Optional[IndexBuilder] = None,
-    ):
-        """
-        :param lambda_1: l1 regularization term
-        :param lambda_2: l2 regularization term
-        :param seed: random seed
-        :param index_builder: `IndexBuilder` instance that adds ANN functionality.
-            If not set, then ann will not be used.
-        """
-        if lambda_1 < 0 or lambda_2 <= 0:
-            raise ValueError("Invalid regularization parameters")
-        self.lambda_1 = lambda_1
-        self.lambda_2 = lambda_2
-        self.rho = lambda_2
-        self.seed = seed
-        if isinstance(index_builder, (IndexBuilder, type(None))):
-            self.index_builder = index_builder
-        elif isinstance(index_builder, dict):
-            self.init_builder_from_dict(index_builder)
-
-    @property
-    def _init_args(self):
-        return {
-            "lambda_1": self.lambda_1,
-            "lambda_2": self.lambda_2,
-            "seed": self.seed,
-        }
-
-    # pylint: disable=too-many-locals
-    def _fit(
-        self,
-        log: SparkDataFrame,
-        user_features: Optional[SparkDataFrame] = None,
-        item_features: Optional[SparkDataFrame] = None,
-    ) -> None:
-        self.logger.debug("Fitting ADMM SLIM")
-        pandas_log = log.select("user_idx", "item_idx", "relevance").toPandas()
-        interactions_matrix = csr_matrix(
-            (
-                pandas_log["relevance"],
-                (pandas_log["user_idx"], pandas_log["item_idx"]),
-            ),
-            shape=(self._user_dim, self._item_dim),
-        )
-        self.logger.debug("Gram matrix")
-        xtx = (interactions_matrix.T @ interactions_matrix).toarray()
-        self.logger.debug("Inverse matrix")
-        inv_matrix = np.linalg.inv(
-            xtx + (self.lambda_2 + self.rho) * np.eye(self._item_dim)
-        )
-        self.logger.debug("Main calculations")
-        p_x = inv_matrix @ xtx
-        mat_b, mat_c, mat_gamma = self._init_matrix(self._item_dim)
-        r_primal = np.linalg.norm(mat_b - mat_c)
-        r_dual = np.linalg.norm(self.rho * mat_c)
-        eps_primal, eps_dual = 0.0, 0.0
-        iteration = 0
-        while (
-            r_primal > eps_primal or r_dual > eps_dual
-        ) and iteration < self.max_iteration:
-            iteration += 1
-            (
-                mat_b,
-                mat_c,
-                mat_gamma,
-                self.rho,
-                r_primal,
-                r_dual,
-                eps_primal,
-                eps_dual,
-            ) = _main_iteration(
-                inv_matrix,
-                p_x,
-                mat_b,
-                mat_c,
-                mat_gamma,
-                self.rho,
-                self.eps_abs,
-                self.eps_rel,
-                self.lambda_1,
-                self._item_dim,
-                self.threshold,
-                self.multiplicator,
-            )
-            result_message = (
-                f"Iteration: {iteration}. primal gap: "
-                f"{r_primal - eps_primal:.5}; dual gap: "
-                f" {r_dual - eps_dual:.5}; rho: {self.rho}"
-            )
-            self.logger.debug(result_message)
-
-        mat_c_sparse = coo_matrix(mat_c)
-        mat_c_pd = pd.DataFrame(
-            {
-                "item_idx_one": mat_c_sparse.row.astype(np.int32),
-                "item_idx_two": mat_c_sparse.col.astype(np.int32),
-                "similarity": mat_c_sparse.data,
-            }
-        )
-        self.similarity = State().session.createDataFrame(
-            mat_c_pd,
-            schema="item_idx_one int, item_idx_two int, similarity double",
-        )
-        self.similarity.cache().count()
-
-    def _init_matrix(
-        self, size: int
-    ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]:
-        """Matrix initialization"""
-        if self.seed is not None:
-            np.random.seed(self.seed)
-        mat_b = np.random.rand(size, size)  # type: ignore
-        mat_c = np.random.rand(size, size)  # type: ignore
-        mat_gamma = np.random.rand(size, size)  # type: ignore
-        return mat_b, mat_c, mat_gamma

replay/experimental/models/base_neighbour_rec.py

@@ -1,222 +0,0 @@
-# pylint: disable=too-many-lines
-"""
-NeighbourRec - base class that requires log at prediction time.
-Part of set of abstract classes (from base_rec.py)
-"""
-
-from abc import ABC
-from typing import Any, Dict, Iterable, Optional, Union
-
-from replay.experimental.models.base_rec import Recommender
-from replay.models.extensions.ann.ann_mixin import ANNMixin
-from replay.utils import PYSPARK_AVAILABLE, SparkDataFrame
-
-if PYSPARK_AVAILABLE:
-    from pyspark.sql import functions as sf
-    from pyspark.sql.column import Column
-
-
-class NeighbourRec(Recommender, ANNMixin, ABC):
-    """Base class that requires log at prediction time"""
-
-    similarity: Optional[SparkDataFrame]
-    can_predict_item_to_item: bool = True
-    can_predict_cold_users: bool = True
-    can_change_metric: bool = False
-    item_to_item_metrics = ["similarity"]
-    _similarity_metric = "similarity"
-
-    @property
-    def _dataframes(self):
-        return {"similarity": self.similarity}
-
-    def _clear_cache(self):
-        if hasattr(self, "similarity"):
-            self.similarity.unpersist()
-
-    # pylint: disable=missing-function-docstring
-    @property
-    def similarity_metric(self):
-        return self._similarity_metric
-
-    @similarity_metric.setter
-    def similarity_metric(self, value):
-        if not self.can_change_metric:
-            raise ValueError(
-                "This class does not support changing similarity metrics"
-            )
-        if value not in self.item_to_item_metrics:
-            raise ValueError(
-                f"Select one of the valid metrics for predict: "
-                f"{self.item_to_item_metrics}"
-            )
-        self._similarity_metric = value
-
-    def _predict_pairs_inner(
-        self,
-        log: SparkDataFrame,
-        filter_df: SparkDataFrame,
-        condition: Column,
-        users: SparkDataFrame,
-    ) -> SparkDataFrame:
-        """
-        Get recommendations for all provided users
-        and filter results with ``filter_df`` by ``condition``.
-        It allows to implement both ``predict_pairs`` and usual ``predict``@k.
-
-        :param log: historical interactions, SparkDataFrame
-            ``[user_idx, item_idx, timestamp, relevance]``.
-        :param filter_df: SparkDataFrame use to filter items:
-            ``[item_idx_filter]`` or ``[user_idx_filter, item_idx_filter]``.
-        :param condition: condition used for inner join with ``filter_df``
-        :param users: users to calculate recommendations for
-        :return: SparkDataFrame ``[user_idx, item_idx, relevance]``
-        """
-        if log is None:
-            raise ValueError(
-                "log is not provided, but it is required for prediction"
-            )
-
-        recs = (
-            log.join(users, how="inner", on="user_idx")
-            .join(
-                self.similarity,
-                how="inner",
-                on=sf.col("item_idx") == sf.col("item_idx_one"),
-            )
-            .join(
-                filter_df,
-                how="inner",
-                on=condition,
-            )
-            .groupby("user_idx", "item_idx_two")
-            .agg(sf.sum(self.similarity_metric).alias("relevance"))
-            .withColumnRenamed("item_idx_two", "item_idx")
-        )
-        return recs
-
-    # pylint: disable=too-many-arguments
-    def _predict(
-        self,
-        log: SparkDataFrame,
-        k: int,
-        users: SparkDataFrame,
-        items: SparkDataFrame,
-        user_features: Optional[SparkDataFrame] = None,
-        item_features: Optional[SparkDataFrame] = None,
-        filter_seen_items: bool = True,
-    ) -> SparkDataFrame:
-
-        return self._predict_pairs_inner(
-            log=log,
-            filter_df=items.withColumnRenamed("item_idx", "item_idx_filter"),
-            condition=sf.col("item_idx_two") == sf.col("item_idx_filter"),
-            users=users,
-        )
-
-    def _predict_pairs(
-        self,
-        pairs: SparkDataFrame,
-        log: Optional[SparkDataFrame] = None,
-        user_features: Optional[SparkDataFrame] = None,
-        item_features: Optional[SparkDataFrame] = None,
-    ) -> SparkDataFrame:
-
-        if log is None:
-            raise ValueError(
-                "log is not provided, but it is required for prediction"
-            )
-
-        return self._predict_pairs_inner(
-            log=log,
-            filter_df=(
-                pairs.withColumnRenamed(
-                    "user_idx", "user_idx_filter"
-                ).withColumnRenamed("item_idx", "item_idx_filter")
-            ),
-            condition=(sf.col("user_idx") == sf.col("user_idx_filter"))
-            & (sf.col("item_idx_two") == sf.col("item_idx_filter")),
-            users=pairs.select("user_idx").distinct(),
-        )
-
-    def get_nearest_items(
-        self,
-        items: Union[SparkDataFrame, Iterable],
-        k: int,
-        metric: Optional[str] = None,
-        candidates: Optional[Union[SparkDataFrame, Iterable]] = None,
-    ) -> SparkDataFrame:
-        """
-        Get k most similar items be the `metric` for each of the `items`.
-
-        :param items: spark dataframe or list of item ids to find neighbors
-        :param k: number of neighbors
-        :param metric: metric is not used to find neighbours in NeighbourRec,
-            the parameter is ignored
-        :param candidates: spark dataframe or list of items
-            to consider as similar, e.g. popular/new items. If None,
-            all items presented during model training are used.
-        :return: dataframe with the most similar items an distance,
-            where bigger value means greater similarity.
-            spark-dataframe with columns ``[item_idx, neighbour_item_idx, similarity]``
-        """
-
-        if metric is not None:
-            self.logger.debug(
-                "Metric is not used to determine nearest items in %s model",
-                str(self),
-            )
-
-        return self._get_nearest_items_wrap(
-            items=items,
-            k=k,
-            metric=metric,
-            candidates=candidates,
-        )
-
-    def _get_nearest_items(
-        self,
-        items: SparkDataFrame,
-        metric: Optional[str] = None,
-        candidates: Optional[SparkDataFrame] = None,
-    ) -> SparkDataFrame:
-
-        similarity_filtered = self.similarity.join(
-            items.withColumnRenamed("item_idx", "item_idx_one"),
-            on="item_idx_one",
-        )
-
-        if candidates is not None:
-            similarity_filtered = similarity_filtered.join(
-                candidates.withColumnRenamed("item_idx", "item_idx_two"),
-                on="item_idx_two",
-            )
-
-        return similarity_filtered.select(
-            "item_idx_one",
-            "item_idx_two",
-            "similarity" if metric is None else metric,
-        )
-
-    def _get_ann_build_params(self, interactions: SparkDataFrame) -> Dict[str, Any]:
-        self.index_builder.index_params.items_count = interactions.select(sf.max("item_idx")).first()[0] + 1
-        return {
-            "features_col": None,
-        }
-
-    def _get_vectors_to_build_ann(self, interactions: SparkDataFrame) -> SparkDataFrame:
-        similarity_df = self.similarity.select(
-            "similarity", "item_idx_one", "item_idx_two"
-        )
-        return similarity_df
-
-    def _get_vectors_to_infer_ann_inner(
-            self, interactions: SparkDataFrame, queries: SparkDataFrame
-    ) -> SparkDataFrame:
-
-        user_vectors = (
-            interactions.groupBy("user_idx").agg(
-                sf.collect_list("item_idx").alias("vector_items"),
-                sf.collect_list("relevance").alias("vector_relevances"))
-        )
-        return user_vectors