replay-rec 0.17.1rc0__py3-none-any.whl → 0.18.0__py3-none-any.whl

replay/__init__.py

@@ -1,2 +1,3 @@
 """ RecSys library """
-__version__ = "0.17.1.preview"
+
+__version__ = "0.18.0"

replay/data/dataset.py

@@ -1,6 +1,7 @@
 """
 ``Dataset`` universal dataset class for manipulating interactions and feed data to models.
 """
+
 from __future__ import annotations
 
 import json
@@ -606,7 +607,7 @@ class Dataset:
         if self.is_pandas:
             min_id = data[column].min()
         elif self.is_spark:
-            min_id = data.agg(sf.min(column).alias("min_index")).collect()[0][0]
+            min_id = data.agg(sf.min(column).alias("min_index")).first()[0]
         else:
             min_id = data[column].min()
         if min_id < 0:
@@ -616,7 +617,7 @@ class Dataset:
         if self.is_pandas:
             max_id = data[column].max()
         elif self.is_spark:
-            max_id = data.agg(sf.max(column).alias("max_index")).collect()[0][0]
+            max_id = data.agg(sf.max(column).alias("max_index")).first()[0]
         else:
             max_id = data[column].max()
 

replay/data/dataset_utils/dataset_label_encoder.py

@@ -4,6 +4,7 @@ Contains classes for encoding categorical data
 ``LabelEncoderTransformWarning`` new category of warning for DatasetLabelEncoder.
 ``DatasetLabelEncoder`` to encode categorical features in `Dataset` objects.
 """
+
 import warnings
 from typing import Dict, Iterable, Iterator, Optional, Sequence, Set, Union
 

replay/data/nn/schema.py

@@ -418,11 +418,11 @@ class TensorSchema(Mapping[str, TensorFeatureInfo]):
                 "feature_type": feature.feature_type.name,
                 "is_seq": feature.is_seq,
                 "feature_hint": feature.feature_hint.name if feature.feature_hint else None,
-                "feature_sources": [
-                    {"source": x.source.name, "column": x.column, "index": x.index} for x in feature.feature_sources
-                ]
-                if feature.feature_sources
-                else None,
+                "feature_sources": (
+                    [{"source": x.source.name, "column": x.column, "index": x.index} for x in feature.feature_sources]
+                    if feature.feature_sources
+                    else None
+                ),
                 "cardinality": feature.cardinality if feature.feature_type == FeatureType.CATEGORICAL else None,
                 "embedding_dim": feature.embedding_dim if feature.feature_type == FeatureType.CATEGORICAL else None,
                 "tensor_dim": feature.tensor_dim if feature.feature_type == FeatureType.NUMERICAL else None,

replay/metrics/__init__.py

@@ -42,6 +42,7 @@ For each metric, a formula for its calculation is given, because this is
 important for the correct comparison of algorithms, as mentioned in our
 `article <https://arxiv.org/abs/2206.12858>`_.
 """
+
 from .base_metric import Metric
 from .categorical_diversity import CategoricalDiversity
 from .coverage import Coverage

replay/models/als.py

@@ -115,7 +115,7 @@ class ALSWrap(Recommender, ItemVectorModel):
                     .groupBy(self.query_column)
                     .agg(sf.count(self.query_column).alias("num_seen"))
                     .select(sf.max("num_seen"))
-                    .collect()[0][0]
+                    .first()[0]
                 )
                 max_seen = max_seen_in_interactions if max_seen_in_interactions is not None else 0
 

replay/models/base_rec.py

@@ -401,8 +401,8 @@ class BaseRecommender(RecommenderCommons, IsSavable, ABC):
         self.fit_items = sf.broadcast(items)
         self._num_queries = self.fit_queries.count()
         self._num_items = self.fit_items.count()
-        self._query_dim_size = self.fit_queries.agg({self.query_column: "max"}).collect()[0][0] + 1
-        self._item_dim_size = self.fit_items.agg({self.item_column: "max"}).collect()[0][0] + 1
+        self._query_dim_size = self.fit_queries.agg({self.query_column: "max"}).first()[0] + 1
+        self._item_dim_size = self.fit_items.agg({self.item_column: "max"}).first()[0] + 1
         self._fit(dataset)
 
     @abstractmethod
@@ -431,7 +431,7 @@ class BaseRecommender(RecommenderCommons, IsSavable, ABC):
         # count maximal number of items seen by queries
         max_seen = 0
         if num_seen.count() > 0:
-            max_seen = num_seen.select(sf.max("seen_count")).collect()[0][0]
+            max_seen = num_seen.select(sf.max("seen_count")).first()[0]
 
         # crop recommendations to first k + max_seen items for each query
         recs = recs.withColumn(
@@ -708,7 +708,7 @@ class BaseRecommender(RecommenderCommons, IsSavable, ABC):
             setattr(
                 self,
                 dim_size,
-                fit_entities.agg({column: "max"}).collect()[0][0] + 1,
+                fit_entities.agg({column: "max"}).first()[0] + 1,
             )
         return getattr(self, dim_size)
 
@@ -1426,7 +1426,7 @@ class NonPersonalizedRecommender(Recommender, ABC):
         Calculating a fill value a the minimal rating
         calculated during model training multiplied by weight.
         """
-        return item_popularity.select(sf.min(rating_column)).collect()[0][0] * weight
+        return item_popularity.select(sf.min(rating_column)).first()[0] * weight
 
     @staticmethod
     def _check_rating(dataset: Dataset):
@@ -1460,7 +1460,7 @@ class NonPersonalizedRecommender(Recommender, ABC):
                 .agg(sf.countDistinct(item_column).alias("items_count"))
             )
             .select(sf.max("items_count"))
-            .collect()[0][0]
+            .first()[0]
         )
         # all queries have empty history
         if max_hist_len is None:
@@ -1495,7 +1495,7 @@ class NonPersonalizedRecommender(Recommender, ABC):
             queries = queries.join(query_to_num_items, on=self.query_column, how="left")
             queries = queries.fillna(0, "num_items")
             # 'selected_item_popularity' truncation by k + max_seen
-            max_seen = queries.select(sf.coalesce(sf.max("num_items"), sf.lit(0))).collect()[0][0]
+            max_seen = queries.select(sf.coalesce(sf.max("num_items"), sf.lit(0))).first()[0]
             selected_item_popularity = selected_item_popularity.filter(sf.col("rank") <= k + max_seen)
             return queries.join(selected_item_popularity, on=(sf.col("rank") <= k + sf.col("num_items")), how="left")
 

replay/models/extensions/ann/index_inferers/nmslib_filter_index_inferer.py

@@ -32,9 +32,9 @@ class NmslibFilterIndexInferer(IndexInferer):
             index = index_store.load_index(
                 init_index=lambda: create_nmslib_index_instance(index_params),
                 load_index=lambda index, path: index.loadIndex(path, load_data=True),
-                configure_index=lambda index: index.setQueryTimeParams({"efSearch": index_params.ef_s})
-                if index_params.ef_s
-                else None,
+                configure_index=lambda index: (
+                    index.setQueryTimeParams({"efSearch": index_params.ef_s}) if index_params.ef_s else None
+                ),
             )
 
             # max number of items to retrieve per batch

replay/models/extensions/ann/index_inferers/nmslib_index_inferer.py

@@ -30,9 +30,9 @@ class NmslibIndexInferer(IndexInferer):
             index = index_store.load_index(
                 init_index=lambda: create_nmslib_index_instance(index_params),
                 load_index=lambda index, path: index.loadIndex(path, load_data=True),
-                configure_index=lambda index: index.setQueryTimeParams({"efSearch": index_params.ef_s})
-                if index_params.ef_s
-                else None,
+                configure_index=lambda index: (
+                    index.setQueryTimeParams({"efSearch": index_params.ef_s}) if index_params.ef_s else None
+                ),
             )
 
             user_vectors = get_csr_matrix(user_idx, vector_items, vector_ratings)

replay/models/nn/sequential/bert4rec/model.py

@@ -1,7 +1,7 @@
 import contextlib
 import math
 from abc import ABC, abstractmethod
-from typing import Dict, Optional, Tuple, Union, cast
+from typing import Dict, Optional, Union
 
 import torch
 
@@ -115,13 +115,10 @@ class Bert4RecModel(torch.nn.Module):
         # (B x L x E)
         x = self.item_embedder(inputs, token_mask)
 
-        # (B x 1 x L x L)
-        pad_mask_for_attention = self._get_attention_mask_from_padding(pad_mask)
-
         # Running over multiple transformer blocks
         for transformer in self.transformer_blocks:
             for _ in range(self.num_passes_over_block):
-                x = transformer(x, pad_mask_for_attention)
+                x = transformer(x, pad_mask)
 
         return x
 
@@ -147,11 +144,6 @@ class Bert4RecModel(torch.nn.Module):
         """
         return self.forward_step(inputs, pad_mask, token_mask)[:, -1, :]
 
-    def _get_attention_mask_from_padding(self, pad_mask: torch.BoolTensor) -> torch.BoolTensor:
-        # (B x L) -> (B x 1 x L x L)
-        pad_mask_for_attention = pad_mask.unsqueeze(1).repeat(1, self.max_len, 1).unsqueeze(1)
-        return cast(torch.BoolTensor, pad_mask_for_attention)
-
     def _init(self) -> None:
         for _, param in self.named_parameters():
             with contextlib.suppress(ValueError):
@@ -456,7 +448,7 @@ class TransformerBlock(torch.nn.Module):
         :param dropout: Dropout rate.
         """
         super().__init__()
-        self.attention = MultiHeadedAttention(h=attn_heads, d_model=hidden_size, dropout=dropout)
+        self.attention = torch.nn.MultiheadAttention(hidden_size, attn_heads, dropout=dropout, batch_first=True)
         self.attention_dropout = torch.nn.Dropout(dropout)
         self.attention_norm = LayerNorm(hidden_size)
 
@@ -479,7 +471,8 @@ class TransformerBlock(torch.nn.Module):
         """
         # Attention + skip-connection
         x_norm = self.attention_norm(x)
-        y = x + self.attention_dropout(self.attention(x_norm, x_norm, x_norm, mask))
+        attent_emb, _ = self.attention(x_norm, x_norm, x_norm, key_padding_mask=~mask, need_weights=False)
+        y = x + self.attention_dropout(attent_emb)
 
         # PFF + skip-connection
         z = y + self.pff_dropout(self.pff(self.pff_norm(y)))
@@ -487,106 +480,6 @@ class TransformerBlock(torch.nn.Module):
         return self.dropout(z)
 
 
-class Attention(torch.nn.Module):
-    """
-    Compute Scaled Dot Product Attention
-    """
-
-    def __init__(self, dropout: float) -> None:
-        """
-        :param dropout: Dropout rate.
-        """
-        super().__init__()
-        self.dropout = torch.nn.Dropout(p=dropout)
-
-    def forward(
-        self, query: torch.Tensor, key: torch.Tensor, value: torch.Tensor, mask: torch.BoolTensor
-    ) -> Tuple[torch.Tensor, torch.Tensor]:
-        """
-        :param query: Query feature vector.
-        :param key: Key feature vector.
-        :param value: Value feature vector.
-        :param mask: Mask where 0 - <MASK>, 1 - otherwise.
-
-        :returns: Tuple of scaled dot product attention
-                and attention logits for each element.
-        """
-        scores = torch.matmul(query, key.transpose(-2, -1)) / math.sqrt(query.size(-1))
-
-        scores = scores.masked_fill(mask == 0, -1e9)
-        p_attn = torch.nn.functional.softmax(scores, dim=-1)
-        p_attn = self.dropout(p_attn)
-
-        return torch.matmul(p_attn, value), p_attn
-
-
-class MultiHeadedAttention(torch.nn.Module):
-    """
-    Take in model size and number of heads.
-    """
-
-    def __init__(self, h: int, d_model: int, dropout: float = 0.1) -> None:
-        """
-        :param h: Head sizes of multi-head attention.
-        :param d_model: Embedding dimension.
-        :param dropout: Dropout rate.
-            Default: ``0.1``.
-        """
-        super().__init__()
-        assert d_model % h == 0
-
-        # We assume d_v always equals d_k
-        self.d_k = d_model // h
-        self.h = h
-
-        # 3 linear projections for Q, K, V
-        self.qkv_linear_layers = torch.nn.ModuleList([torch.nn.Linear(d_model, d_model) for _ in range(3)])
-
-        # 2 linear projections for P -> P_q, P_k
-        self.pos_linear_layers = torch.nn.ModuleList([torch.nn.Linear(d_model, d_model) for _ in range(2)])
-
-        self.output_linear = torch.nn.Linear(d_model, d_model)
-
-        self.attention = Attention(dropout)
-
-    def forward(
-        self,
-        query: torch.Tensor,
-        key: torch.Tensor,
-        value: torch.Tensor,
-        mask: torch.BoolTensor,
-    ) -> torch.Tensor:
-        """
-        :param query: Query feature vector.
-        :param key: Key feature vector.
-        :param value: Value feature vector.
-        :param mask: Mask where 0 - <MASK>, 1 - otherwise.
-
-        :returns: Attention outputs.
-        """
-        batch_size = query.size(0)
-
-        # B - batch size
-        # L - sequence length (max_len)
-        # E - embedding size for tokens fed into transformer
-        # K - max relative distance
-        # H - attention head count
-
-        # Do all the linear projections in batch from d_model => h x d_k
-        # (B x L x E) -> (B x H x L x (E / H))
-        query, key, value = [
-            layer(x).view(batch_size, -1, self.h, self.d_k).transpose(1, 2)
-            for layer, x in zip(self.qkv_linear_layers, (query, key, value))
-        ]
-
-        x, _ = self.attention(query, key, value, mask)
-
-        # Concat using a view and apply a final linear.
-        x = x.transpose(1, 2).contiguous().view(batch_size, -1, self.h * self.d_k)
-
-        return self.output_linear(x)
-
-
 class LayerNorm(torch.nn.Module):
     """
     Construct a layernorm module (See citation for details).

replay/models/nn/sequential/sasrec/model.py

@@ -401,7 +401,12 @@ class SasRecLayers(torch.nn.Module):
         """
         super().__init__()
         self.attention_layers = self._layers_stacker(
-            num_blocks, torch.nn.MultiheadAttention, hidden_size, num_heads, dropout
+            num_blocks,
+            torch.nn.MultiheadAttention,
+            hidden_size,
+            num_heads,
+            dropout,
+            batch_first=True,
         )
         self.attention_layernorms = self._layers_stacker(num_blocks, torch.nn.LayerNorm, hidden_size, eps=1e-8)
         self.forward_layers = self._layers_stacker(num_blocks, SasRecPointWiseFeedForward, hidden_size, dropout)
@@ -422,11 +427,9 @@ class SasRecLayers(torch.nn.Module):
         """
         length = len(self.attention_layers)
         for i in range(length):
-            seqs = torch.transpose(seqs, 0, 1)
             query = self.attention_layernorms[i](seqs)
-            attent_emb, _ = self.attention_layers[i](query, seqs, seqs, attn_mask=attention_mask)
+            attent_emb, _ = self.attention_layers[i](query, seqs, seqs, attn_mask=attention_mask, need_weights=False)
             seqs = query + attent_emb
-            seqs = torch.transpose(seqs, 0, 1)
 
             seqs = self.forward_layernorms[i](seqs)
             seqs = self.forward_layers[i](seqs)
@@ -492,7 +495,7 @@ class SasRecPointWiseFeedForward(torch.nn.Module):
 
         :returns: Output tensors.
         """
-        outputs = self.dropout2(self.conv2(self.relu(self.dropout1(self.conv1(inputs.transpose(-1, -2))))))
+        outputs = self.dropout2(self.conv2(self.dropout1(self.relu(self.conv1(inputs.transpose(-1, -2))))))
         outputs = outputs.transpose(-1, -2)
         outputs += inputs
 

replay/optimization/optuna_objective.py

@@ -1,6 +1,7 @@
 """
 This class calculates loss function for optimization process
 """
+
 import collections
 import logging
 from functools import partial

replay/preprocessing/converter.py

@@ -102,6 +102,6 @@ class CSRConverter:
         row_count = self.row_count if self.row_count is not None else _get_max(rows_data) + 1
         col_count = self.column_count if self.column_count is not None else _get_max(cols_data) + 1
         return csr_matrix(
-            (data, (rows_data, cols_data)),
+            (data.tolist(), (rows_data.tolist(), cols_data.tolist())),
             shape=(row_count, col_count),
         )

replay/preprocessing/filters.py

@@ -1,6 +1,7 @@
 """
 Select or remove data by some criteria
 """
+
 from abc import ABC, abstractmethod
 from datetime import datetime, timedelta
 from typing import Callable, Optional, Tuple, Union
@@ -355,8 +356,8 @@ class NumInteractionsFilter(_BaseFilter):
     >>> log_pd = pd.DataFrame({"user_id": ["u1", "u2", "u2", "u3", "u3", "u3"],
     ...                     "item_id": ["i1", "i2","i3", "i1", "i2","i3"],
     ...                     "rating": [1., 0.5, 3, 1, 0, 1],
-    ...                     "timestamp": ["2020-01-01 23:59:59", "2020-02-01",
-    ...                                   "2020-02-01", "2020-01-01 00:04:15",
+    ...                     "timestamp": ["2020-01-01 23:59:59", "2020-02-01 00:00:00",
+    ...                                   "2020-02-01 00:00:01", "2020-01-01 00:04:15",
     ...                                   "2020-01-02 00:04:14", "2020-01-05 23:59:59"]},
     ...             )
     >>> log_pd["timestamp"] = pd.to_datetime(log_pd["timestamp"], format="ISO8601")
@@ -367,7 +368,7 @@ class NumInteractionsFilter(_BaseFilter):
     +-------+-------+------+-------------------+
     |     u1|     i1|   1.0|2020-01-01 23:59:59|
     |     u2|     i2|   0.5|2020-02-01 00:00:00|
-    |     u2|     i3|   3.0|2020-02-01 00:00:00|
+    |     u2|     i3|   3.0|2020-02-01 00:00:01|
     |     u3|     i1|   1.0|2020-01-01 00:04:15|
     |     u3|     i2|   0.0|2020-01-02 00:04:14|
     |     u3|     i3|   1.0|2020-01-05 23:59:59|
@@ -393,7 +394,7 @@ class NumInteractionsFilter(_BaseFilter):
     |user_id|item_id|rating|          timestamp|
     +-------+-------+------+-------------------+
     |     u1|     i1|   1.0|2020-01-01 23:59:59|
-    |     u2|     i2|   0.5|2020-02-01 00:00:00|
+    |     u2|     i3|   3.0|2020-02-01 00:00:01|
     |     u3|     i3|   1.0|2020-01-05 23:59:59|
     +-------+-------+------+-------------------+
     <BLANKLINE>
@@ -403,7 +404,7 @@ class NumInteractionsFilter(_BaseFilter):
     |user_id|item_id|rating|          timestamp|
     +-------+-------+------+-------------------+
     |     u1|     i1|   1.0|2020-01-01 23:59:59|
-    |     u2|     i3|   3.0|2020-02-01 00:00:00|
+    |     u2|     i3|   3.0|2020-02-01 00:00:01|
     |     u3|     i3|   1.0|2020-01-05 23:59:59|
     +-------+-------+------+-------------------+
     <BLANKLINE>
@@ -482,7 +483,7 @@ class NumInteractionsFilter(_BaseFilter):
 
         return (
             interactions.sort(sorting_columns, descending=descending)
-            .with_columns(pl.col(self.query_column).cumcount().over(self.query_column).alias("temp_rank"))
+            .with_columns(pl.col(self.query_column).cum_count().over(self.query_column).alias("temp_rank"))
             .filter(pl.col("temp_rank") <= self.num_interactions)
             .drop("temp_rank")
         )
@@ -497,8 +498,8 @@ class EntityDaysFilter(_BaseFilter):
     >>> log_pd = pd.DataFrame({"user_id": ["u1", "u2", "u2", "u3", "u3", "u3"],
     ...                     "item_id": ["i1", "i2","i3", "i1", "i2","i3"],
     ...                     "rating": [1., 0.5, 3, 1, 0, 1],
-    ...                     "timestamp": ["2020-01-01 23:59:59", "2020-02-01",
-    ...                                   "2020-02-01", "2020-01-01 00:04:15",
+    ...                     "timestamp": ["2020-01-01 23:59:59", "2020-02-01 00:00:00",
+    ...                                   "2020-02-01 00:00:01", "2020-01-01 00:04:15",
     ...                                   "2020-01-02 00:04:14", "2020-01-05 23:59:59"]},
     ...             )
     >>> log_pd["timestamp"] = pd.to_datetime(log_pd["timestamp"], format="ISO8601")
@@ -509,7 +510,7 @@ class EntityDaysFilter(_BaseFilter):
     +-------+-------+------+-------------------+
     |     u1|     i1|   1.0|2020-01-01 23:59:59|
     |     u2|     i2|   0.5|2020-02-01 00:00:00|
-    |     u2|     i3|   3.0|2020-02-01 00:00:00|
+    |     u2|     i3|   3.0|2020-02-01 00:00:01|
     |     u3|     i1|   1.0|2020-01-01 00:04:15|
     |     u3|     i2|   0.0|2020-01-02 00:04:14|
     |     u3|     i3|   1.0|2020-01-05 23:59:59|
@@ -524,7 +525,7 @@ class EntityDaysFilter(_BaseFilter):
     +-------+-------+------+-------------------+
     |     u1|     i1|   1.0|2020-01-01 23:59:59|
     |     u2|     i2|   0.5|2020-02-01 00:00:00|
-    |     u2|     i3|   3.0|2020-02-01 00:00:00|
+    |     u2|     i3|   3.0|2020-02-01 00:00:01|
     |     u3|     i1|   1.0|2020-01-01 00:04:15|
     |     u3|     i2|   0.0|2020-01-02 00:04:14|
     +-------+-------+------+-------------------+
@@ -539,7 +540,7 @@ class EntityDaysFilter(_BaseFilter):
     |     u1|     i1|   1.0|2020-01-01 23:59:59|
     |     u3|     i1|   1.0|2020-01-01 00:04:15|
     |     u2|     i2|   0.5|2020-02-01 00:00:00|
-    |     u2|     i3|   3.0|2020-02-01 00:00:00|
+    |     u2|     i3|   3.0|2020-02-01 00:00:01|
     +-------+-------+------+-------------------+
     <BLANKLINE>
     """
@@ -636,8 +637,8 @@ class GlobalDaysFilter(_BaseFilter):
     >>> log_pd = pd.DataFrame({"user_id": ["u1", "u2", "u2", "u3", "u3", "u3"],
     ...                     "item_id": ["i1", "i2","i3", "i1", "i2","i3"],
     ...                     "rating": [1., 0.5, 3, 1, 0, 1],
-    ...                     "timestamp": ["2020-01-01 23:59:59", "2020-02-01",
-    ...                                   "2020-02-01", "2020-01-01 00:04:15",
+    ...                     "timestamp": ["2020-01-01 23:59:59", "2020-02-01 00:00:00",
+    ...                                   "2020-02-01 00:00:01", "2020-01-01 00:04:15",
     ...                                   "2020-01-02 00:04:14", "2020-01-05 23:59:59"]},
     ...             )
     >>> log_pd["timestamp"] = pd.to_datetime(log_pd["timestamp"], format="ISO8601")
@@ -648,7 +649,7 @@ class GlobalDaysFilter(_BaseFilter):
     +-------+-------+------+-------------------+
     |     u1|     i1|   1.0|2020-01-01 23:59:59|
     |     u2|     i2|   0.5|2020-02-01 00:00:00|
-    |     u2|     i3|   3.0|2020-02-01 00:00:00|
+    |     u2|     i3|   3.0|2020-02-01 00:00:01|
     |     u3|     i1|   1.0|2020-01-01 00:04:15|
     |     u3|     i2|   0.0|2020-01-02 00:04:14|
     |     u3|     i3|   1.0|2020-01-05 23:59:59|
@@ -670,7 +671,7 @@ class GlobalDaysFilter(_BaseFilter):
     |user_id|item_id|rating|          timestamp|
     +-------+-------+------+-------------------+
     |     u2|     i2|   0.5|2020-02-01 00:00:00|
-    |     u2|     i3|   3.0|2020-02-01 00:00:00|
+    |     u2|     i3|   3.0|2020-02-01 00:00:01|
     +-------+-------+------+-------------------+
     <BLANKLINE>
     """
@@ -738,8 +739,8 @@ class TimePeriodFilter(_BaseFilter):
     >>> log_pd = pd.DataFrame({"user_id": ["u1", "u2", "u2", "u3", "u3", "u3"],
     ...                     "item_id": ["i1", "i2","i3", "i1", "i2","i3"],
     ...                     "rating": [1., 0.5, 3, 1, 0, 1],
-    ...                     "timestamp": ["2020-01-01 23:59:59", "2020-02-01",
-    ...                                   "2020-02-01", "2020-01-01 00:04:15",
+    ...                     "timestamp": ["2020-01-01 23:59:59", "2020-02-01 00:00:00",
+    ...                                   "2020-02-01 00:00:01", "2020-01-01 00:04:15",
     ...                                   "2020-01-02 00:04:14", "2020-01-05 23:59:59"]},
     ...             )
     >>> log_pd["timestamp"] = pd.to_datetime(log_pd["timestamp"], format="ISO8601")
@@ -750,7 +751,7 @@ class TimePeriodFilter(_BaseFilter):
     +-------+-------+------+-------------------+
     |     u1|     i1|   1.0|2020-01-01 23:59:59|
     |     u2|     i2|   0.5|2020-02-01 00:00:00|
-    |     u2|     i3|   3.0|2020-02-01 00:00:00|
+    |     u2|     i3|   3.0|2020-02-01 00:00:01|
     |     u3|     i1|   1.0|2020-01-01 00:04:15|
     |     u3|     i2|   0.0|2020-01-02 00:04:14|
     |     u3|     i3|   1.0|2020-01-05 23:59:59|

replay/preprocessing/history_based_fp.py

@@ -179,8 +179,8 @@ class LogStatFeaturesProcessor(EmptyFeatureProcessor):
         abnormality_aggs = [sf.mean(sf.col("abnormality")).alias("abnormality")]
 
         # Abnormality CR:
-        max_std = item_features.select(sf.max("i_std")).collect()[0][0]
-        min_std = item_features.select(sf.min("i_std")).collect()[0][0]
+        max_std = item_features.select(sf.max("i_std")).first()[0]
+        min_std = item_features.select(sf.min("i_std")).first()[0]
         if max_std - min_std != 0:
             abnormality_df = abnormality_df.withColumn(
                 "controversy",
@@ -201,15 +201,15 @@ class LogStatFeaturesProcessor(EmptyFeatureProcessor):
          :param log: input SparkDataFrame ``[user_idx, item_idx, timestamp, relevance]``
         """
         self.calc_timestamp_based = (isinstance(log.schema["timestamp"].dataType, TimestampType)) & (
-            log.select(sf.countDistinct(sf.col("timestamp"))).collect()[0][0] > 1
+            log.select(sf.countDistinct(sf.col("timestamp"))).first()[0] > 1
         )
-        self.calc_relevance_based = log.select(sf.countDistinct(sf.col("relevance"))).collect()[0][0] > 1
+        self.calc_relevance_based = log.select(sf.countDistinct(sf.col("relevance"))).first()[0] > 1
 
         user_log_features = log.groupBy("user_idx").agg(*self._create_log_aggregates(agg_col="user_idx"))
         item_log_features = log.groupBy("item_idx").agg(*self._create_log_aggregates(agg_col="item_idx"))
 
         if self.calc_timestamp_based:
-            last_date = log.select(sf.max("timestamp")).collect()[0][0]
+            last_date = log.select(sf.max("timestamp")).first()[0]
             user_log_features = self._add_ts_based(features=user_log_features, max_log_date=last_date, prefix="u")
 
             item_log_features = self._add_ts_based(features=item_log_features, max_log_date=last_date, prefix="i")

replay/preprocessing/label_encoder.py

@@ -5,6 +5,7 @@ Contains classes for encoding categorical data
     Recommended to use together with the LabelEncoder.
 ``LabelEncoder`` to apply multiple LabelEncodingRule to dataframe.
 """
+
 import abc
 import warnings
 from typing import Dict, List, Literal, Mapping, Optional, Sequence, Union

replay/scenarios/__init__.py

@@ -1,4 +1,5 @@
 """
 Scenarios are a series of actions for recommendations
 """
+
 from .fallback import Fallback

replay/splitters/last_n_splitter.py

@@ -193,7 +193,7 @@ class LastNSplitter(Splitter):
 
     def _add_time_partition_to_polars(self, interactions: PolarsDataFrame) -> PolarsDataFrame:
         res = interactions.sort(self.timestamp_column).with_columns(
-            pl.col(self.divide_column).cumcount().over(pl.col(self.divide_column)).alias("row_num")
+            pl.col(self.divide_column).cum_count().over(pl.col(self.divide_column)).alias("row_num")
         )
 
         return res

replay/splitters/time_splitter.py

@@ -193,7 +193,7 @@ class TimeSplitter(Splitter):
             )
             test_start = int(dates.count() * (1 - threshold)) + 1
             test_start = (
-                dates.filter(sf.col("_row_number_by_ts") == test_start).select(self.timestamp_column).collect()[0][0]
+                dates.filter(sf.col("_row_number_by_ts") == test_start).select(self.timestamp_column).first()[0]
             )
             res = interactions.withColumn("is_test", sf.col(self.timestamp_column) >= test_start)
         else:

replay/splitters/two_stage_splitter.py

@@ -1,8 +1,10 @@
 """
 This splitter split data by two columns.
 """
+
 from typing import Optional, Tuple
 
+import numpy as np
 import polars as pl
 
 from replay.utils import PYSPARK_AVAILABLE, DataFrameLike, PandasDataFrame, PolarsDataFrame, SparkDataFrame
@@ -124,15 +126,15 @@ class TwoStageSplitter(Splitter):
         :return: DataFrame with single column `first_divide_column`
         """
         if isinstance(interactions, SparkDataFrame):
-            all_values = interactions.select(self.first_divide_column).distinct()
+            all_values = interactions.select(self.first_divide_column).distinct().sort(self.first_divide_column)
             user_count = all_values.count()
         elif isinstance(interactions, PandasDataFrame):
             all_values = PandasDataFrame(
-                interactions[self.first_divide_column].unique(), columns=[self.first_divide_column]
+                np.sort(interactions[self.first_divide_column].unique()), columns=[self.first_divide_column]
             )
             user_count = len(all_values)
         else:
-            all_values = interactions.select(self.first_divide_column).unique()
+            all_values = interactions.select(self.first_divide_column).unique().sort(self.first_divide_column)
             user_count = len(all_values)
 
         value_error = False
@@ -152,7 +154,7 @@ class TwoStageSplitter(Splitter):
         if isinstance(interactions, SparkDataFrame):
             test_users = (
                 all_values.withColumn("_rand", sf.rand(self.seed))
-                .withColumn("_row_num", sf.row_number().over(Window.orderBy("_rand")))
+                .withColumn("_row_num", sf.row_number().over(Window.partitionBy(sf.lit(0)).orderBy("_rand")))
                 .filter(f"_row_num <= {test_user_count}")
                 .drop("_rand", "_row_num")
             )
@@ -240,10 +242,10 @@ class TwoStageSplitter(Splitter):
         res = res.fill_null(False)
 
         train = res.filter((pl.col("_frac") > self.second_divide_size) | (~pl.col("is_test"))).drop(
-            "_rand", "_row_num", "count", "_frac", "is_test"
+            "_row_num", "count", "_frac", "is_test"
         )
         test = res.filter((pl.col("_frac") <= self.second_divide_size) & pl.col("is_test")).drop(
-            "_rand", "_row_num", "count", "_frac", "is_test"
+            "_row_num", "count", "_frac", "is_test"
         )
 
         return train, test

replay/utils/distributions.py

@@ -1,4 +1,5 @@
 """Distribution calculations"""
+
 from .types import PYSPARK_AVAILABLE, DataFrameLike, PandasDataFrame
 
 if PYSPARK_AVAILABLE: