replay-rec 0.18.0rc0__py3-none-any.whl → 0.18.1__py3-none-any.whl

replay/__init__.py

@@ -1,3 +1,3 @@
 """ RecSys library """
 
-__version__ = "0.18.0.preview"
+__version__ = "0.18.1"

replay/data/dataset.py

@@ -458,13 +458,23 @@ class Dataset:
             if feature.feature_hint in [FeatureHint.ITEM_ID, FeatureHint.QUERY_ID]:
                 return nunique(self._ids_feature_map[feature.feature_hint], column)
             assert feature.feature_source
+            if feature.feature_type == FeatureType.CATEGORICAL_LIST:
+                if self.is_spark:
+                    data = (
+                        self._feature_source_map[feature.feature_source]
+                        .select(column)
+                        .withColumn(column, sf.explode(column))
+                    )
+                else:
+                    data = self._feature_source_map[feature.feature_source][[column]].explode(column)
+                return nunique(data, column)
             return nunique(self._feature_source_map[feature.feature_source], column)
 
         return callback
 
     def _set_cardinality(self, features_list: Sequence[FeatureInfo]) -> None:
         for feature in features_list:
-            if feature.feature_type == FeatureType.CATEGORICAL:
+            if feature.feature_type in [FeatureType.CATEGORICAL, FeatureType.CATEGORICAL_LIST]:
                 feature._set_cardinality_callback(self._get_cardinality(feature))
 
     def _fill_feature_schema(self, feature_schema: FeatureSchema) -> FeatureSchema:
@@ -581,6 +591,7 @@ class Dataset:
         data: DataFrameLike,
         column: str,
         source: FeatureSource,
+        feature_type: FeatureType,
         cardinality: Optional[int],
     ) -> None:
         """
@@ -593,6 +604,16 @@ class Dataset:
         Option: Keep this criterion, but suggest the user to disable the check if he understands
         that the criterion will not pass.
         """
+        if feature_type == FeatureType.CATEGORICAL_LIST:  # explode column if list
+            data = data.withColumn(column, sf.explode(column)) if self.is_spark else data[[column]].explode(column)
+
+            if self.is_pandas:
+                try:
+                    data[column] = data[column].astype(int)
+                except Exception:
+                    msg = f"IDs in {source.name}.{column} are not encoded. They are not int."
+                    raise ValueError(msg)
+
         if self.is_pandas:
             is_int = np.issubdtype(dict(data.dtypes)[column], int)
         elif self.is_spark:
@@ -632,6 +653,7 @@ class Dataset:
                     self.interactions,
                     feature.column,
                     FeatureSource.INTERACTIONS,
+                    feature.feature_type,
                     feature.cardinality,
                 )
                 if self.item_features is not None:
@@ -639,6 +661,7 @@ class Dataset:
                         self.item_features,
                         feature.column,
                         FeatureSource.ITEM_FEATURES,
+                        feature.feature_type,
                         feature.cardinality,
                     )
             elif feature.feature_hint == FeatureHint.QUERY_ID:
@@ -646,6 +669,7 @@ class Dataset:
                     self.interactions,
                     feature.column,
                     FeatureSource.INTERACTIONS,
+                    feature.feature_type,
                     feature.cardinality,
                 )
                 if self.query_features is not None:
@@ -653,6 +677,7 @@ class Dataset:
                         self.query_features,
                         feature.column,
                         FeatureSource.QUERY_FEATURES,
+                        feature.feature_type,
                         feature.cardinality,
                     )
             else:
@@ -661,6 +686,7 @@ class Dataset:
                     data,
                     feature.column,
                     feature.feature_source,
+                    feature.feature_type,
                     feature.cardinality,
                 )
 

replay/data/dataset_utils/dataset_label_encoder.py

@@ -8,8 +8,8 @@ Contains classes for encoding categorical data
 import warnings
 from typing import Dict, Iterable, Iterator, Optional, Sequence, Set, Union
 
-from replay.data import Dataset, FeatureHint, FeatureSchema, FeatureSource
-from replay.preprocessing import LabelEncoder, LabelEncodingRule
+from replay.data import Dataset, FeatureHint, FeatureSchema, FeatureSource, FeatureType
+from replay.preprocessing import LabelEncoder, LabelEncodingRule, SequenceEncodingRule
 from replay.preprocessing.label_encoder import HandleUnknownStrategies
 
 
@@ -62,7 +62,10 @@ class DatasetLabelEncoder:
 
         self._fill_features_columns(dataset.feature_schema)
         for column, feature_info in dataset.feature_schema.categorical_features.items():
-            encoding_rule = LabelEncodingRule(
+            encoding_rule_class = (
+                SequenceEncodingRule if feature_info.feature_type == FeatureType.CATEGORICAL_LIST else LabelEncodingRule
+            )
+            encoding_rule = encoding_rule_class(
                 column, handle_unknown=self._handle_unknown_rule, default_value=self._default_value_rule
             )
             if feature_info.feature_hint == FeatureHint.QUERY_ID:

replay/data/nn/schema.py

@@ -70,6 +70,8 @@ class TensorFeatureInfo:
     Information about a tensor feature.
     """
 
+    DEFAULT_EMBEDDING_DIM = 64
+
     def __init__(
         self,
         name: str,
@@ -78,6 +80,7 @@ class TensorFeatureInfo:
         feature_hint: Optional[FeatureHint] = None,
         feature_sources: Optional[List[TensorFeatureSource]] = None,
         cardinality: Optional[int] = None,
+        padding_value: int = 0,
         embedding_dim: Optional[int] = None,
         tensor_dim: Optional[int] = None,
     ) -> None:
@@ -94,6 +97,7 @@ class TensorFeatureInfo:
         :param cardinality: cardinality of categorical feature, required for ids columns,
             optional for others,
             default: ``None``.
+        :param padding_value: value to pad sequences to desired length
         :param embedding_dim: embedding dimensions of categorical feature,
             default: ``None``.
         :param tensor_dim: tensor dimensions of numerical feature,
@@ -103,24 +107,24 @@ class TensorFeatureInfo:
         self._feature_hint = feature_hint
         self._feature_sources = feature_sources
         self._is_seq = is_seq
+        self._padding_value = padding_value
 
         if not isinstance(feature_type, FeatureType):
             msg = "Unknown feature type"
             raise ValueError(msg)
         self._feature_type = feature_type
 
-        if feature_type == FeatureType.NUMERICAL and (cardinality or embedding_dim):
+        if feature_type in [FeatureType.NUMERICAL, FeatureType.NUMERICAL_LIST] and (cardinality or embedding_dim):
             msg = "Cardinality and embedding dimensions are needed only with categorical feature type."
             raise ValueError(msg)
         self._cardinality = cardinality
 
-        if feature_type == FeatureType.CATEGORICAL and tensor_dim:
+        if feature_type in [FeatureType.CATEGORICAL, FeatureType.CATEGORICAL_LIST] and tensor_dim:
             msg = "Tensor dimensions is needed only with numerical feature type."
             raise ValueError(msg)
 
-        if feature_type == FeatureType.CATEGORICAL:
-            default_embedding_dim = 64
-            self._embedding_dim = embedding_dim or default_embedding_dim
+        if feature_type in [FeatureType.CATEGORICAL, FeatureType.CATEGORICAL_LIST]:
+            self._embedding_dim = embedding_dim or self.DEFAULT_EMBEDDING_DIM
         else:
             self._tensor_dim = tensor_dim
 
@@ -176,7 +180,8 @@ class TensorFeatureInfo:
     @property
     def is_seq(self) -> bool:
         """
-        :returns: Flag that feature is sequential.
+        :returns: Flag that feature is sequential.\n
+        Sequential means that the value of the feature will be determined for each element of the user's sequence.
         """
         return self._is_seq
 
@@ -185,21 +190,35 @@ class TensorFeatureInfo:
         """
         :returns: Flag that feature is categorical.
         """
-        return self.feature_type == FeatureType.CATEGORICAL
+        return self.feature_type in [FeatureType.CATEGORICAL, FeatureType.CATEGORICAL_LIST]
 
     @property
     def is_num(self) -> bool:
         """
         :returns: Flag that feature is numerical.
         """
-        return self.feature_type == FeatureType.NUMERICAL
+        return self.feature_type in [FeatureType.NUMERICAL, FeatureType.NUMERICAL_LIST]
+
+    @property
+    def is_list(self) -> bool:
+        """
+        :returns: Flag that feature is numerical list or categorical list.
+        """
+        return self.feature_type in [FeatureType.CATEGORICAL_LIST, FeatureType.NUMERICAL_LIST]
+
+    @property
+    def padding_value(self) -> int:
+        """
+        :returns: value to pad sequences to desired length.
+        """
+        return self._padding_value
 
     @property
     def cardinality(self) -> Optional[int]:
         """
         :returns: Cardinality of the feature.
         """
-        if self.feature_type != FeatureType.CATEGORICAL:
+        if not self.is_cat:
             msg = f"Can not get cardinality because feature type of {self.name} column is not categorical."
             raise RuntimeError(msg)
         return self._cardinality
@@ -212,7 +231,7 @@ class TensorFeatureInfo:
         """
         :returns: Dimensions of the numerical feature.
         """
-        if self.feature_type != FeatureType.NUMERICAL:
+        if not self.is_num:
             msg = f"Can not get tensor dimensions because feature type of {self.name} feature is not numerical."
             raise RuntimeError(msg)
         return self._tensor_dim
@@ -225,7 +244,7 @@ class TensorFeatureInfo:
         """
         :returns: Embedding dimensions of the feature.
         """
-        if self.feature_type != FeatureType.CATEGORICAL:
+        if not self.is_cat:
             msg = f"Can not get embedding dimensions because feature type of {self.name} feature is not categorical."
             raise RuntimeError(msg)
         return self._embedding_dim
@@ -317,14 +336,16 @@ class TensorSchema(Mapping[str, TensorFeatureInfo]):
         """
         :returns: Sequence of categorical features in a schema.
         """
-        return self.filter(feature_type=FeatureType.CATEGORICAL)
+        return self.filter(feature_type=FeatureType.CATEGORICAL) + self.filter(
+            feature_type=FeatureType.CATEGORICAL_LIST
+        )
 
     @property
     def numerical_features(self) -> "TensorSchema":
         """
         :returns: Sequence of numerical features in a schema.
         """
-        return self.filter(feature_type=FeatureType.NUMERICAL)
+        return self.filter(feature_type=FeatureType.NUMERICAL) + self.filter(feature_type=FeatureType.NUMERICAL_LIST)
 
     @property
     def query_id_features(self) -> "TensorSchema":
@@ -423,9 +444,9 @@ class TensorSchema(Mapping[str, TensorFeatureInfo]):
                     if feature.feature_sources
                     else None
                 ),
-                "cardinality": feature.cardinality if feature.feature_type == FeatureType.CATEGORICAL else None,
-                "embedding_dim": feature.embedding_dim if feature.feature_type == FeatureType.CATEGORICAL else None,
-                "tensor_dim": feature.tensor_dim if feature.feature_type == FeatureType.NUMERICAL else None,
+                "cardinality": feature.cardinality if feature.is_cat else None,
+                "embedding_dim": feature.embedding_dim if feature.is_cat else None,
+                "tensor_dim": feature.tensor_dim if feature.is_num else None,
             }
             for feature in self.all_features
         ]