recnexteval 0.1.0__py3-none-any.whl

recnexteval/datasets/datasets/yelp.py

@@ -0,0 +1,103 @@
+import logging
+from typing import ClassVar, NoReturn
+
+import numpy as np
+import pandas as pd
+
+from ..config.yelp import YelpDatasetConfig
+from .base import Dataset
+
+
+logger = logging.getLogger(__name__)
+
+
+class YelpDataset(Dataset):
+    """Yelp dataset
+
+    The Yelp dataset contains user reviews of businesses. The main columns that
+    will be used are:
+
+    - user_id: The user identifier
+    - business_id: The business identifier
+    - stars: The rating given by the user to the business
+    - date: The date of the review
+
+    The dataset can be downloaded from https://www.yelp.com/dataset/download.
+    The dataset is in a zip file, there are online codes that will aid you in
+    converting the json file to a csv file for usage. Note that for the purposes
+    of this class, it is assumed that the dataset has been converted to a csv file
+    and is named `yelp_academic_dataset_review.csv`.
+
+    Reference is made to the following code from the official repo from
+    Yelp: https://github.com/Yelp/dataset-examples/blob/master/json_to_csv_converter.py
+
+    you can use the following command to convert the json file to a csv file:
+
+    .. code-block:: shell
+        python json_to_csv_converter.py yelp_academic_dataset_review.json
+
+    """
+
+    IS_BASE: bool = False
+    config: ClassVar[YelpDatasetConfig] = YelpDatasetConfig()
+
+    def _download_dataset(self) -> NoReturn:
+        raise ValueError(
+            "Yelp dataset has not been downloaded. Please head over"
+            f"to {self.config.dataset_url} to download the dataset."
+            "As there is a license agreement, we cannot download it for you."
+            "Place the unzip dataset under the data directory when done."
+            f"Expected filename: {self.config.default_filename}"
+        )
+
+    def _load_dataframe(self) -> pd.DataFrame:
+        """Load the raw dataset from file, and return it as a pandas DataFrame.
+
+        Transform the dataset downloaded to have integer user and item ids. This
+        will be needed for representation in the interaction matrix.
+
+        Returns:
+            The interaction data as a DataFrame with a row per interaction.
+        """
+        self.fetch_dataset()
+
+        df = pd.read_csv(
+            self.file_path,
+            dtype={
+                self.config.item_ix: str,
+                self.config.user_ix: str,
+                self.config.rating_ix: np.float32,
+                self.config.timestamp_ix: str,
+            },
+            usecols=[
+                self.config.item_ix,
+                self.config.user_ix,
+                self.config.rating_ix,
+                self.config.timestamp_ix,
+            ],
+            parse_dates=[self.config.timestamp_ix],
+            date_format="%Y-%m-%d %H:%M:%S",
+            header=0,
+            sep=",",
+            encoding="utf-8",
+        )
+
+        # remove the byte literal char from the string columns
+        str_df = df.select_dtypes(["object"])
+        str_df = str_df.stack().str[2:-1].unstack()
+        for col in str_df:
+            df[col] = str_df[col]
+
+        # convert the timestamp to epoch time
+        df[self.config.timestamp_ix] = pd.to_datetime(
+            df[self.config.timestamp_ix], format="%Y-%m-%d %H:%M:%S", errors="coerce"
+        )
+        df.dropna(inplace=True)
+        df[self.config.timestamp_ix] = df[self.config.timestamp_ix].astype(np.int64) // 10**9
+
+        return df
+
+    def _fetch_dataset_metadata(
+        self, user_id_mapping: pd.DataFrame, item_id_mapping: pd.DataFrame
+    ) -> None:
+        pass

recnexteval/datasets/metadata/__init__.py

@@ -0,0 +1,58 @@
+"""Metadata module for dataset information.
+
+This module allows users to include metadata information corresponding to datasets.
+Metadata classes are built on top of the `Metadata` base class, allowing for easy
+extension and customization.
+
+## Important Notes
+
+User and item IDs in the metadata module are mapped according to RecNextEval's
+internal mapping, not the original IDs. Developers should not load metadata from
+source separately. Instead, implement the metadata class and load metadata while
+loading the dataset.
+
+## Available Metadata
+
+- `Metadata`: Abstract base class for metadata implementations
+- `MovieLens100kUserMetadata`: User metadata from MovieLens 100K dataset
+- `MovieLens100kItemMetadata`: Item metadata from MovieLens 100K dataset
+- `AmazonBookItemMetadata`: Item metadata from Amazon Books dataset
+- `AmazonMovieItemMetadata`: Item metadata from Amazon Movies dataset
+- `AmazonMusicItemMetadata`: Item metadata from Amazon Music dataset
+- `LastFMUserMetadata`: User metadata from Last.FM dataset
+- `LastFMItemMetadata`: Item metadata from Last.FM dataset
+- `LastFMTagMetadata`: Tag metadata from Last.FM dataset
+
+## Example
+
+Load metadata from the MovieLens 100K dataset:
+
+```python
+from recnexteval.datasets.movielens import MovieLens100K
+
+dataset = MovieLens100K(fetch_dataset=True)
+data = dataset.load()
+```
+"""
+
+from .amazon import (
+    AmazonBookItemMetadata,
+    AmazonMovieItemMetadata,
+    AmazonMusicItemMetadata,
+)
+from .base import Metadata
+from .lastfm import LastFMItemMetadata, LastFMTagMetadata, LastFMUserMetadata
+from .movielens import MovieLens100kItemMetadata, MovieLens100kUserMetadata
+
+
+__all__ = [
+    "Metadata",
+    "MovieLens100kUserMetadata",
+    "MovieLens100kItemMetadata",
+    "AmazonBookItemMetadata",
+    "AmazonMovieItemMetadata",
+    "AmazonMusicItemMetadata",
+    "LastFMUserMetadata",
+    "LastFMItemMetadata",
+    "LastFMTagMetadata",
+]

recnexteval/datasets/metadata/amazon.py

@@ -0,0 +1,68 @@
+import logging
+import os
+from typing import ClassVar
+
+import pandas as pd
+
+from recnexteval.datasets.config import (
+    AmazonBooksItemMetadataConfig,
+    AmazonDigitalMusicItemMetadataConfig,
+    AmazonItemMetadataConfig,
+    AmazonMoviesAndTVItemMetadataConfig,
+    AmazonSubscriptionBoxesItemMetadataConfig,
+)
+from .base import Metadata
+
+
+logger = logging.getLogger(__name__)
+
+
+class AmazonItemMetadata(Metadata):
+    config: ClassVar[AmazonItemMetadataConfig] = AmazonItemMetadataConfig()
+
+    def __init__(self, item_id_mapping: pd.DataFrame) -> None:
+        super().__init__()
+        self.item_id_mapping = item_id_mapping
+
+    def _load_dataframe(self) -> pd.DataFrame:
+        self.fetch_dataset()
+        df = pd.read_json(
+            self.file_path,  # Ensure file_path contains the JSONL file path
+            dtype=self.config.dtype_dict,
+            lines=True,  # Required for JSONL format
+        )
+
+        item_id_to_iid = dict(zip(self.item_id_mapping[self.config.item_ix], self.item_id_mapping["iid"]))
+
+        # Map config.item_ix in metadata_df using the optimized function
+        df[self.config.item_ix] = df[self.config.item_ix].map(lambda x: item_id_to_iid.get(x, x))
+
+        return df
+
+    def _download_dataset(self) -> None:
+        """Downloads the metadata for the dataset.
+
+        Downloads the zipfile, and extracts the ratings file to `self.file_path`
+        """
+        if not self.config.dataset_url:
+            raise ValueError(f"{self.name} does not have URL specified.")
+
+        self._fetch_remote(
+            self.config.dataset_url, os.path.join(self.base_path, f"{self.config.remote_filename}")
+        )
+
+
+class AmazonMusicItemMetadata(AmazonItemMetadata):
+    config: ClassVar[AmazonDigitalMusicItemMetadataConfig] = AmazonDigitalMusicItemMetadataConfig()
+
+
+class AmazonMovieItemMetadata(AmazonItemMetadata):
+    config: ClassVar[AmazonMoviesAndTVItemMetadataConfig] = AmazonMoviesAndTVItemMetadataConfig()
+
+
+class AmazonSubscriptionBoxesItemMetadata(AmazonItemMetadata):
+    config: ClassVar[AmazonSubscriptionBoxesItemMetadataConfig] = AmazonSubscriptionBoxesItemMetadataConfig()
+
+
+class AmazonBookItemMetadata(AmazonItemMetadata):
+    config: ClassVar[AmazonBooksItemMetadataConfig] = AmazonBooksItemMetadataConfig()

recnexteval/datasets/metadata/base.py

@@ -0,0 +1,38 @@
+import logging
+from typing import ClassVar, Optional
+
+import pandas as pd
+
+from recnexteval.datasets.base import DataFetcher
+from recnexteval.datasets.config import MetadataConfig
+from recnexteval.utils.path import safe_dir
+
+
+logger = logging.getLogger(__name__)
+
+
+class Metadata(DataFetcher):
+    config: ClassVar[MetadataConfig] = MetadataConfig()
+
+    def __init__(
+        self,
+        filename: Optional[str] = None,
+        base_path: Optional[str] = None,
+    ) -> None:
+        self.base_path = base_path if base_path else self.config.default_base_path
+        logger.debug(f"{self.name} being initialized with '{self.base_path}' as the base path.")
+
+        self.filename = filename if filename else self.config.default_filename
+        if not self.filename:
+            raise ValueError("No filename specified, and no default known.")
+
+        safe_dir(self.base_path)
+        logger.debug(f"{self.name} is initialized.")
+
+    def load(self) -> pd.DataFrame:
+        """Load the metadata from file and return it as a DataFrame.
+
+        Returns:
+            DataFrame containing the metadata.
+        """
+        return self._load_dataframe()

recnexteval/datasets/metadata/lastfm.py

@@ -0,0 +1,110 @@
+import logging
+import os
+import zipfile
+from typing import ClassVar
+
+import pandas as pd
+
+from recnexteval.datasets.config import (
+    LastFMItemMetadataConfig,
+    LastFMTagMetadataConfig,
+    LastFMUserMetadataConfig,
+)
+from .base import Metadata
+
+
+logger = logging.getLogger(__name__)
+
+
+class LastFMMetadata(Metadata):
+    config: ClassVar[LastFMUserMetadataConfig] = LastFMUserMetadataConfig()  # type: ignore
+
+    def _download_dataset(self) -> None:
+        """Downloads the metadata for the dataset.
+
+        Downloads the zipfile, and extracts the ratings file to `self.file_path`
+        """
+        # Download the zip into the data directory
+        self._fetch_remote(
+            f"{self.config.dataset_url}/{self.config.remote_filename}.zip",
+            os.path.join(self.base_path, f"{self.config.remote_filename}.zip"),
+        )
+
+        # Extract the interaction file which we will use
+        with zipfile.ZipFile(
+            os.path.join(self.base_path, f"{self.config.remote_filename}.zip"), "r"
+        ) as zip_ref:
+            zip_ref.extract(f"{self.config.remote_filename}", self.base_path)
+
+
+class LastFMUserMetadata(Metadata):
+    config: ClassVar[LastFMUserMetadataConfig] = LastFMUserMetadataConfig()  # type: ignore
+
+    def __init__(self, user_id_mapping: pd.DataFrame) -> None:
+        super().__init__()
+        self.user_id_mapping = user_id_mapping
+
+    def _load_dataframe(self) -> pd.DataFrame:
+        self.fetch_dataset()
+        df = pd.read_csv(
+            self.file_path,
+            sep=self.config.sep,
+            names=self.config.column_names,
+            converters={
+                self.config.user_ix: self._map_user_id,
+                self.config.friend_ix: self._map_user_id,
+            },
+            header=0,
+        )
+        return df
+
+    def _map_user_id(self, user_id):
+        user_id_to_uid = dict(
+            zip(self.user_id_mapping[self.config.user_ix], self.user_id_mapping["uid"])
+        )
+        return user_id_to_uid.get(int(user_id), user_id)
+
+
+class LastFMItemMetadata(Metadata):
+    config: ClassVar[LastFMItemMetadataConfig] = LastFMItemMetadataConfig()  # type: ignore
+
+    def __init__(self, item_id_mapping: pd.DataFrame) -> None:
+        super().__init__()
+        self.item_id_mapping = item_id_mapping
+
+    def _load_dataframe(self) -> pd.DataFrame:
+        self.fetch_dataset()
+        df = pd.read_csv(
+            self.file_path,
+            dtype=self.config.dtype_dict,
+            sep=self.config.sep,
+            names=self.config.column_names,
+            converters={
+                self.config.item_ix: self._map_item_id,
+            },
+            header=0,
+        )
+        return df
+
+    def _map_item_id(self, item_id):
+        item_id_to_iid = dict(zip(self.item_id_mapping["artistID"], self.item_id_mapping["iid"]))
+        return item_id_to_iid.get(int(item_id), item_id)
+
+
+class LastFMTagMetadata(Metadata):
+    config: ClassVar[LastFMTagMetadataConfig] = LastFMTagMetadataConfig()  # type: ignore
+
+    def __init__(self) -> None:
+        super().__init__()
+
+    def _load_dataframe(self) -> pd.DataFrame:
+        self.fetch_dataset()
+        df = pd.read_csv(
+            self.file_path,
+            dtype=self.config.dtype_dict,
+            sep=self.config.sep,
+            names=self.config.column_names,
+            encoding="ISO-8859-1",
+            header=0,
+        )
+        return df

recnexteval/datasets/metadata/movielens.py

@@ -0,0 +1,87 @@
+import os
+import zipfile
+from typing import ClassVar
+
+import pandas as pd
+
+from recnexteval.datasets.config import (
+    MovieLens100kItemMetadataConfig,
+    MovieLens100kUserMetadataConfig,
+)
+from .base import Metadata
+
+
+class MovieLens100kMetadata(Metadata):
+    def _download_dataset(self) -> None:
+        # Download the zip into the data directory
+        self._fetch_remote(
+            f"{self.config.dataset_url}/{self.config.remote_zipname}.zip",
+            os.path.join(self.base_path, f"{self.config.remote_zipname}.zip"),
+        )
+
+        # Extract the ratings file which we will use
+        with zipfile.ZipFile(
+            os.path.join(self.base_path, f"{self.config.remote_zipname}.zip"), "r"
+        ) as zip_ref:
+            zip_ref.extract(
+                f"{self.config.remote_zipname}/{self.config.remote_filename}", self.base_path
+            )
+
+        # Rename the ratings file to the specified filename
+        os.rename(
+            os.path.join(
+                self.base_path, f"{self.config.remote_zipname}/{self.config.remote_filename}"
+            ),
+            self.file_path,
+        )
+
+
+class MovieLens100kUserMetadata(MovieLens100kMetadata):
+    config: ClassVar[MovieLens100kUserMetadataConfig] = MovieLens100kUserMetadataConfig()  # type: ignore
+
+    def __init__(self, user_id_mapping: pd.DataFrame) -> None:
+        super().__init__()
+        self.user_id_mapping = user_id_mapping
+
+    def _load_dataframe(self) -> pd.DataFrame:
+        self.fetch_dataset()
+        df = pd.read_table(
+            self.file_path,
+            dtype=self.config.dtype_dict,
+            sep=self.config.sep,
+            names=self.config.column_names,
+            converters={self.config.user_ix: self._map_user_id},
+        )
+        return df
+
+    def _map_user_id(self, user_id):
+        user_id_to_uid = dict(
+            zip(self.user_id_mapping[self.config.user_ix], self.user_id_mapping["uid"])
+        )
+        return user_id_to_uid.get(int(user_id), user_id)
+
+
+class MovieLens100kItemMetadata(MovieLens100kMetadata):
+    config: ClassVar[MovieLens100kItemMetadataConfig] = MovieLens100kItemMetadataConfig()  # type: ignore
+
+    def __init__(self, item_id_mapping: pd.DataFrame) -> None:
+        super().__init__()
+        self.item_id_mapping = item_id_mapping
+
+    def _load_dataframe(self) -> pd.DataFrame:
+        self.fetch_dataset()
+        df = pd.read_table(
+            self.file_path,
+            dtype=self.config.dtype_dict,
+            sep=self.config.sep,
+            names=self.config.column_names,
+            converters={self.config.item_ix: self._map_item_id},
+            encoding=self.config.encoding,
+        )
+        return df
+
+    def _map_item_id(self, item_id):
+        item_id_to_iid = dict(
+            zip(self.item_id_mapping[self.config.item_ix], self.item_id_mapping["iid"])
+        )
+        return item_id_to_iid.get(int(item_id), item_id)

recnexteval/evaluators/__init__.py

@@ -0,0 +1,189 @@
+"""Evaluator module for algorithm evaluation in streaming settings.
+
+This module provides classes and utilities for evaluating recommendation algorithms
+in streaming environments. It supports both batch pipeline evaluation and interactive
+streaming evaluation with comprehensive metric computation and result analysis.
+
+Evaluator Builder
+=================
+
+The evaluator module contains builder classes for constructing evaluator objects.
+The builders provide a fluent API for configuring evaluators with proper validation
+and error checking.
+
+For detailed information about the builder classes and usage examples,
+see the `recnexteval.evaluators.builder` module.
+
+Available Builders:
+- `Builder`: Abstract base class for all builder implementations
+- `EvaluatorPipelineBuilder`: Builder for pipeline evaluators
+- `EvaluatorStreamerBuilder`: Builder for streaming evaluators
+
+Evaluator Classes
+=================
+
+The core evaluator classes handle the evaluation of recommendation algorithms
+on streaming data. The evaluators manage data splitting, algorithm training,
+prediction generation, and metric computation.
+
+EvaluatorPipeline
+-----------------
+
+For batch evaluation of multiple algorithms on static or sliding window settings.
+This evaluator runs algorithms through a complete pipeline including training,
+prediction, and evaluation phases.
+
+EvaluatorStreamer
+-----------------
+
+For interactive streaming evaluation where algorithms can be registered and
+evaluated in real-time as data streams in. This allows for more flexible
+evaluation scenarios where algorithms can request data and submit predictions
+asynchronously.
+
+Both evaluators inherit from `EvaluatorBase` which provides common functionality
+for metric computation, data masking, and result aggregation.
+
+Key Features
+------------
+
+- **Multi-algorithm evaluation**: Evaluate multiple algorithms simultaneously
+- **Streaming support**: Handle temporal data streams with sliding windows
+- **Metric aggregation**: Compute metrics at different levels (user, window, macro, micro)
+- **Data masking**: Properly handle unknown users and items during evaluation
+- **Result analysis**: Rich DataFrame outputs for metric analysis and comparison
+
+Basic Usage
+-----------
+
+Pipeline Evaluation:
+
+```python
+from recnexteval.evaluators import EvaluatorPipeline
+from recnexteval.evaluators.builder import EvaluatorPipelineBuilder
+from recnexteval.settings import Setting
+from recnexteval.datasets import AmazonMusicDataset
+
+# Load data and create setting
+dataset = AmazonMusicDataset()
+data = dataset.load()
+setting = Setting(data=data, top_K=10)
+setting.split()
+
+# Build evaluator
+builder = EvaluatorPipelineBuilder()
+builder.add_setting(setting)
+builder.set_metric_K(10)
+builder.add_metric("PrecisionK")
+builder.add_algorithm("MostPopular")
+evaluator = builder.build()
+
+# Run evaluation
+evaluator.run()
+
+# Get results
+results = evaluator.metric_results(level="macro")
+```
+
+Streaming Evaluation:
+
+```python
+from recnexteval.evaluators import EvaluatorStreamer
+from recnexteval.evaluators.builder import EvaluatorStreamerBuilder
+from recnexteval.algorithms import MostPopular
+
+# Build streaming evaluator
+builder = EvaluatorStreamerBuilder()
+builder.add_setting(setting)
+builder.set_metric_K(10)
+builder.add_metric("HitK")
+evaluator = builder.build()
+
+# Start streaming
+evaluator.start_stream()
+
+# Register algorithm
+algo_id = evaluator.register_algorithm(MostPopular())
+
+# Stream evaluation loop
+while True:
+    try:
+        # Get training data
+        training_data = evaluator.get_training_data(algo_id)
+
+        # Get unlabeled data
+        unlabeled_data = evaluator.get_unlabeled_data(algo_id)
+
+        # Algorithm makes predictions
+        predictions = algorithm.predict(unlabeled_data)
+
+        # Submit predictions
+        evaluator.submit_prediction(algo_id, predictions)
+
+    except EOWSettingError:
+        break
+```
+
+Metric Levels
+-------------
+
+Evaluators support computing metrics at different aggregation levels:
+
+- **User level**: Metrics computed per user across all timestamps
+- **Window level**: Metrics computed per time window, averaging user scores within each window
+- **Macro level**: Overall metrics averaging across all windows equally
+- **Micro level**: Overall metrics averaging across all user-timestamp combinations equally
+
+Available Evaluator Classes:
+- `EvaluatorBase`: Base class providing common evaluation functionality
+- `EvaluatorPipeline`: Pipeline-based batch evaluator
+- `EvaluatorStreamer`: Interactive streaming evaluator
+
+Accumulator
+==========
+
+The `MetricAccumulator` class accumulates and stores metric results during
+evaluation. The accumulator maintains a collection of metric objects organized
+by algorithm name and provides methods for retrieving results in various formats.
+
+Features:
+- Storing metrics for multiple algorithms
+- Computing aggregated results at different levels
+- Exporting results to pandas DataFrames
+- Filtering results by algorithm, timestamp, or metric type
+
+Utility Classes
+===============
+
+Utility classes that support the evaluation process:
+
+- `MetricLevelEnum`: Enumeration for different metric aggregation levels
+- `UserItemBaseStatus`: Tracks known and unknown users/items during evaluation
+
+These utilities handle the complex state management required for streaming
+evaluation scenarios.
+"""
+
+from .accumulator import MetricAccumulator
+from .base import EvaluatorBase
+from .builder import (
+    Builder,
+    EvaluatorPipelineBuilder,
+    EvaluatorStreamerBuilder,
+)
+from .evaluator_pipeline import EvaluatorPipeline
+from .evaluator_stream import EvaluatorStreamer
+from .util import MetricLevelEnum, UserItemBaseStatus
+
+
+__all__ = [
+    "Builder",
+    "EvaluatorPipelineBuilder",
+    "EvaluatorStreamerBuilder",
+    "EvaluatorBase",
+    "EvaluatorPipeline",
+    "EvaluatorStreamer",
+    "MetricAccumulator",
+    "MetricLevelEnum",
+    "UserItemBaseStatus",
+]