recnexteval 0.1.0__py3-none-any.whl

recnexteval/algorithms/utils.py

@@ -0,0 +1,51 @@
+import numpy as np
+from scipy.sparse import csr_matrix
+
+
+def get_top_K_ranks(X: csr_matrix, K: None | int = None) -> csr_matrix:
+    """Returns a matrix of ranks assigned to the largest K values in X.
+
+    Selects K largest values for every row in X and assigns a rank to each.
+
+    :param X: Matrix from which we will select K values in every row.
+    :type X: csr_matrix
+    :param K: Amount of values to select.
+    :type K: int, optional
+    :return: Matrix with K values per row.
+    :rtype: csr_matrix
+    """
+    U, I, V = [], [], []
+    for row_ix, (le, ri) in enumerate(zip(X.indptr[:-1], X.indptr[1:])):
+        K_row_pick = min(K, ri - le) if K is not None else ri - le
+
+        if K_row_pick != 0:
+            top_k_row = X.indices[le + np.argpartition(X.data[le:ri], list(range(-K_row_pick, 0)))[-K_row_pick:]]
+
+            for rank, col_ix in enumerate(reversed(top_k_row)):
+                U.append(row_ix)
+                I.append(col_ix)
+                V.append(rank + 1)
+    # data, (row, col) = (V, (U, I)
+    X_top_K = csr_matrix((V, (U, I)), shape=X.shape)
+
+    return X_top_K
+
+
+def get_top_K_values(X: csr_matrix, K: None | int = None) -> csr_matrix:
+    """Returns a matrix of only the K largest values for every row in X.
+
+    Selects the top-K items for every user (which is equal to the K nearest neighbours.)
+    In case of a tie for the last position, the item with the largest index of the tied items is used.
+
+    :param X: Matrix from which we will select K values in every row.
+    :type X: csr_matrix
+    :param K: Amount of values to select.
+    :type K: int, optional
+    :return: Matrix with K values per row.
+    :rtype: csr_matrix
+    """
+    top_K_ranks = get_top_K_ranks(X, K)
+    # Convert the position into binary values (1 if in top K, 0 otherwise)
+    top_K_ranks[top_K_ranks > 0] = 1
+    # elementwise multiplication with orignal matrix to get values
+    return top_K_ranks.multiply(X)

recnexteval/datasets/__init__.py

@@ -0,0 +1,109 @@
+"""Dataset module for public datasets in streaming experiments.
+
+This module provides easy access to publicly available datasets for use in streaming
+experiments. Dataset classes are built on top of the `Dataset` base class, allowing
+for easy extension and customization.
+
+## Dataset Overview
+
+Multiple public datasets are available from various sources. Additionally, a
+lightweight test dataset is provided for testing algorithm functionality.
+
+### Data Chunking Note
+
+The MovieLens 100K dataset is available but not chunked into "blocks". Setting
+a global timeline to split the data could potentially cause a chunk of data to
+be lost. Other publicly available datasets are recommended.
+
+## Available Datasets
+
+- `AmazonBookDataset`: Amazon Books reviews
+- `AmazonMovieDataset`: Amazon Movies reviews
+- `AmazonMusicDataset`: Amazon Music reviews
+- `AmazonSubscriptionBoxesDataset`: Amazon Subscription Boxes reviews
+- `LastFMDataset`: Last.FM music listening history
+- `MovieLens100K`: MovieLens 100K rating dataset
+- `YelpDataset`: Yelp business reviews
+- `TestDataset`: Lightweight dataset for testing algorithms
+
+## Loading Datasets
+
+Basic loading:
+
+```python
+from recnexteval.datasets import AmazonMusicDataset
+
+dataset = AmazonMusicDataset()
+data = dataset.load()
+```
+
+If the file does not exist, it will be downloaded and written. Subsequent loads
+will retrieve the file from disk without downloading again.
+
+### Using Default Filters
+
+```python
+from recnexteval.datasets import AmazonMusicDataset
+
+dataset = AmazonMusicDataset(use_default_filters=True)
+data = dataset.load(apply_filters=False)
+```
+
+Each dataset can be loaded with default filters applied. Default filters ensure
+that user and item IDs increment in the order of time. **This is the recommended
+loading approach.**
+
+## Extending the Framework
+
+To add custom datasets, inherit from `Dataset` and implement all abstract methods.
+Refer to the base class documentation for implementation details.
+
+## Related Modules
+
+- `recnexteval.preprocessing`: Data preprocessing and filtering utilities
+"""
+
+from .datasets import (
+    AmazonBookDataset,
+    AmazonMovieDataset,
+    AmazonMusicDataset,
+    AmazonSubscriptionBoxesDataset,
+    Dataset,
+    LastFMDataset,
+    MovieLens100K,
+    TestDataset,
+    YelpDataset,
+)
+from .metadata import (
+    AmazonBookItemMetadata,
+    AmazonMovieItemMetadata,
+    AmazonMusicItemMetadata,
+    LastFMItemMetadata,
+    LastFMTagMetadata,
+    LastFMUserMetadata,
+    Metadata,
+    MovieLens100kItemMetadata,
+    MovieLens100kUserMetadata,
+)
+
+
+__all__ = [
+    "AmazonBookDataset",
+    "AmazonMovieDataset",
+    "AmazonMusicDataset",
+    "AmazonSubscriptionBoxesDataset",
+    "LastFMDataset",
+    "MovieLens100K",
+    "YelpDataset",
+    "TestDataset",
+    "Dataset",
+    "Metadata",
+    "MovieLens100kUserMetadata",
+    "MovieLens100kItemMetadata",
+    "AmazonBookItemMetadata",
+    "AmazonMovieItemMetadata",
+    "AmazonMusicItemMetadata",
+    "LastFMUserMetadata",
+    "LastFMItemMetadata",
+    "LastFMTagMetadata",
+]

recnexteval/datasets/base.py

@@ -0,0 +1,316 @@
+import logging
+import os
+import time
+from abc import abstractmethod
+from typing import ClassVar
+
+import httpx
+import pandas as pd
+
+from ..matrix import InteractionMatrix
+from ..models import BaseModel
+from ..preprocessing.filter import Filter, MinItemsPerUser, MinUsersPerItem
+from ..preprocessing.preprocessor import DataFramePreprocessor
+from ..utils.path import safe_dir
+from .config import DatasetConfig
+
+
+logger = logging.getLogger(__name__)
+
+
+class DataFetcher(BaseModel):
+    """Represents a abstract class to be used by Dataset or Metadata subclass.
+    """
+    config: ClassVar[DatasetConfig] = DatasetConfig()
+    """Configuration for the dataset."""
+
+    @property
+    def file_path(self) -> str:
+        """File path of the dataset."""
+        return os.path.join(self.config.default_base_path, self.config.default_filename)
+
+    @property
+    def processed_cache_path(self) -> str:
+        """Path for cached processed data."""
+        return os.path.join(
+            self.config.default_base_path, f"{self.config.default_filename}.processed.parquet"
+        )
+
+    def fetch_dataset(self) -> None:
+        """Check if dataset is present, if not download"""
+        if os.path.exists(self.file_path):
+            logger.debug("Data file is in memory and in dir specified.")
+            return
+        logger.debug(f"{self.name} dataset not found in {self.file_path}.")
+        self._download_dataset()
+
+    def fetch_dataset_force(self) -> None:
+        """Force re-download of the dataset."""
+        logger.debug(f"{self.name} force re-download of dataset.")
+        self._download_dataset()
+
+    def _fetch_remote(self, url: str, filename: str) -> str:
+        """Fetch data from remote url and save locally (synchronous fallback).
+
+        This function keeps the previous synchronous behaviour but uses
+        `httpx.Client` to stream the response and write to disk. If you
+        want async behaviour, use :meth:`_fetch_remote_async` instead.
+
+        Args:
+            url: url to fetch data from
+            filename: Path to save file to
+
+        Returns:
+            The filename where data was saved
+        """
+        logger.debug(f"{self.name} will fetch dataset from remote url at {url}.")
+
+        with httpx.Client(timeout=httpx.Timeout(60.0)) as client, client.stream("GET", url) as resp:
+            resp.raise_for_status()
+            with open(filename, "wb") as fd:
+                for chunk in resp.iter_bytes():
+                    if chunk:
+                        fd.write(chunk)
+
+        return filename
+
+    async def _fetch_remote_async(self, url: str, filename: str) -> str:
+        """Asynchronously fetch data from a remote URL and save locally.
+
+        Uses `httpx.AsyncClient` and streams the response to disk. Callers
+        running inside an event loop should use this coroutine instead of
+        the synchronous `_fetch_remote`.
+
+        Args:
+            url: url to fetch data from
+            filename: Path to save file to
+
+        Returns:
+            The filename where data was saved
+        """
+        logger.debug(f"{self.name} will asynchronously fetch dataset from {url}.")
+
+        async with (
+            httpx.AsyncClient(timeout=httpx.Timeout(60.0)) as client,
+            client.stream("GET", url) as resp,
+        ):
+            resp.raise_for_status()
+            # Write bytes as they arrive
+            with open(filename, "wb") as fd:
+                async for chunk in resp.aiter_bytes():
+                    if chunk:
+                        fd.write(chunk)
+
+        return filename
+
+    def _cache_processed_dataframe(self, df: pd.DataFrame) -> None:
+        """Cache the processed DataFrame to disk.
+
+        :param df: DataFrame to cache
+        :type df: pd.DataFrame
+        """
+        logger.debug(f"Caching processed DataFrame to {self.processed_cache_path}")
+        df.to_parquet(self.processed_cache_path)
+        logger.debug("Processed DataFrame cached successfully.")
+
+    @abstractmethod
+    def _load_dataframe(self) -> pd.DataFrame:
+        """Load the raw dataset from file, and return it as a pandas DataFrame.
+
+        Warning:
+            This does not apply any preprocessing, and returns the raw dataset.
+
+        Returns:
+            Interaction with minimal columns of {user, item, timestamp}.
+        """
+        raise NotImplementedError("Needs to be implemented")
+
+    @abstractmethod
+    def _download_dataset(self) -> None:
+        """Downloads the dataset.
+
+        Downloads the csv file from the dataset URL and saves it to the file path.
+        """
+        raise NotImplementedError("Needs to be implemented")
+
+
+class Dataset(DataFetcher):
+    """Represents a collaborative filtering dataset.
+
+    Dataset must minimally contain user, item and timestamp columns for the
+    other modules to work.
+
+    Assumption
+    ===========
+    User/item ID increments in the order of time. This is an assumption that will
+    be made for the purposes of splitting the dataset and eventually passing
+    the dataset to the model. The ID incrementing in the order of time allows us
+    to set the shape of the currently known user and item matrix allowing easier
+    manipulation of the data by the evaluator.
+
+    :param filename: Name of the file, if no name is provided the dataset default will be used if known.
+        If the dataset does not have a default filename, a ValueError will be raised.
+    :type filename: str, optional
+    :param base_path: The base_path to the data directory.
+        Defaults to `data`
+    :type base_path: str, optional
+    :param use_default_filters: If True, the default filters will be applied to the dataset.
+        Defaults to False.
+    :type use_default_filters: bool, optional
+    """
+
+    config: ClassVar[DatasetConfig] = DatasetConfig()
+    """Configuration for the dataset."""
+
+    def __init__(
+        self,
+        use_default_filters: bool = False,  # noqa: FBT001, FBT002
+        fetch_metadata: bool = False,  # noqa: FBT001, FBT002
+    ) -> None:
+        if not self.config.user_ix or not self.config.item_ix or not self.config.timestamp_ix:
+            raise AttributeError("user_ix, item_ix or timestamp_ix not set in config.")
+
+        logger.debug(
+            f"{self.name} being initialized with '{self.config.default_base_path}' as the base path."
+        )
+
+        if not self.config.default_filename:
+            raise ValueError("No filename specified, and no default known.")
+
+        self.fetch_metadata = fetch_metadata
+        self.preprocessor = DataFramePreprocessor(
+            self.config.item_ix, self.config.user_ix, self.config.timestamp_ix
+        )
+
+        if use_default_filters:
+            for f in self._default_filters:
+                self.add_filter(f)
+
+        safe_dir(self.config.default_base_path)
+        logger.debug(f"{self.name} is initialized.")
+
+    @property
+    def _default_filters(self) -> list[Filter]:
+        """The default filters for all datasets
+
+        Concrete classes can override this property to add more filters.
+
+        Returns:
+            List of filters to be applied to the dataset.
+        """
+        if not self.config.user_ix or not self.config.item_ix:
+            raise AttributeError("config.user_ix or config.item_ix not set.")
+
+        filters: list[Filter] = []
+        filters.append(
+            MinItemsPerUser(
+                min_items_per_user=3,
+                item_ix=self.config.item_ix,
+                user_ix=self.config.user_ix,
+            )
+        )
+        filters.append(
+            MinUsersPerItem(
+                min_users_per_item=3,
+                item_ix=self.config.item_ix,
+                user_ix=self.config.user_ix,
+            )
+        )
+        return filters
+
+    def add_filter(self, filter_: Filter) -> None:
+        """Add a filter to be applied when loading the data.
+
+        Utilize :class:`DataFramePreprocessor` class to add filters to the
+        dataset to load. The filter will be applied when the data is loaded into
+        an :class:`InteractionMatrix` object when :meth:`load` is called.
+
+        :param filter_: Filter to be applied to the loaded DataFrame
+                    processing to interaction matrix.
+        :type filter_: Filter
+        """
+        self.preprocessor.add_filter(filter_)
+
+    def _load_dataframe_from_cache(self) -> pd.DataFrame:
+        if not os.path.exists(self.processed_cache_path):
+            raise FileNotFoundError("Processed cache file not found.")
+        logger.info(f"Loading from cache: {self.processed_cache_path}")
+        df = pd.read_parquet(self.processed_cache_path)
+        return df
+
+    def load(self, apply_filters: bool = True, use_cache: bool = True) -> InteractionMatrix:
+        """Loads data into an InteractionMatrix object.
+
+        Data is loaded into a DataFrame using the :func:`_load_dataframe` function.
+        Resulting DataFrame is parsed into an :class:`InteractionMatrix` object. If
+        :data:`apply_filters` is set to True, the filters set will be applied to the
+        dataset and mapping of user and item ids will be done. This is advised
+        even if there is no filter set, as it will ensure that the user and item
+        ids are incrementing in the order of time.
+
+        Args:
+            apply_filters: To apply the filters set and preprocessing,
+                defaults to True
+            use_cache: Whether to use cached processed data, defaults to True
+
+        Returns:
+            Resulting interaction matrix.
+        """
+        logger.info(f"{self.name} is loading dataset...")
+        start = time.time()
+        try:
+            df = self._load_dataframe_from_cache() if use_cache else self._load_dataframe()
+        except FileNotFoundError:
+            logger.warning("Processed cache not found, loading raw dataframe.")
+            df = self._load_dataframe()
+            self._cache_processed_dataframe(df)
+        if apply_filters:
+            logger.debug(f"{self.name} applying filters set.")
+            im = self.preprocessor.process(df)
+        else:
+            im = self._dataframe_to_matrix(df)
+            logger.warning(
+                "No filters applied, user and item ids may not be incrementing in the order of time. "
+                "Classes that use this dataset may not work as expected."
+            )
+
+        if self.fetch_metadata:
+            user_id_mapping, item_id_mapping = (
+                self.preprocessor.user_id_mapping,
+                self.preprocessor.item_id_mapping,
+            )
+            self._fetch_dataset_metadata(
+                user_id_mapping=user_id_mapping, item_id_mapping=item_id_mapping
+            )
+
+        end = time.time()
+        logger.info(f"{self.name} dataset loaded - Took {end - start:.3}s")
+        return im
+
+    def _dataframe_to_matrix(self, df: pd.DataFrame) -> InteractionMatrix:
+        """Converts a DataFrame to an InteractionMatrix.
+
+        Args:
+            df: DataFrame to convert
+
+        Returns:
+            InteractionMatrix object.
+        """
+        if not self.config.user_ix or not self.config.item_ix or not self.config.timestamp_ix:
+            raise AttributeError("config.user_ix, config.item_ix or config.timestamp_ix not set.")
+        return InteractionMatrix(
+            df,
+            user_ix=self.config.user_ix,
+            item_ix=self.config.item_ix,
+            timestamp_ix=self.config.timestamp_ix,
+        )
+
+    @abstractmethod
+    def _fetch_dataset_metadata(
+        self, user_id_mapping: pd.DataFrame, item_id_mapping: pd.DataFrame
+    ) -> None:
+        """Fetch metadata for the dataset.
+
+        Fetch metadata for the dataset, if available.
+        """
+        raise NotImplementedError("Needs to be implemented")

recnexteval/datasets/config/__init__.py

@@ -0,0 +1,113 @@
+"""Dataset configuration module.
+
+This module provides configuration classes for dataset loading and metadata
+handling. Configurations define dataset properties such as paths, URLs, and
+processing parameters.
+
+## Available Configurations
+
+### Base Classes
+
+- `DatasetConfig`: Base class for dataset configurations
+- `MetadataConfig`: Base class for metadata configurations
+
+### Dataset Configurations
+
+- `MovieLensDatasetConfig`: Base configuration for MovieLens datasets
+- `MovieLens100KDatasetConfig`: Configuration for MovieLens 100K dataset
+- `AmazonDatasetConfig`: Base configuration for Amazon datasets
+- `AmazonMusicDatasetConfig`: Configuration for Amazon Music dataset
+- `AmazonMovieDatasetConfig`: Configuration for Amazon Movies dataset
+- `AmazonBookDatasetConfig`: Configuration for Amazon Books dataset
+- `AmazonSubscriptionBoxesDatasetConfig`: Configuration for Amazon Subscription Boxes dataset
+- `LastFMDatasetConfig`: Configuration for Last.FM dataset
+- `YelpDatasetConfig`: Configuration for Yelp dataset
+
+### Metadata Configurations
+
+- `MovieLens100kItemMetadataConfig`: Item metadata configuration for MovieLens 100K
+- `MovieLens100kUserMetadataConfig`: User metadata configuration for MovieLens 100K
+- `AmazonItemMetadataConfig`: Base Amazon item metadata configuration
+- `AmazonBooksItemMetadataConfig`: Item metadata for Amazon Books
+- `AmazonDigitalMusicItemMetadataConfig`: Item metadata for Amazon Digital Music
+- `AmazonMoviesAndTVItemMetadataConfig`: Item metadata for Amazon Movies and TV
+- `AmazonSubscriptionBoxesItemMetadataConfig`: Item metadata for Amazon Subscription Boxes
+- `LastFMUserMetadataConfig`: User metadata configuration for Last.FM
+- `LastFMItemMetadataConfig`: Item metadata configuration for Last.FM
+- `LastFMTagMetadataConfig`: Tag metadata configuration for Last.FM
+
+## Usage
+
+A typical usage pattern is to import a dataset config, optionally override fields,
+and pass it to dataset-loading utilities or custom convenience wrappers:
+
+```python
+from recnexteval.datasets.config import AmazonMusicDatasetConfig
+
+# Create config instance using defaults
+cfg = AmazonMusicDatasetConfig()
+
+# Inspect config values
+print(cfg.name)
+print(cfg.local_path)
+print(cfg.source_url)
+
+# Optionally override defaults at runtime
+custom_cfg = AmazonMusicDatasetConfig(
+    min_user_interactions=5,
+    min_item_interactions=10
+)
+```
+"""
+
+from .amazon import (
+    AmazonBookDatasetConfig,
+    AmazonBooksItemMetadataConfig,
+    AmazonDatasetConfig,
+    AmazonDigitalMusicItemMetadataConfig,
+    AmazonItemMetadataConfig,
+    AmazonMovieDatasetConfig,
+    AmazonMoviesAndTVItemMetadataConfig,
+    AmazonMusicDatasetConfig,
+    AmazonSubscriptionBoxesDatasetConfig,
+    AmazonSubscriptionBoxesItemMetadataConfig,
+)
+from .base import DatasetConfig, MetadataConfig
+from .lastfm import (
+    LastFMDatasetConfig,
+    LastFMItemMetadataConfig,
+    LastFMTagMetadataConfig,
+    LastFMUserMetadataConfig,
+)
+from .movielens import (
+    MovieLens100KDatasetConfig,
+    MovieLens100kItemMetadataConfig,
+    MovieLens100kUserMetadataConfig,
+    MovieLensDatasetConfig,
+)
+from .yelp import YelpDatasetConfig
+
+
+__all__ = [
+    "AmazonDatasetConfig",
+    "AmazonMusicDatasetConfig",
+    "AmazonMovieDatasetConfig",
+    "AmazonBookDatasetConfig",
+    "AmazonSubscriptionBoxesDatasetConfig",
+    "LastFMDatasetConfig",
+    "YelpDatasetConfig",
+    "DatasetConfig",
+    "MetadataConfig",
+    "MovieLensDatasetConfig",
+    "MovieLens100KDatasetConfig",
+    "MovieLens100kItemMetadataConfig",
+    "MovieLens100kUserMetadataConfig",
+    "AmazonBooksItemMetadataConfig",
+    "AmazonDigitalMusicItemMetadataConfig",
+    "AmazonItemMetadataConfig",
+    "AmazonMoviesAndTVItemMetadataConfig",
+    "AmazonSubscriptionBoxesItemMetadataConfig",
+    "LastFMItemMetadataConfig",
+    "LastFMTagMetadataConfig",
+    "LastFMUserMetadataConfig",
+]