recnexteval 0.1.0__py3-none-any.whl

recnexteval/datasets/datasets/amazon.py

@@ -0,0 +1,151 @@
+import logging
+from typing import ClassVar
+
+import numpy as np
+import pandas as pd
+
+from ..config import (
+    AmazonBookDatasetConfig,
+    AmazonDatasetConfig,
+    AmazonMovieDatasetConfig,
+    AmazonMusicDatasetConfig,
+    AmazonSubscriptionBoxesDatasetConfig,
+)
+from ..metadata.amazon import (
+    AmazonBookItemMetadata,
+    AmazonMovieItemMetadata,
+    AmazonMusicItemMetadata,
+    AmazonSubscriptionBoxesItemMetadata,
+)
+from .base import Dataset
+
+
+logger = logging.getLogger(__name__)
+
+
+class AmazonDataset(Dataset):
+    """Base class for Amazon datasets.
+
+    Other Amazon datasets should inherit from this class.
+    """
+
+    ITEM_METADATA = None
+    IS_BASE: bool = True
+
+    config: ClassVar[AmazonDatasetConfig] = AmazonDatasetConfig()
+
+    def _download_dataset(self) -> None:
+        """Downloads the dataset.
+
+        Downloads the csv file from the dataset URL and saves it to the file path.
+        """
+        if not self.config.dataset_url:
+            raise ValueError(f"{self.name} does not have URL specified in config.")
+
+        logger.debug(f"Downloading {self.name} dataset from {self.config.dataset_url}")
+        self._fetch_remote(
+            self.config.dataset_url,
+            self.file_path,
+        )
+
+    def _load_dataframe(self) -> pd.DataFrame:
+        """Load the raw dataset from file, and return it as a pandas DataFrame.
+
+        Transform the dataset downloaded to have integer user and item ids. This
+        will be needed for representation in the interaction matrix.
+
+        Returns:
+            The interaction data as a DataFrame with a row per interaction.
+        """
+        self.fetch_dataset()
+
+        # Read JSONL in chunks and show progress per chunk. We import tqdm
+        # locally to avoid global pandas monkeypatching (`tqdm.pandas()`).
+        from tqdm.auto import tqdm
+
+        chunksize = 100_000
+        chunks = pd.read_json(
+            self.file_path,
+            dtype={
+                self.config.item_ix: str,
+                self.config.user_ix: str,
+                self.config.timestamp_ix: np.int64,
+                self.config.rating_ix: np.float32,
+                self.config.helpful_vote_ix: np.int64,
+            },
+            lines=True,
+            chunksize=chunksize,
+        )
+        df = pd.concat(
+            [chunk for chunk in tqdm(chunks, desc="Reading JSONL", unit="chunk")], ignore_index=True
+        )
+
+        df = df[
+            [
+                self.config.item_ix,
+                self.config.user_ix,
+                self.config.timestamp_ix,
+                self.config.rating_ix,
+                self.config.helpful_vote_ix,
+            ]
+        ]
+
+        # Convert nanosecond timestamps to seconds
+        df[self.config.timestamp_ix] = df[self.config.timestamp_ix] // 1_000_000_000
+
+        logger.debug(f"Loaded {len(df)} interactions")
+        return df
+
+
+class AmazonMusicDataset(AmazonDataset):
+    """Handles Amazon Music dataset."""
+
+    IS_BASE: bool = False
+
+    config: ClassVar[AmazonMusicDatasetConfig] = AmazonMusicDatasetConfig()
+
+    def _fetch_dataset_metadata(
+        self, user_id_mapping: pd.DataFrame, item_id_mapping: pd.DataFrame
+    ) -> None:
+        self.ITEM_METADATA = AmazonMusicItemMetadata(item_id_mapping=item_id_mapping).load()
+
+
+class AmazonMovieDataset(AmazonDataset):
+    """Handles Amazon Movie dataset."""
+
+    IS_BASE: bool = False
+
+    config: ClassVar[AmazonMovieDatasetConfig] = AmazonMovieDatasetConfig()
+
+    def _fetch_dataset_metadata(
+        self, user_id_mapping: pd.DataFrame, item_id_mapping: pd.DataFrame
+    ) -> None:
+        self.ITEM_METADATA = AmazonMovieItemMetadata(item_id_mapping=item_id_mapping).load()
+
+
+class AmazonSubscriptionBoxesDataset(AmazonDataset):
+    """Handles Amazon Computer dataset."""
+
+    IS_BASE: bool = False
+
+    config: ClassVar[AmazonSubscriptionBoxesDatasetConfig] = AmazonSubscriptionBoxesDatasetConfig()
+
+    def _fetch_dataset_metadata(
+        self, user_id_mapping: pd.DataFrame, item_id_mapping: pd.DataFrame
+    ) -> None:
+        self.ITEM_METADATA = AmazonSubscriptionBoxesItemMetadata(
+            item_id_mapping=item_id_mapping
+        ).load()
+
+
+class AmazonBookDataset(AmazonDataset):
+    """Handles Amazon Book dataset."""
+
+    IS_BASE: bool = False
+
+    config: ClassVar[AmazonBookDatasetConfig] = AmazonBookDatasetConfig()
+
+    def _fetch_dataset_metadata(
+        self, user_id_mapping: pd.DataFrame, item_id_mapping: pd.DataFrame
+    ) -> None:
+        self.ITEM_METADATA = AmazonBookItemMetadata(item_id_mapping=item_id_mapping).load()

recnexteval/datasets/datasets/base.py

@@ -0,0 +1,250 @@
+import logging
+import os
+import time
+from abc import abstractmethod
+from datetime import datetime
+from typing import ClassVar
+
+import pandas as pd
+
+from recnexteval.matrix import InteractionMatrix
+from recnexteval.preprocessing.filter import Filter, MinItemsPerUser, MinUsersPerItem
+from recnexteval.preprocessing.preprocessor import DataFramePreprocessor
+from recnexteval.utils.path import safe_dir
+from ..base import DataFetcher
+from ..config import DatasetConfig
+
+
+logger = logging.getLogger(__name__)
+
+
+class Dataset(DataFetcher):
+    """Represents a collaborative filtering dataset.
+
+    Dataset must minimally contain user, item and timestamp columns for the
+    other modules to work.
+
+    Assumption
+    ===========
+    User/item ID increments in the order of time. This is an assumption that will
+    be made for the purposes of splitting the dataset and eventually passing
+    the dataset to the model. The ID incrementing in the order of time allows us
+    to set the shape of the currently known user and item matrix allowing easier
+    manipulation of the data by the evaluator.
+
+    :param filename: Name of the file, if no name is provided the dataset default will be used if known.
+        If the dataset does not have a default filename, a ValueError will be raised.
+    :type filename: str, optional
+    :param base_path: The base_path to the data directory.
+        Defaults to `data`
+    :type base_path: str, optional
+    :param use_default_filters: If True, the default filters will be applied to the dataset.
+        Defaults to False.
+    :type use_default_filters: bool, optional
+    """
+
+    config: ClassVar[DatasetConfig] = DatasetConfig()
+    """Configuration for the dataset."""
+
+    def __init__(
+        self,
+        use_default_filters: bool = False,  # noqa: FBT001, FBT002
+        fetch_metadata: bool = False,  # noqa: FBT001, FBT002
+    ) -> None:
+        if not self.config.user_ix or not self.config.item_ix or not self.config.timestamp_ix:
+            raise AttributeError("user_ix, item_ix or timestamp_ix not set in config.")
+
+        logger.debug(
+            f"{self.name} being initialized with '{self.config.default_base_path}' as the base path."
+        )
+
+        if not self.config.default_filename:
+            raise ValueError("No filename specified, and no default known.")
+
+        self.fetch_metadata = fetch_metadata
+        self.preprocessor = DataFramePreprocessor(
+            self.config.item_ix, self.config.user_ix, self.config.timestamp_ix
+        )
+        self._timestamp_min: int | None = None
+        self._timestamp_max: int | None = None
+
+        if use_default_filters:
+            for f in self._default_filters:
+                self.add_filter(f)
+
+        safe_dir(self.config.default_base_path)
+        logger.debug(f"{self.name} is initialized.")
+
+    @property
+    def _default_filters(self) -> list[Filter]:
+        """The default filters for all datasets
+
+        Concrete classes can override this property to add more filters.
+
+        Returns:
+            List of filters to be applied to the dataset.
+        """
+        if not self.config.user_ix or not self.config.item_ix:
+            raise AttributeError("config.user_ix or config.item_ix not set.")
+
+        filters: list[Filter] = []
+        filters.append(
+            MinItemsPerUser(
+                min_items_per_user=3,
+                item_ix=self.config.item_ix,
+                user_ix=self.config.user_ix,
+            )
+        )
+        filters.append(
+            MinUsersPerItem(
+                min_users_per_item=3,
+                item_ix=self.config.item_ix,
+                user_ix=self.config.user_ix,
+            )
+        )
+        return filters
+
+    @property
+    def timestamp_min(self) -> int:
+        """Minimum timestamp in the dataset.
+
+        Returns:
+            Minimum timestamp in the dataset.
+        """
+        if self._timestamp_min is None:
+            raise RuntimeError("timestamp_min can only be accessed after load() has been called.")
+        return self._timestamp_min
+
+    @property
+    def timestamp_max(self) -> int:
+        """Maximum timestamp in the dataset.
+
+        Returns:
+            Maximum timestamp in the dataset.
+        """
+        if self._timestamp_max is None:
+            raise RuntimeError("timestamp_max can only be accessed after load() has been called.")
+        return self._timestamp_max
+
+    def get_timestamp_range_in_epoch(self) -> tuple[int, int]:
+        """Get the minimum and maximum timestamps in the dataset.
+
+        Returns:
+            A tuple of (min_timestamp, max_timestamp).
+
+        Raises:
+            RuntimeError: If load() has not been called yet.
+        """
+        return self.timestamp_min, self.timestamp_max
+
+    def get_timestamp_range_in_datetime(self) -> tuple[datetime, datetime]:
+        """Get the minimum and maximum timestamps in the dataset.
+
+        Returns:
+            A tuple of (min_timestamp, max_timestamp).
+
+        Raises:
+            RuntimeError: If load() has not been called yet.
+        """
+        min_dt = datetime.fromtimestamp(self.timestamp_min)
+        max_dt = datetime.fromtimestamp(self.timestamp_max)
+        return min_dt, max_dt
+
+    def add_filter(self, filter_: Filter) -> None:
+        """Add a filter to be applied when loading the data.
+
+        Utilize :class:`DataFramePreprocessor` class to add filters to the
+        dataset to load. The filter will be applied when the data is loaded into
+        an :class:`InteractionMatrix` object when :meth:`load` is called.
+
+        :param filter_: Filter to be applied to the loaded DataFrame
+                    processing to interaction matrix.
+        :type filter_: Filter
+        """
+        self.preprocessor.add_filter(filter_)
+
+    def _load_dataframe_from_cache(self) -> pd.DataFrame:
+        if not os.path.exists(self.processed_cache_path):
+            raise FileNotFoundError("Processed cache file not found.")
+        logger.info(f"Loading from cache: {self.processed_cache_path}")
+        df = pd.read_parquet(self.processed_cache_path)
+        return df
+
+    def load(self, apply_filters: bool = True, use_cache: bool = True) -> InteractionMatrix:
+        """Loads data into an InteractionMatrix object.
+
+        Data is loaded into a DataFrame using the :func:`_load_dataframe` function.
+        Resulting DataFrame is parsed into an :class:`InteractionMatrix` object. If
+        :data:`apply_filters` is set to True, the filters set will be applied to the
+        dataset and mapping of user and item ids will be done. This is advised
+        even if there is no filter set, as it will ensure that the user and item
+        ids are incrementing in the order of time.
+
+        Args:
+            apply_filters: To apply the filters set and preprocessing,
+                defaults to True
+            use_cache: Whether to use cached processed data, defaults to True
+
+        Returns:
+            Resulting interaction matrix.
+        """
+        logger.info(f"{self.name} is loading dataset...")
+        start = time.time()
+        try:
+            df = self._load_dataframe_from_cache() if use_cache else self._load_dataframe()
+        except FileNotFoundError:
+            logger.warning("Processed cache not found, loading raw dataframe.")
+            df = self._load_dataframe()
+            self._cache_processed_dataframe(df)
+        if apply_filters:
+            logger.debug(f"{self.name} applying filters set.")
+            im = self.preprocessor.process(df)
+        else:
+            im = self._dataframe_to_matrix(df)
+            logger.warning(
+                "No filters applied, user and item ids may not be incrementing in the order of time. "
+                "Classes that use this dataset may not work as expected."
+            )
+        self._timestamp_min = int(df[self.config.timestamp_ix].min())
+        self._timestamp_max = int(df[self.config.timestamp_ix].max())
+
+        if self.fetch_metadata:
+            user_id_mapping, item_id_mapping = (
+                self.preprocessor.user_id_mapping,
+                self.preprocessor.item_id_mapping,
+            )
+            self._fetch_dataset_metadata(
+                user_id_mapping=user_id_mapping, item_id_mapping=item_id_mapping
+            )
+
+        end = time.time()
+        logger.info(f"{self.name} dataset loaded - Took {end - start:.3}s")
+        return im
+
+    def _dataframe_to_matrix(self, df: pd.DataFrame) -> InteractionMatrix:
+        """Converts a DataFrame to an InteractionMatrix.
+
+        Args:
+            df: DataFrame to convert
+
+        Returns:
+            InteractionMatrix object.
+        """
+        if not self.config.user_ix or not self.config.item_ix or not self.config.timestamp_ix:
+            raise AttributeError("config.user_ix, config.item_ix or config.timestamp_ix not set.")
+        return InteractionMatrix(
+            df,
+            user_ix=self.config.user_ix,
+            item_ix=self.config.item_ix,
+            timestamp_ix=self.config.timestamp_ix,
+        )
+
+    @abstractmethod
+    def _fetch_dataset_metadata(
+        self, user_id_mapping: pd.DataFrame, item_id_mapping: pd.DataFrame
+    ) -> None:
+        """Fetch metadata for the dataset.
+
+        Fetch metadata for the dataset, if available.
+        """
+        raise NotImplementedError("Needs to be implemented")

recnexteval/datasets/datasets/lastfm.py

@@ -0,0 +1,121 @@
+import logging
+import os
+import zipfile
+
+import numpy as np
+import pandas as pd
+from tqdm import tqdm
+from typing_extensions import ClassVar
+
+from ..config import LastFMDatasetConfig
+from ..metadata.lastfm import LastFMItemMetadata, LastFMTagMetadata, LastFMUserMetadata
+from .base import Dataset
+
+
+logger = logging.getLogger(__name__)
+tqdm.pandas()
+
+
+class LastFMDataset(Dataset):
+    """
+    Last FM dataset.
+
+    The Last FM dataset contains user interactions with artists. The tags in this
+    datasets are not used in this implementation. The dataset that will be used
+    would the the user_taggedartists-timestamps.dat file. The dataset contains
+    the following columns: [user, artist, tags, timestamp].
+
+    The dataset is downloaded from the GroupLens website :cite:`Cantador_RecSys2011`.
+    """
+    IS_BASE: bool = False
+
+    config: ClassVar[LastFMDatasetConfig] = LastFMDatasetConfig()
+
+    ITEM_METADATA = None
+    USER_METADATA = None
+    TAG_METADATA = None
+
+    def fetch_dataset(self) -> None:
+        """Check if dataset is present, if not download.
+
+        This method overrides the base class to handle the special case where
+        the zipfile may exist but the extracted file doesn't.
+        """
+        zip_path = os.path.join(
+            self.config.default_base_path, f"{self.config.remote_zipname}.zip"
+        )
+
+        if not os.path.exists(zip_path):
+            logger.debug(f"{self.name} dataset zipfile not found in {zip_path}.")
+            self._download_dataset()
+        elif not os.path.exists(self.file_path):
+            logger.debug(
+                f"{self.name} dataset file not found, but zipfile already downloaded. "
+                f"Extracting file from zipfile."
+            )
+            with zipfile.ZipFile(zip_path, "r") as zip_ref:
+                zip_ref.extract(self.config.remote_filename, self.config.default_base_path)
+        else:
+            logger.debug("Data zipfile is in memory and in dir specified.")
+
+    def _download_dataset(self) -> None:
+        """Downloads the dataset.
+
+        Downloads the zipfile, and extracts the interaction file to `self.file_path`
+        """
+        zip_path = os.path.join(
+            self.config.default_base_path, f"{self.config.remote_zipname}.zip"
+        )
+
+        logger.debug(f"Downloading {self.name} dataset from {self.config.dataset_url}")
+
+        # Download the zip into the data directory
+        self._fetch_remote(
+            f"{self.config.dataset_url}/{self.config.remote_zipname}.zip",
+            zip_path,
+        )
+
+        # Extract the interaction file which we will use
+        logger.debug(f"Extracting {self.config.remote_filename} from zip")
+        with zipfile.ZipFile(zip_path, "r") as zip_ref:
+            zip_ref.extract(self.config.remote_filename, self.config.default_base_path)
+
+    def _load_dataframe(self) -> pd.DataFrame:
+        """Load the raw dataset from file, and return it as a pandas DataFrame.
+
+        Transform the dataset downloaded to have integer user and item ids. This
+        will be needed for representation in the interaction matrix.
+
+        Returns:
+            The interaction data as a DataFrame with a row per interaction.
+        """
+        self.fetch_dataset()
+        df = pd.read_csv(
+            self.file_path,
+            dtype={
+                self.config.item_ix: np.int32,
+                self.config.user_ix: np.int32,
+                self.config.tag_ix: np.int32,
+                self.config.timestamp_ix: np.int64,
+            },
+            sep="\t",
+            names=[
+                self.config.user_ix,
+                self.config.item_ix,
+                self.config.tag_ix,
+                self.config.timestamp_ix,
+            ],
+            header=0,
+        )
+        # Convert from milliseconds to seconds
+        df[self.config.timestamp_ix] = df[self.config.timestamp_ix] // 1_000
+
+        logger.debug(f"Loaded {len(df)} interactions")
+        return df
+
+    def _fetch_dataset_metadata(
+        self, user_id_mapping: pd.DataFrame, item_id_mapping: pd.DataFrame
+    ) -> None:
+        self.USER_METADATA = LastFMUserMetadata(user_id_mapping=user_id_mapping).load()
+        self.ITEM_METADATA = LastFMItemMetadata(item_id_mapping=item_id_mapping).load()
+        self.TAG_METADATA = LastFMTagMetadata().load()

recnexteval/datasets/datasets/movielens.py

@@ -0,0 +1,93 @@
+import logging
+import os
+import zipfile
+from typing import ClassVar
+
+import numpy as np
+import pandas as pd
+from tqdm import tqdm
+
+from ..config import MovieLens100KDatasetConfig, MovieLensDatasetConfig
+from ..metadata.movielens import MovieLens100kItemMetadata, MovieLens100kUserMetadata
+from .base import Dataset
+
+
+logger = logging.getLogger(__name__)
+
+
+class MovieLensDataset(Dataset):
+    """Base class for Movielens datasets.
+
+    Other Movielens datasets should inherit from this class.
+
+    This code is adapted from RecPack :cite:`recpack`
+    """
+
+    IS_BASE: bool = True
+    config: ClassVar[MovieLensDatasetConfig] = MovieLensDatasetConfig()
+
+    def _download_dataset(self) -> None:
+        # Download the zip into the data directory
+        zip_file_path = os.path.join(
+            self.config.default_base_path, f"{self.config.remote_zipname}.zip"
+        )
+        self._fetch_remote(
+            url=f"{self.config.dataset_url}/{self.config.remote_zipname}.zip",
+            filename=zip_file_path,
+        )
+
+        # Extract the ratings file which we will use
+        with zipfile.ZipFile(zip_file_path, "r") as zip_ref:
+            zip_ref.extract(
+                f"{self.config.remote_zipname}/{self.config.remote_filename}",
+                self.config.default_base_path,
+            )
+
+        # Rename the ratings file to the specified filename
+        os.rename(
+            os.path.join(
+                self.config.default_base_path,
+                f"{self.config.remote_zipname}/{self.config.remote_filename}",
+            ),
+            self.file_path,
+        )
+
+
+class MovieLens100K(MovieLensDataset):
+    """MovieLens 100K dataset."""
+
+    ITEM_METADATA = None
+    USER_METADATA = None
+    IS_BASE: bool = False
+
+    config: ClassVar[MovieLens100KDatasetConfig] = MovieLens100KDatasetConfig()
+
+    def _load_dataframe(self) -> pd.DataFrame:
+        self.fetch_dataset()
+        chunks = pd.read_table(
+            self.file_path,
+            dtype={
+                self.config.user_ix: np.int64,
+                self.config.item_ix: np.int64,
+                self.config.rating_ix: np.float64,
+                self.config.timestamp_ix: np.int64,
+            },
+            sep="\t",
+            names=[
+                self.config.user_ix,
+                self.config.item_ix,
+                self.config.rating_ix,
+                self.config.timestamp_ix,
+            ],
+            chunksize=100_000,
+        )
+        df = pd.concat(
+            [chunk for chunk in tqdm(chunks, desc="Reading table", unit="chunk")], ignore_index=True
+        )
+        return df
+
+    def _fetch_dataset_metadata(
+        self, user_id_mapping: pd.DataFrame, item_id_mapping: pd.DataFrame
+    ) -> None:
+        self.USER_METADATA = MovieLens100kUserMetadata(user_id_mapping=user_id_mapping).load()
+        self.ITEM_METADATA = MovieLens100kItemMetadata(item_id_mapping=item_id_mapping).load()

recnexteval/datasets/datasets/test.py

@@ -0,0 +1,46 @@
+import pandas as pd
+
+from .base import Dataset
+
+
+class TestDataset(Dataset):
+    """
+    Test dataset.
+
+    The test dataset is a dummy dataset that is used for testing purposes.
+    """
+
+    USER_IX = "user_id"
+    """Name of the column in the DataFrame that contains user identifiers."""
+    ITEM_IX = "item_id"
+    """Name of the column in the DataFrame that contains item identifiers."""
+    TIMESTAMP_IX = "timestamp"
+    """Name of the column in the DataFrame that contains time of interaction in seconds since epoch."""
+    RATING_IX = "rating"
+    """Name of the column in the DataFrame that contains the rating a user gave to the item."""
+    DEFAULT_FILENAME = "dummy_input.csv"
+
+    def _download_dataset(self) -> None:
+        pass
+
+    def _load_dataframe(self) -> pd.DataFrame:
+        """Load the raw dataset from file, and return it as a pandas DataFrame.
+
+        Warning:
+            This does not apply any preprocessing, and returns the raw dataset.
+
+        Returns:
+            The interaction data as a DataFrame with a row per interaction.
+        """
+        input_dict = {
+            self.USER_IX: [1, 2, 3, 1, 2, 2, 4, 3, 3, 4, 5, 5, 5],
+            self.ITEM_IX: [1, 1, 2, 3, 2, 3, 2, 1, 3, 3, 1, 2, 3],
+            self.TIMESTAMP_IX: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 9, 10, 10],
+            self.RATING_IX: [1, 2, 3, 4, 5, 1, 2, 3, 4, 5, 1, 2, 3],
+        }
+
+        df = pd.DataFrame.from_dict(input_dict)
+        return df
+
+    def _fetch_dataset_metadata(self, user_id_mapping: pd.DataFrame, item_id_mapping: pd.DataFrame) -> None:
+        pass