nextrec 0.3.4__py3-none-any.whl → 0.3.6__py3-none-any.whl

nextrec/__init__.py

@@ -1,33 +1,3 @@
-"""
-NextRec - A Unified Deep Learning Framework for Recommender Systems
-===================================================================
-
-NextRec provides a comprehensive suite of recommendation models including:
-- Ranking models (CTR prediction)
-- Matching models (retrieval)
-- Multi-task learning models
-- Generative recommendation models
-
-Quick Start
------------
->>> from nextrec.basic.features import DenseFeature, SparseFeature
->>> from nextrec.models.ranking.deepfm import DeepFM
->>> 
->>> # Define features
->>> dense_features = [DenseFeature('age')]
->>> sparse_features = [SparseFeature('category', vocab_size=100, embedding_dim=16)]
->>> 
->>> # Build model
->>> model = DeepFM(
-...     dense_features=dense_features,
-...     sparse_features=sparse_features,
-...     targets=['label']
-... )
->>> 
->>> # Train model
->>> model.fit(train_data=df_train, valid_data=df_valid)
-"""
-
 from nextrec.__version__ import __version__
 
 __all__ = [

nextrec/__version__.py

@@ -1 +1 @@
-__version__ = "0.3.4"
+__version__ = "0.3.6"

nextrec/basic/features.py

@@ -7,7 +7,7 @@ Author: Yang Zhou, zyaztec@gmail.com
 """
 import torch
 from nextrec.utils.embedding import get_auto_embedding_dim
-from nextrec.utils.common import normalize_to_list
+from nextrec.utils.feature import normalize_to_list
 
 class BaseFeature(object):
     def __repr__(self):

nextrec/basic/loggers.py

@@ -99,7 +99,7 @@ def setup_logger(session_id: str | os.PathLike | None = None):
     session = create_session(str(session_id) if session_id is not None else None)
     log_dir = session.logs_dir
     log_dir.mkdir(parents=True, exist_ok=True)
-    log_file = log_dir / f"{session.experiment_id}.log"
+    log_file = log_dir / f"{session.log_basename}.log"
 
     console_format = '%(message)s'
     file_format = '%(asctime)s - %(levelname)s - %(message)s'

nextrec/basic/model.py

@@ -31,10 +31,12 @@ from nextrec.basic.session import resolve_save_path, create_session
 from nextrec.basic.metrics import configure_metrics, evaluate_metrics, check_user_id
 
 from nextrec.data.dataloader import build_tensors_from_data
-from nextrec.data.data_utils import get_column_data, collate_fn, batch_to_dict, get_user_ids
+from nextrec.data.data_processing import get_column_data, get_user_ids
+from nextrec.data.batch_utils import collate_fn, batch_to_dict
 
 from nextrec.loss import get_loss_fn, get_loss_kwargs
-from nextrec.utils import get_optimizer, get_scheduler, to_tensor
+from nextrec.utils import get_optimizer, get_scheduler
+from nextrec.utils.tensor import to_tensor
 
 from nextrec import __version__
 
@@ -153,7 +155,7 @@ class BaseModel(FeatureSet, nn.Module):
                 raise ValueError("[BaseModel-input Error] Labels are required but none were found in the input batch.")
         return X_input, y
 
-    def handle_validation_split(self, train_data: dict | pd.DataFrame, validation_split: float, batch_size: int, shuffle: bool,) -> tuple[DataLoader, dict | pd.DataFrame]:
+    def handle_validation_split(self, train_data: dict | pd.DataFrame, validation_split: float, batch_size: int, shuffle: bool, num_workers: int = 0,) -> tuple[DataLoader, dict | pd.DataFrame]:
         """This function will split training data into training and validation sets when: 1. valid_data is None; 2. validation_split is provided."""
         if not (0 < validation_split < 1):
             raise ValueError(f"[BaseModel-validation Error] validation_split must be between 0 and 1, got {validation_split}")
@@ -182,7 +184,7 @@ class BaseModel(FeatureSet, nn.Module):
                 arr = np.asarray(value)      
                 train_split[key] = arr[train_indices]
                 valid_split[key] = arr[valid_indices]
-        train_loader = self.prepare_data_loader(train_split, batch_size=batch_size, shuffle=shuffle)
+        train_loader = self.prepare_data_loader(train_split, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers)
         logging.info(f"Split data: {len(train_indices)} training samples, {len(valid_indices)} validation samples")
         return train_loader, valid_split
 
@@ -263,14 +265,14 @@ class BaseModel(FeatureSet, nn.Module):
                 task_losses.append(task_loss)
             return torch.stack(task_losses).sum()
 
-    def prepare_data_loader(self, data: dict | pd.DataFrame | DataLoader, batch_size: int = 32, shuffle: bool = True,):
+    def prepare_data_loader(self, data: dict | pd.DataFrame | DataLoader, batch_size: int = 32, shuffle: bool = True, num_workers: int = 0,) -> DataLoader:
         if isinstance(data, DataLoader):
             return data
         tensors = build_tensors_from_data(data=data, raw_data=data, features=self.all_features, target_columns=self.target_columns, id_columns=self.id_columns,)
         if tensors is None:
             raise ValueError("[BaseModel-prepare_data_loader Error] No data available to create DataLoader.")
         dataset = TensorDictDataset(tensors)
-        return DataLoader(dataset, batch_size=batch_size, shuffle=shuffle, collate_fn=collate_fn)
+        return DataLoader(dataset, batch_size=batch_size, shuffle=shuffle, collate_fn=collate_fn, num_workers=num_workers)
 
     def fit(self, 
             train_data: dict | pd.DataFrame | DataLoader, 
@@ -279,6 +281,7 @@ class BaseModel(FeatureSet, nn.Module):
             epochs:int=1, shuffle:bool=True, batch_size:int=32,
             user_id_column: str | None = None,
             validation_split: float | None = None,
+            num_workers: int = 0,
             tensorboard: bool = True,):
         self.to(self.device)
         if not self.logger_initialized:
@@ -295,11 +298,11 @@ class BaseModel(FeatureSet, nn.Module):
         self.best_metric = float('-inf') if self.best_metrics_mode == 'max' else float('inf')
 
         if validation_split is not None and valid_data is None:
-            train_loader, valid_data = self.handle_validation_split(train_data=train_data, validation_split=validation_split, batch_size=batch_size, shuffle=shuffle,) # type: ignore
+            train_loader, valid_data = self.handle_validation_split(train_data=train_data, validation_split=validation_split, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers) # type: ignore
         else:
-            train_loader = (train_data if isinstance(train_data, DataLoader) else self.prepare_data_loader(train_data, batch_size=batch_size, shuffle=shuffle))
+            train_loader = (train_data if isinstance(train_data, DataLoader) else self.prepare_data_loader(train_data, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers))
         
-        valid_loader, valid_user_ids = self.prepare_validation_data(valid_data=valid_data, batch_size=batch_size, needs_user_ids=self.needs_user_ids, user_id_column=user_id_column)
+        valid_loader, valid_user_ids = self.prepare_validation_data(valid_data=valid_data, batch_size=batch_size, needs_user_ids=self.needs_user_ids, user_id_column=user_id_column, num_workers=num_workers)
         try:
             self.steps_per_epoch = len(train_loader)
             is_streaming = False
@@ -386,7 +389,7 @@ class BaseModel(FeatureSet, nn.Module):
                 self.training_logger.log_metrics(train_log_payload, step=epoch + 1, split="train")
             if valid_loader is not None:
                 # pass user_ids only if needed for GAUC metric
-                val_metrics = self.evaluate(valid_loader, user_ids=valid_user_ids if self.needs_user_ids else None) # {'auc': 0.75, 'logloss': 0.45} or {'auc_target1': 0.75, 'logloss_target1': 0.45, 'mse_target2': 3.2}
+                val_metrics = self.evaluate(valid_loader, user_ids=valid_user_ids if self.needs_user_ids else None, num_workers=num_workers) # {'auc': 0.75, 'logloss': 0.45} or {'auc_target1': 0.75, 'logloss_target1': 0.45, 'mse_target2': 3.2}
                 if self.nums_task == 1:
                     metrics_str = ", ".join([f"{k}={v:.4f}" for k, v in val_metrics.items()])
                     logging.info(colorize(f"  Epoch {epoch + 1}/{epochs} - Valid: {metrics_str}", color="cyan"))
@@ -511,12 +514,12 @@ class BaseModel(FeatureSet, nn.Module):
             return avg_loss, metrics_dict
         return avg_loss
 
-    def prepare_validation_data(self, valid_data: dict | pd.DataFrame | DataLoader | None, batch_size: int, needs_user_ids: bool, user_id_column: str | None = 'user_id') -> tuple[DataLoader | None, np.ndarray | None]:
+    def prepare_validation_data(self, valid_data: dict | pd.DataFrame | DataLoader | None, batch_size: int, needs_user_ids: bool, user_id_column: str | None = 'user_id', num_workers: int = 0,) -> tuple[DataLoader | None, np.ndarray | None]:
         if valid_data is None:
             return None, None
         if isinstance(valid_data, DataLoader):
             return valid_data, None
-        valid_loader = self.prepare_data_loader(valid_data, batch_size=batch_size, shuffle=False)
+        valid_loader = self.prepare_data_loader(valid_data, batch_size=batch_size, shuffle=False, num_workers=num_workers)
         valid_user_ids = None
         if needs_user_ids:
             if user_id_column is None:
@@ -529,7 +532,8 @@ class BaseModel(FeatureSet, nn.Module):
                  metrics: list[str] | dict[str, list[str]] | None = None,
                  batch_size: int = 32,
                  user_ids: np.ndarray | None = None,
-                 user_id_column: str = 'user_id') -> dict:
+                 user_id_column: str = 'user_id',
+                 num_workers: int = 0,) -> dict:
         self.eval()
         eval_metrics = metrics if metrics is not None else self.metrics
         if eval_metrics is None:
@@ -541,7 +545,7 @@ class BaseModel(FeatureSet, nn.Module):
         else:
             if user_ids is None and needs_user_ids:
                 user_ids = get_user_ids(data=data, id_columns=user_id_column)
-            data_loader = self.prepare_data_loader(data, batch_size=batch_size, shuffle=False)
+            data_loader = self.prepare_data_loader(data, batch_size=batch_size, shuffle=False, num_workers=num_workers)
         y_true_list = []
         y_pred_list = []
         collected_user_ids = []
@@ -601,6 +605,7 @@ class BaseModel(FeatureSet, nn.Module):
         include_ids: bool | None = None,
         return_dataframe: bool = True,
         streaming_chunk_size: int = 10000,
+        num_workers: int = 0,
     ) -> pd.DataFrame | np.ndarray:
         self.eval()
         if include_ids is None:
@@ -613,7 +618,7 @@ class BaseModel(FeatureSet, nn.Module):
             rec_loader = RecDataLoader(dense_features=self.dense_features, sparse_features=self.sparse_features, sequence_features=self.sequence_features, target=self.target_columns, id_columns=self.id_columns,)
             data_loader = rec_loader.create_dataloader(data=data, batch_size=batch_size, shuffle=False, load_full=False, chunk_size=streaming_chunk_size,)
         elif not isinstance(data, DataLoader):
-            data_loader = self.prepare_data_loader(data, batch_size=batch_size, shuffle=False,)
+            data_loader = self.prepare_data_loader(data, batch_size=batch_size, shuffle=False, num_workers=num_workers)
         else:
             data_loader = data
         

nextrec/basic/session.py

@@ -1,14 +1,5 @@
 """Session and experiment utilities.
 
-This module centralizes session/experiment management so the rest of the
-framework writes all artifacts to a consistent location:: <pwd>/log/<experiment_id>/
-
-Within that folder we keep model parameters, checkpoints, training metrics,
-evaluation metrics, and consolidated log output. When users do not provide an
-``experiment_id`` a timestamp-based identifier is generated once per process to
-avoid scattering files across multiple directories. Test runs are redirected to
-temporary folders so local trees are not polluted.
-
 Date: create on 23/11/2025
 Author: Yang Zhou,zyaztec@gmail.com
 """
@@ -16,7 +7,7 @@ Author: Yang Zhou,zyaztec@gmail.com
 import os
 import tempfile
 from dataclasses import dataclass
-from datetime import datetime
+from datetime import datetime, timezone
 from pathlib import Path
 
 __all__ = [
@@ -31,6 +22,7 @@ class Session:
 
     experiment_id: str
     root: Path
+    log_basename: str  # The base name for log files, without path separators
 
     @property
     def logs_dir(self) -> Path:
@@ -69,13 +61,15 @@ class Session:
         return path
 
 def create_session(experiment_id: str | Path | None = None) -> Session:
-    """Create a :class:`Session` instance with prepared directories."""
 
     if experiment_id is not None and str(experiment_id).strip():
         exp_id = str(experiment_id).strip()
     else:
+        # Use local time for session naming
         exp_id = "nextrec_session_" + datetime.now().strftime("%Y%m%d")
 
+    log_basename = Path(exp_id).name if exp_id else exp_id
+
     if (
         os.getenv("PYTEST_CURRENT_TEST")
         or os.getenv("PYTEST_RUNNING")
@@ -90,7 +84,7 @@ def create_session(experiment_id: str | Path | None = None) -> Session:
     session_path.mkdir(parents=True, exist_ok=True)
     root = session_path.resolve()
 
-    return Session(experiment_id=exp_id, root=root)
+    return Session(experiment_id=exp_id, root=root, log_basename=log_basename)
 
 def resolve_save_path(
     path: str | os.PathLike | Path | None,
@@ -111,6 +105,7 @@ def resolve_save_path(
       timestamp.
     - Parent directories are created.
     """
+    # Use local time for file timestamps
     timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") if add_timestamp else None
 
     normalized_suffix = suffix if suffix.startswith(".") else f".{suffix}"

nextrec/data/__init__.py

@@ -1,48 +1,61 @@
-"""
-Data utilities package for NextRec
-
-This package provides data processing and manipulation utilities.
-
-Date: create on 13/11/2025
-Author: Yang Zhou, zyaztec@gmail.com
-"""
-
-from nextrec.data.data_utils import (
-    collate_fn,
+from nextrec.data.batch_utils import collate_fn, batch_to_dict, stack_section
+from nextrec.data.data_processing import (
     get_column_data,
-    default_output_dir,
     split_dict_random,
     build_eval_candidates,
+    get_user_ids,
+)
+
+from nextrec.utils.file import (
     resolve_file_paths,
     iter_file_chunks,
     read_table,
     load_dataframes,
+    default_output_dir,
 )
-from nextrec.basic.features import FeatureSet
-from nextrec.data import data_utils
+
 from nextrec.data.dataloader import (
     TensorDictDataset,
     FileDataset,
     RecDataLoader,
     build_tensors_from_data,
 )
+
 from nextrec.data.preprocessor import DataProcessor
+from nextrec.basic.features import FeatureSet
+from nextrec.data import data_utils
 
 __all__ = [
+    # Batch utilities
     'collate_fn',
+    'batch_to_dict',
+    'stack_section',
+    
+    # Data processing
     'get_column_data',
-    'default_output_dir',
     'split_dict_random',
     'build_eval_candidates',
+    'get_user_ids',
+    
+    # File utilities
     'resolve_file_paths',
     'iter_file_chunks',
     'read_table',
     'load_dataframes',
-    'FeatureSet',
-    'data_utils',
+    'default_output_dir',
+    
+    # DataLoader
     'TensorDictDataset',
     'FileDataset',
     'RecDataLoader',
     'build_tensors_from_data',
+    
+    # Preprocessor
     'DataProcessor',
+    
+    # Features
+    'FeatureSet',
+    
+    # Legacy module
+    'data_utils',
 ]

nextrec/data/batch_utils.py

@@ -0,0 +1,80 @@
+"""
+Batch collation utilities for NextRec
+
+Date: create on 03/12/2025
+Author: Yang Zhou, zyaztec@gmail.com
+"""
+
+import torch
+import numpy as np
+from typing import Any, Mapping
+
+def stack_section(batch: list[dict], section: str):
+    entries = [item.get(section) for item in batch if item.get(section) is not None]
+    if not entries:
+        return None
+    merged: dict = {}
+    for name in entries[0]:  # type: ignore
+        tensors = [item[section][name] for item in batch if item.get(section) is not None and name in item[section]]
+        merged[name] = torch.stack(tensors, dim=0)
+    return merged
+
+def collate_fn(batch):
+    """
+    Collate a list of sample dicts into the unified batch format:
+    {
+        "features": {name: Tensor(B, ...)},
+        "labels": {target: Tensor(B, ...)} or None,
+        "ids": {id_name: Tensor(B, ...)} or None,
+    }
+    Args: batch: List of samples from DataLoader
+        
+    Returns: dict: Batched data in unified format
+    """
+    if not batch:
+        return {"features": {}, "labels": None, "ids": None}
+
+    first = batch[0]
+    if isinstance(first, dict) and "features" in first:
+        # Streaming dataset yields already-batched chunks; avoid adding an extra dim.
+        if first.get("_already_batched") and len(batch) == 1:
+            return {
+                "features": first.get("features", {}),
+                "labels": first.get("labels"),
+                "ids": first.get("ids"),
+            }
+        return {
+            "features": stack_section(batch, "features") or {},
+            "labels": stack_section(batch, "labels"),
+            "ids": stack_section(batch, "ids"),
+        }
+
+    # Fallback: stack tuples/lists of tensors
+    num_tensors = len(first)
+    result = []
+    for i in range(num_tensors):
+        tensor_list = [item[i] for item in batch]
+        first_item = tensor_list[0]
+        if isinstance(first_item, torch.Tensor):
+            stacked = torch.cat(tensor_list, dim=0)
+        elif isinstance(first_item, np.ndarray):
+            stacked = np.concatenate(tensor_list, axis=0)
+        elif isinstance(first_item, list):
+            combined = []
+            for entry in tensor_list:
+                combined.extend(entry)
+            stacked = combined
+        else:
+            stacked = tensor_list
+        result.append(stacked)
+    return tuple(result)
+
+
+def batch_to_dict(batch_data: Any, include_ids: bool = True) -> dict:
+    if not (isinstance(batch_data, Mapping) and "features" in batch_data):
+        raise TypeError("[BaseModel-batch_to_dict Error] Batch data must be a dict with 'features' produced by the current DataLoader.")
+    return {
+        "features": batch_data.get("features", {}),
+        "labels": batch_data.get("labels"),
+        "ids": batch_data.get("ids") if include_ids else None,
+    }

nextrec/data/data_processing.py

@@ -0,0 +1,152 @@
+"""
+Data processing utilities for NextRec
+
+Date: create on 03/12/2025
+Author: Yang Zhou, zyaztec@gmail.com
+"""
+
+import torch
+import numpy as np
+import pandas as pd
+from typing import Any, Mapping
+
+
+def get_column_data(data: dict | pd.DataFrame, name: str):
+    if isinstance(data, dict):
+        return data[name] if name in data else None
+    elif isinstance(data, pd.DataFrame):
+        if name not in data.columns:
+            return None
+        return data[name].values
+    else:
+        if hasattr(data, name):
+            return getattr(data, name)
+        raise KeyError(f"Unsupported data type for extracting column {name}")
+
+def split_dict_random(
+    data_dict: dict,
+    test_size: float = 0.2,
+    random_state: int | None = None
+):
+    lengths = [len(v) for v in data_dict.values()]
+    if len(set(lengths)) != 1:
+        raise ValueError(f"Length mismatch: {lengths}")
+    
+    n = lengths[0]
+    rng = np.random.default_rng(random_state)
+    perm = rng.permutation(n)
+    cut = int(round(n * (1 - test_size)))
+    train_idx, test_idx = perm[:cut], perm[cut:]
+    
+    def take(v, idx):
+        if isinstance(v, np.ndarray):
+            return v[idx]
+        elif isinstance(v, pd.Series):
+            return v.iloc[idx].to_numpy()
+        else:
+            v_arr = np.asarray(v, dtype=object)
+            return v_arr[idx]
+    
+    train_dict = {k: take(v, train_idx) for k, v in data_dict.items()}
+    test_dict = {k: take(v, test_idx) for k, v in data_dict.items()}
+    return train_dict, test_dict
+
+
+def build_eval_candidates(
+    df_all: pd.DataFrame,
+    user_col: str,
+    item_col: str,
+    label_col: str,
+    user_features: pd.DataFrame,
+    item_features: pd.DataFrame,
+    num_pos_per_user: int = 5,
+    num_neg_per_pos: int = 50,
+    random_seed: int = 2025,
+) -> pd.DataFrame:
+    """
+    Build evaluation candidates with positive and negative samples for each user.
+    
+    Args:
+        df_all: Full interaction DataFrame
+        user_col: Name of the user ID column
+        item_col: Name of the item ID column
+        label_col: Name of the label column
+        user_features: DataFrame containing user features
+        item_features: DataFrame containing item features
+        num_pos_per_user: Number of positive samples per user (default: 5)
+        num_neg_per_pos: Number of negative samples per positive (default: 50)
+        random_seed: Random seed for reproducibility (default: 2025)
+        
+    Returns:
+        pd.DataFrame: Evaluation candidates with features
+    """
+    rng = np.random.default_rng(random_seed)
+
+    users = df_all[user_col].unique()
+    all_items = item_features[item_col].unique()
+    rows = []
+    user_hist_items = {u: df_all[df_all[user_col] == u][item_col].unique() for u in users}
+    
+    for u in users:
+        df_user = df_all[df_all[user_col] == u]
+        pos_items = df_user[df_user[label_col] == 1][item_col].unique()
+        if len(pos_items) == 0:
+            continue
+        pos_items = pos_items[:num_pos_per_user]
+        seen_items = set(user_hist_items[u])
+        neg_pool = np.setdiff1d(all_items, np.fromiter(seen_items, dtype=all_items.dtype))
+        if len(neg_pool) == 0:
+            continue
+        for pos in pos_items:
+            if len(neg_pool) <= num_neg_per_pos:
+                neg_items = neg_pool
+            else:
+                neg_items = rng.choice(neg_pool, size=num_neg_per_pos, replace=False)
+            rows.append((u, pos, 1))
+            for ni in neg_items:
+                rows.append((u, ni, 0))
+    
+    eval_df = pd.DataFrame(rows, columns=[user_col, item_col, label_col])
+    eval_df = eval_df.merge(user_features, on=user_col, how='left')
+    eval_df = eval_df.merge(item_features, on=item_col, how='left')
+    return eval_df
+
+
+def get_user_ids(
+    data: Any,
+    id_columns: list[str] | str | None = None
+) -> np.ndarray | None:
+    """
+    Extract user IDs from various data structures.
+    
+    Args:
+        data: Data source (DataFrame, dict, or batch dict)
+        id_columns: List or single ID column name(s) (default: None)
+        
+    Returns:
+        np.ndarray | None: User IDs as numpy array, or None if not found
+    """
+    id_columns = (
+        id_columns if isinstance(id_columns, list)
+        else [id_columns] if isinstance(id_columns, str)
+        else []
+    )
+    if not id_columns:
+        return None
+
+    main_id = id_columns[0]
+    if isinstance(data, pd.DataFrame) and main_id in data.columns:
+        arr = np.asarray(data[main_id].values)
+        return arr.reshape(arr.shape[0])
+    
+    if isinstance(data, dict):
+        ids_container = data.get("ids")
+        if isinstance(ids_container, dict) and main_id in ids_container:
+            val = ids_container[main_id]
+            val = val.detach().cpu().numpy() if isinstance(val, torch.Tensor) else np.asarray(val)
+            return val.reshape(val.shape[0])
+        if main_id in data:
+            arr = np.asarray(data[main_id])
+            return arr.reshape(arr.shape[0])
+
+    return None