nextrec 0.3.4__py3-none-any.whl → 0.3.6__py3-none-any.whl

nextrec/data/data_utils.py

@@ -1,268 +1,35 @@
-"""Data processing utilities for NextRec."""
-
-import torch
-import numpy as np
-import pandas as pd
-import pyarrow.parquet as pq
-from pathlib import Path
-from typing import Any, Mapping, Sequence
-
-def stack_section(batch: list[dict], section: str):
-    """Stack one section of the batch (features/labels/ids)."""
-    entries = [item.get(section) for item in batch if item.get(section) is not None]
-    if not entries:
-        return None
-    merged: dict = {}
-    for name in entries[0]: # type: ignore
-        tensors = [item[section][name] for item in batch if item.get(section) is not None and name in item[section]]
-        merged[name] = torch.stack(tensors, dim=0)
-    return merged
-
-def collate_fn(batch):
-    """
-    Collate a list of sample dicts into the unified batch format:
-    {
-        "features": {name: Tensor(B, ...)},
-        "labels": {target: Tensor(B, ...)} or None,
-        "ids": {id_name: Tensor(B, ...)} or None,
-    }
-    """
-    if not batch:
-        return {"features": {}, "labels": None, "ids": None}
-
-    first = batch[0]
-    if isinstance(first, dict) and "features" in first:
-        # Streaming dataset yields already-batched chunks; avoid adding an extra dim.
-        if first.get("_already_batched") and len(batch) == 1:
-            return {
-                "features": first.get("features", {}),
-                "labels": first.get("labels"),
-                "ids": first.get("ids"),
-            }
-        return {
-            "features": stack_section(batch, "features") or {},
-            "labels": stack_section(batch, "labels"),
-            "ids": stack_section(batch, "ids"),
-        }
-
-    # Fallback: stack tuples/lists of tensors
-    num_tensors = len(first)
-    result = []
-    for i in range(num_tensors):
-        tensor_list = [item[i] for item in batch]
-        first_item = tensor_list[0]
-        if isinstance(first_item, torch.Tensor):
-            stacked = torch.cat(tensor_list, dim=0)
-        elif isinstance(first_item, np.ndarray):
-            stacked = np.concatenate(tensor_list, axis=0)
-        elif isinstance(first_item, list):
-            combined = []
-            for entry in tensor_list:
-                combined.extend(entry)
-            stacked = combined
-        else:
-            stacked = tensor_list
-        result.append(stacked)
-    return tuple(result)
-
-def get_column_data(data: dict | pd.DataFrame, name: str):
-    """Extract column data from various data structures."""
-    if isinstance(data, dict):
-        return data[name] if name in data else None
-    elif isinstance(data, pd.DataFrame):
-        if name not in data.columns:
-            return None
-        return data[name].values
-    else:
-        if hasattr(data, name):
-            return getattr(data, name)
-        raise KeyError(f"Unsupported data type for extracting column {name}")
-
-def resolve_file_paths(path: str) -> tuple[list[str], str]:
-    """Resolve file or directory path into a sorted list of files and file type."""
-    path_obj = Path(path)
-
-    if path_obj.is_file():
-        file_type = path_obj.suffix.lower().lstrip(".")
-        assert file_type in ["csv", "parquet"], f"Unsupported file extension: {file_type}"
-        return [str(path_obj)], file_type
-
-    if path_obj.is_dir():
-        collected_files = [p for p in path_obj.iterdir() if p.is_file()]
-        csv_files = [str(p) for p in collected_files if p.suffix.lower() == ".csv"]
-        parquet_files = [str(p) for p in collected_files if p.suffix.lower() == ".parquet"]
-
-        if csv_files and parquet_files:
-            raise ValueError("Directory contains both CSV and Parquet files. Please keep a single format.")
-        file_paths = csv_files if csv_files else parquet_files
-        if not file_paths:
-            raise ValueError(f"No CSV or Parquet files found in directory: {path}")
-        file_paths.sort()
-        file_type = "csv" if csv_files else "parquet"
-        return file_paths, file_type
-
-    raise ValueError(f"Invalid path: {path}")
-
-def iter_file_chunks(file_path: str, file_type: str, chunk_size: int):
-    """Yield DataFrame chunks for CSV/Parquet without loading the whole file."""
-    if file_type == "csv":
-        yield from pd.read_csv(file_path, chunksize=chunk_size)
-        return
-    parquet_file = pq.ParquetFile(file_path)
-    for batch in parquet_file.iter_batches(batch_size=chunk_size):
-        yield batch.to_pandas()
-
-def read_table(file_path: str, file_type: str) -> pd.DataFrame:
-    """Read a single CSV/Parquet file."""
-    if file_type == "csv":
-        return pd.read_csv(file_path)
-    return pd.read_parquet(file_path)
-
-def load_dataframes(file_paths: list[str], file_type: str) -> list[pd.DataFrame]:
-    """Load multiple files of the same type into DataFrames."""
-    return [read_table(fp, file_type) for fp in file_paths]
-
-def default_output_dir(path: str) -> Path:
-    """Generate a default output directory path based on the input path."""
-    path_obj = Path(path)
-    if path_obj.is_file():
-        return path_obj.parent / f"{path_obj.stem}_preprocessed"
-    return path_obj.with_name(f"{path_obj.name}_preprocessed")
-
-def split_dict_random(data_dict: dict, test_size: float = 0.2, random_state: int | None = None):
-    """Randomly split a dictionary of data into training and testing sets."""
-    lengths = [len(v) for v in data_dict.values()]
-    if len(set(lengths)) != 1:
-        raise ValueError(f"Length mismatch: {lengths}")
-    n = lengths[0]
-    rng = np.random.default_rng(random_state)
-    perm = rng.permutation(n)
-    cut = int(round(n * (1 - test_size)))
-    train_idx, test_idx = perm[:cut], perm[cut:]
-    def take(v, idx):
-        if isinstance(v, np.ndarray):
-            return v[idx]
-        elif isinstance(v, pd.Series):
-            return v.iloc[idx].to_numpy()  
-        else:
-            v_arr = np.asarray(v, dtype=object)  
-            return v_arr[idx]
-    train_dict = {k: take(v, train_idx) for k, v in data_dict.items()}
-    test_dict  = {k: take(v, test_idx)  for k, v in data_dict.items()}
-    return train_dict, test_dict
-
-def build_eval_candidates(
-    df_all: pd.DataFrame,
-    user_col: str,
-    item_col: str,
-    label_col: str,
-    user_features: pd.DataFrame,
-    item_features: pd.DataFrame,
-    num_pos_per_user: int = 5,
-    num_neg_per_pos: int = 50,
-    random_seed: int = 2025,
-) -> pd.DataFrame:
-    """Build evaluation candidates with positive and negative samples for each user.   """
-    rng = np.random.default_rng(random_seed)
-
-    users = df_all[user_col].unique()
-    all_items = item_features[item_col].unique()
-    rows = []
-    user_hist_items = {u: df_all[df_all[user_col] == u][item_col].unique() for u in users}
-    for u in users:
-        df_user = df_all[df_all[user_col] == u]
-        pos_items = df_user[df_user[label_col] == 1][item_col].unique()
-        if len(pos_items) == 0:
-            continue
-        pos_items = pos_items[:num_pos_per_user]
-        seen_items = set(user_hist_items[u])
-        neg_pool = np.setdiff1d(all_items, np.fromiter(seen_items, dtype=all_items.dtype))
-        if len(neg_pool) == 0:
-            continue
-        for pos in pos_items:
-            if len(neg_pool) <= num_neg_per_pos:
-                neg_items = neg_pool
-            else:
-                neg_items = rng.choice(neg_pool, size=num_neg_per_pos, replace=False)
-            rows.append((u, pos, 1))
-            for ni in neg_items:
-                rows.append((u, ni, 0))
-    eval_df = pd.DataFrame(rows, columns=[user_col, item_col, label_col])
-    eval_df = eval_df.merge(user_features, on=user_col, how='left')
-    eval_df = eval_df.merge(item_features, on=item_col, how='left')
-    return eval_df
-
-def batch_to_dict(batch_data: Any, include_ids: bool = True) -> dict:
-    """Standardize a dataloader batch into a dict of features, labels, and ids."""
-    if not (isinstance(batch_data, Mapping) and "features" in batch_data):
-        raise TypeError(
-            "[BaseModel-batch_to_dict Error] Batch data must be a dict with 'features' produced by the current DataLoader."
-        )
-    return {
-        "features": batch_data.get("features", {}),
-        "labels": batch_data.get("labels"),
-        "ids": batch_data.get("ids") if include_ids else None,
-    }
-
-
-# def get_user_ids(
-#     data: dict | pd.DataFrame | None, user_id_column: str = "user_id"
-# ) -> np.ndarray | None:
-#     """Extract user IDs from a dataset dict or DataFrame."""
-#     if data is None:
-#         return None
-#     if isinstance(data, pd.DataFrame) and user_id_column in data.columns:
-#         return np.asarray(data[user_id_column].values)
-#     if isinstance(data, dict) and user_id_column in data:
-#         return np.asarray(data[user_id_column])
-#     return None
-
-
-# def get_user_ids_from_batch(
-#     batch_dict: Mapping[str, Any], id_columns: Sequence[str] | None = None
-# ) -> np.ndarray | None:
-#     """Extract the prioritized user id column from a batch dict."""
-#     ids_container = batch_dict.get("ids") if isinstance(batch_dict, Mapping) else None
-#     if not ids_container:
-#         return None
-
-#     batch_user_id = None
-#     if id_columns:
-#         for id_name in id_columns:
-#             if id_name in ids_container:
-#                 batch_user_id = ids_container[id_name]
-#                 break
-#     if batch_user_id is None:
-#         batch_user_id = next(iter(ids_container.values()), None)
-#     if batch_user_id is None:
-#         return None
-
-#     if isinstance(batch_user_id, torch.Tensor):
-#         ids_np = batch_user_id.detach().cpu().numpy()
-#     else:
-#         ids_np = np.asarray(batch_user_id)
-#     if ids_np.ndim == 0:
-#         ids_np = ids_np.reshape(1)
-#     return ids_np.reshape(ids_np.shape[0])
-
-
-def get_user_ids(data, id_columns: list[str] | str | None = None) -> np.ndarray | None:
-    id_columns = id_columns if isinstance(id_columns, list) else [id_columns] if isinstance(id_columns, str) else []
-    if not id_columns:
-        return None
-
-    main_id = id_columns[0]  
-    if isinstance(data, pd.DataFrame) and main_id in data.columns:
-        arr = np.asarray(data[main_id].values)
-        return arr.reshape(arr.shape[0])
-    if isinstance(data, dict):
-        ids_container = data.get("ids")
-        if isinstance(ids_container, dict) and main_id in ids_container:
-            val = ids_container[main_id]
-            val = val.detach().cpu().numpy() if isinstance(val, torch.Tensor) else np.asarray(val)
-            return val.reshape(val.shape[0])
-        if main_id in data:
-            arr = np.asarray(data[main_id])
-            return arr.reshape(arr.shape[0])
-
-    return None
+"""
+Data processing utilities for NextRec (Refactored)
+
+This module now re-exports functions from specialized submodules:
+- batch_utils: collate_fn, batch_to_dict
+- data_processing: get_column_data, split_dict_random, build_eval_candidates, get_user_ids
+- nextrec.utils.file_utils: resolve_file_paths, iter_file_chunks, read_table, load_dataframes, default_output_dir
+
+Date: create on 27/10/2025
+Last update: 03/12/2025 (refactored)
+Author: Yang Zhou, zyaztec@gmail.com
+"""
+
+# Import from new organized modules
+from nextrec.data.batch_utils import collate_fn, batch_to_dict, stack_section
+from nextrec.data.data_processing import get_column_data, split_dict_random, build_eval_candidates, get_user_ids
+from nextrec.utils.file import resolve_file_paths, iter_file_chunks, read_table, load_dataframes, default_output_dir
+
+__all__ = [
+    # Batch utilities
+    'collate_fn',
+    'batch_to_dict',
+    'stack_section',
+    # Data processing
+    'get_column_data',
+    'split_dict_random',
+    'build_eval_candidates',
+    'get_user_ids',
+    # File utilities
+    'resolve_file_paths',
+    'iter_file_chunks',
+    'read_table',
+    'load_dataframes',
+    'default_output_dir',
+]

nextrec/data/dataloader.py

@@ -20,8 +20,10 @@ from nextrec.data.preprocessor import DataProcessor
 from nextrec.basic.features import DenseFeature, SparseFeature, SequenceFeature, FeatureSet
 
 from nextrec.basic.loggers import colorize
-from nextrec.data import get_column_data, collate_fn, resolve_file_paths, read_table
-from nextrec.utils import to_tensor
+from nextrec.data.data_processing import get_column_data
+from nextrec.data.batch_utils import collate_fn
+from nextrec.utils.file import resolve_file_paths, read_table
+from nextrec.utils.tensor import to_tensor
 
 class TensorDictDataset(Dataset):
     """Dataset returning sample-level dicts matching the unified batch schema."""
@@ -124,20 +126,22 @@ class RecDataLoader(FeatureSet):
                          batch_size: int = 32,
                          shuffle: bool = True,
                          load_full: bool = True,
-                         chunk_size: int = 10000) -> DataLoader:
+                         chunk_size: int = 10000,
+                         num_workers: int = 0) -> DataLoader:
         if isinstance(data, DataLoader):
             return data
         elif isinstance(data, (str, os.PathLike)):
-            return self.create_from_path(path=data, batch_size=batch_size, shuffle=shuffle, load_full=load_full, chunk_size=chunk_size)
+            return self.create_from_path(path=data, batch_size=batch_size, shuffle=shuffle, load_full=load_full, chunk_size=chunk_size, num_workers=num_workers)
         elif isinstance(data, (dict, pd.DataFrame)):
-            return self.create_from_memory(data=data, batch_size=batch_size, shuffle=shuffle)
+            return self.create_from_memory(data=data, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers)
         else:
             raise ValueError(f"[RecDataLoader Error] Unsupported data type: {type(data)}")
     
     def create_from_memory(self, 
                            data: dict | pd.DataFrame,
                            batch_size: int,
-                           shuffle: bool) -> DataLoader:
+                           shuffle: bool,
+                           num_workers: int = 0) -> DataLoader:
         raw_data = data
 
         if self.processor is not None:
@@ -148,14 +152,15 @@ class RecDataLoader(FeatureSet):
         if tensors is None:
             raise ValueError("[RecDataLoader Error] No valid tensors could be built from the provided data.")
         dataset = TensorDictDataset(tensors)
-        return DataLoader(dataset, batch_size=batch_size, shuffle=shuffle, collate_fn=collate_fn)
+        return DataLoader(dataset, batch_size=batch_size, shuffle=shuffle, collate_fn=collate_fn, num_workers=num_workers)
     
     def create_from_path(self,
                          path: str,
                          batch_size: int,
                          shuffle: bool,
                          load_full: bool,
-                         chunk_size: int = 10000) -> DataLoader:
+                         chunk_size: int = 10000,
+                         num_workers: int = 0) -> DataLoader:
         file_paths, file_type = resolve_file_paths(str(Path(path)))
         # Load full data into memory
         if load_full:
@@ -167,6 +172,7 @@ class RecDataLoader(FeatureSet):
                 except OSError:
                     pass
                 try:
+                    df = read_table(file_path, file_type=file_type)
                     dfs.append(df)
                 except MemoryError as exc:
                     raise MemoryError(f"[RecDataLoader Error] Out of memory while reading {file_path}. Consider using load_full=False with streaming.") from exc
@@ -174,22 +180,23 @@ class RecDataLoader(FeatureSet):
                 combined_df = pd.concat(dfs, ignore_index=True)
             except MemoryError as exc:
                 raise MemoryError(f"[RecDataLoader Error] Out of memory while concatenating loaded data (approx {total_bytes / (1024**3):.2f} GB). Use load_full=False to stream or reduce chunk_size.") from exc
-            return self.create_from_memory(combined_df, batch_size, shuffle,)
+            return self.create_from_memory(combined_df, batch_size, shuffle, num_workers=num_workers)
         else:
-            return self.load_files_streaming(file_paths, file_type, batch_size, chunk_size, shuffle)
+            return self.load_files_streaming(file_paths, file_type, batch_size, chunk_size, shuffle, num_workers=num_workers)
 
     def load_files_streaming(self,
                              file_paths: list[str],
                              file_type: str,
                              batch_size: int,
                              chunk_size: int,
-                             shuffle: bool) -> DataLoader:
+                             shuffle: bool,
+                             num_workers: int = 0) -> DataLoader:
         if shuffle:
             logging.info("[RecDataLoader Info] Shuffle is ignored in streaming mode (IterableDataset).")
         if batch_size != 1:
             logging.info("[RecDataLoader Info] Streaming mode enforces batch_size=1; tune chunk_size to control memory/throughput.")
         dataset = FileDataset(file_paths=file_paths, dense_features=self.dense_features, sparse_features=self.sparse_features, sequence_features=self.sequence_features, target_columns=self.target_columns, id_columns=self.id_columns, chunk_size=chunk_size, file_type=file_type, processor=self.processor)
-        return DataLoader(dataset, batch_size=1, collate_fn=collate_fn)
+        return DataLoader(dataset, batch_size=1, collate_fn=collate_fn, num_workers=num_workers)
 
 def normalize_sequence_column(column, feature: SequenceFeature) -> np.ndarray:
     if isinstance(column, pd.Series):

nextrec/data/preprocessor.py

@@ -16,24 +16,14 @@ import pandas as pd
 import tqdm
 from pathlib import Path
 from typing import Dict, Union, Optional, Literal, Any
-from sklearn.preprocessing import (
-    StandardScaler, 
-    MinMaxScaler, 
-    RobustScaler, 
-    MaxAbsScaler,
-    LabelEncoder
-)
+from sklearn.preprocessing import StandardScaler, MinMaxScaler, RobustScaler, MaxAbsScaler, LabelEncoder
+
 
-from nextrec.basic.loggers import setup_logger, colorize
-from nextrec.data.data_utils import (
-    resolve_file_paths,
-    iter_file_chunks,
-    read_table,
-    load_dataframes,
-    default_output_dir,
-)
-from nextrec.basic.session import resolve_save_path
 from nextrec.basic.features import FeatureSet
+from nextrec.basic.loggers import colorize
+from nextrec.basic.session import resolve_save_path
+from nextrec.utils.file import resolve_file_paths, iter_file_chunks, read_table, load_dataframes, default_output_dir
+
 from nextrec.__version__ import __version__
 
 

nextrec/models/generative/__init__.py

@@ -1,5 +0,0 @@
-from .hstu import HSTU
-
-__all__ = [
-    "HSTU",
-]

nextrec/models/match/__init__.py

@@ -1,13 +0,0 @@
-from .dssm import DSSM
-from .dssm_v2 import DSSM_v2
-from .youtube_dnn import YoutubeDNN
-from .mind import MIND
-from .sdm import SDM
-
-__all__ = [
-    'DSSM',
-    'DSSM_v2',
-    'YoutubeDNN',
-    'MIND',
-    'SDM',
-]

nextrec/models/multi_task/poso.py

@@ -46,7 +46,7 @@ from nextrec.basic.features import DenseFeature, SequenceFeature, SparseFeature
 from nextrec.basic.layers import EmbeddingLayer, MLP, PredictionLayer
 from nextrec.basic.activation import activation_layer
 from nextrec.basic.model import BaseModel
-from nextrec.utils.common import merge_features
+from nextrec.utils.model import merge_features
 
 
 class POSOGate(nn.Module):

nextrec/models/ranking/__init__.py

@@ -1,27 +0,0 @@
-from .fm import FM
-from .afm import AFM
-from .masknet import MaskNet
-from .pnn import PNN
-from .deepfm import DeepFM
-from .autoint import AutoInt
-from .widedeep import WideDeep
-from .xdeepfm import xDeepFM
-from .dcn import DCN
-from .fibinet import FiBiNET
-from .din import DIN
-from .dien import DIEN
-
-__all__ = [
-    'DeepFM',
-    'AutoInt', 
-    'WideDeep',
-    'xDeepFM',
-    'DCN',
-    'DIN',
-    'DIEN',
-    'FM',
-    'AFM',
-    'MaskNet',
-    'PNN',
-    'FiBiNET',
-]

nextrec/utils/__init__.py

@@ -1,18 +1,68 @@
+"""
+Utilities package for NextRec
+
+This package provides various utility functions organized by category:
+- optimizer: Optimizer and scheduler utilities
+- initializer: Weight initialization utilities
+- embedding: Embedding dimension calculation
+- device_utils: Device management and selection
+- tensor_utils: Tensor operations and conversions
+- file_utils: File I/O operations
+- model_utils: Model-related utilities
+- feature_utils: Feature processing utilities
+
+Date: create on 13/11/2025
+Last update: 03/12/2025 (refactored)
+Author: Yang Zhou, zyaztec@gmail.com
+"""
+
 from .optimizer import get_optimizer, get_scheduler
 from .initializer import get_initializer
 from .embedding import get_auto_embedding_dim
-from .common import resolve_device, to_tensor
-from . import optimizer, initializer, embedding, common
+from .device import resolve_device, get_device_info
+from .tensor import to_tensor, stack_tensors, concat_tensors, pad_sequence_tensors
+from .file import resolve_file_paths, read_table, load_dataframes, iter_file_chunks, default_output_dir
+from .model import merge_features, get_mlp_output_dim
+from .feature import normalize_to_list
+from . import optimizer, initializer, embedding
 
 __all__ = [
+    # Optimizer & Scheduler
     'get_optimizer',
     'get_scheduler',
+    
+    # Initializer
     'get_initializer',
+    
+    # Embedding
     'get_auto_embedding_dim',
+    
+    # Device utilities
     'resolve_device',
+    'get_device_info',
+    
+    # Tensor utilities
     'to_tensor',
+    'stack_tensors',
+    'concat_tensors',
+    'pad_sequence_tensors',
+    
+    # File utilities
+    'resolve_file_paths',
+    'read_table',
+    'load_dataframes',
+    'iter_file_chunks',
+    'default_output_dir',
+    
+    # Model utilities
+    'merge_features',
+    'get_mlp_output_dim',
+    
+    # Feature utilities
+    'normalize_to_list',
+    
+    # Module exports
     'optimizer',
     'initializer',
     'embedding',
-    'common',
 ]

nextrec/utils/device.py

@@ -0,0 +1,38 @@
+"""
+Device management utilities for NextRec
+
+Date: create on 03/12/2025
+Author: Yang Zhou, zyaztec@gmail.com
+"""
+import os
+import torch
+import platform
+import multiprocessing
+
+
+def resolve_device() -> str:
+    if torch.cuda.is_available():
+        return "cuda"
+    if torch.backends.mps.is_available():
+        mac_ver = platform.mac_ver()[0]
+        try:
+            major, minor = (int(x) for x in mac_ver.split(".")[:2])
+        except Exception:
+            major, minor = 0, 0
+        if major >= 14:
+            return "mps"
+    return "cpu"
+
+def get_device_info() -> dict:
+    info = {
+        'cuda_available': torch.cuda.is_available(),
+        'cuda_device_count': torch.cuda.device_count() if torch.cuda.is_available() else 0,
+        'mps_available': torch.backends.mps.is_available(),
+        'current_device': resolve_device(),
+    }
+    
+    if torch.cuda.is_available():
+        info['cuda_device_name'] = torch.cuda.get_device_name(0)
+        info['cuda_capability'] = torch.cuda.get_device_capability(0)
+    
+    return info

nextrec/utils/feature.py

@@ -0,0 +1,13 @@
+"""
+Feature processing utilities for NextRec
+
+Date: create on 03/12/2025
+Author: Yang Zhou, zyaztec@gmail.com
+"""
+
+def normalize_to_list(value: str | list[str] | None) -> list[str]:
+    if value is None:
+        return []
+    if isinstance(value, str):
+        return [value]
+    return list(value)