nextrec 0.1.11__py3-none-any.whl → 0.2.2__py3-none-any.whl

nextrec-0.2.2.dist-info/RECORD

@@ -0,0 +1,53 @@
+nextrec/__init__.py,sha256=CvocnY2uBp0cjNkhrT6ogw0q2bN9s1GNp754FLO-7lo,1117
+nextrec/__version__.py,sha256=m6kyaNpwBcP1XYcqrelX2oS3PJuOnElOcRdBa9pEb8c,22
+nextrec/basic/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+nextrec/basic/activation.py,sha256=9EfYmwE0brTSKwx_0FIGQ_rybFBT9n_G-UWA7NAhMsI,2804
+nextrec/basic/callback.py,sha256=qkq3k8rP0g4BW2C3FSCdVt_CyCcJwJ-rUXjhT2p4LP8,1035
+nextrec/basic/features.py,sha256=qNrGm74R6K7dw-kJdA2Sbp_Tjb_oDOo1a2JCfyOaYtw,3957
+nextrec/basic/layers.py,sha256=mDNApSlPkmPSnIPj3BDHfDEjviLybWuSGrh61Zog2uk,38290
+nextrec/basic/loggers.py,sha256=x8lzyyK-uqBN5XGOm1Cb33dmfc2bl114n6QeFTtE54k,3752
+nextrec/basic/metrics.py,sha256=w8tGe2tTbBNz9A1TNZF3jSpxcNC6QvFP5I0lWRd0Nw4,20398
+nextrec/basic/model.py,sha256=r3yWw2RMk-_Ap9hXKzyp7-TKpSaPtqRzpABaJkCrT9M,66336
+nextrec/basic/session.py,sha256=2kogEjgKAN1_ygelbwoqOs187BAcUnDTqXG1w_Pgb9I,4791
+nextrec/data/__init__.py,sha256=SOD64AkPykwKgm0CT89aDcHeiiDQlOOAkLDsAjYiqAM,814
+nextrec/data/data_utils.py,sha256=vGZ378YM_JQXO9npRB7JqojJx1ovjbJCWI-7lQJkicA,6298
+nextrec/data/dataloader.py,sha256=T0i2f5KMohd-hkgTPBNCPy9xaziRrUwYd01oI8GIEI8,20450
+nextrec/data/preprocessor.py,sha256=LzuBeBHcp6nnm7oaHDV-SZNk92ev8xjCj4r-thakRMw,42288
+nextrec/loss/__init__.py,sha256=t-wkqxcu5wdYlrb67-CxX9aOGom0CpMJK8Fe8KGDSEE,857
+nextrec/loss/listwise.py,sha256=LcYIPf6PGRtjV_AoWaAyp3rse904S2MghE5t032I07I,5628
+nextrec/loss/loss_utils.py,sha256=LnTkpMTS2bhbq4Lsjf3AUn1uBaOg1TaH5VO2R8hwARc,5324
+nextrec/loss/pairwise.py,sha256=RuQuTE-EkLaHQvT9m0CTAXxneTnVQLF1Pi9wblEClI8,3289
+nextrec/loss/pointwise.py,sha256=6QveizdohzQTxAoBKTVSoCBpp-fy3JC8vCjImXa7jL0,7157
+nextrec/models/generative/hstu.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+nextrec/models/generative/tiger.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+nextrec/models/match/__init__.py,sha256=ASZB5abqKPhDbk8NErNNNa0DHuWpsVxvUtyEn5XMx6Y,215
+nextrec/models/match/dssm.py,sha256=e0hUqNLJVwTRVz4F4EiO8KLOOprKRBDtI4ID6Y1Tc60,8232
+nextrec/models/match/dssm_v2.py,sha256=ywtqTy3YN9ke_7kzcDp7Fhtldw9RJz6yfewxALJb6Z0,7189
+nextrec/models/match/mind.py,sha256=XSUDlZ-V95JXHHBDUl5sz99SaVuQKDvf3TArVjwUexs,9417
+nextrec/models/match/sdm.py,sha256=96yfMQ6arP6JRhAkDTGEjlBiTteznMykrDV_3jqvvVk,10920
+nextrec/models/match/youtube_dnn.py,sha256=pnrz9LYu65Fj4neOriFF45B5k2-yYiiREtQICxxYXZ0,7546
+nextrec/models/multi_task/esmm.py,sha256=E9B6TlpnPUeyldTofyFg4B7SKByyxbiW2fUGHLOryO4,4883
+nextrec/models/multi_task/mmoe.py,sha256=zhQr43Vfz7Kgi6B9pKPmaenp_38a_D7w4VvlpwCyF6Y,6165
+nextrec/models/multi_task/ple.py,sha256=otP6oLgzrJhwkLFItzNE-AtIPouObDkafRvWzTCxfNo,11335
+nextrec/models/multi_task/share_bottom.py,sha256=LL5HBVlvvBzHV2fLBRQMGIwpqmlxILTgU4c51XyTCo4,4517
+nextrec/models/ranking/__init__.py,sha256=-qe34zQEVwmxeTPGYCa6gbql9quT8DwB7-buHfA7Iig,428
+nextrec/models/ranking/afm.py,sha256=r9m1nEnc0m5d4pMtOxRMqOaXaBNCEkjJBFB-5wSHeFA,4540
+nextrec/models/ranking/autoint.py,sha256=GYzRynjn6Csq4b3qYIFWxLQ4Yl57_OQBeF2IY0Zhr9Q,5654
+nextrec/models/ranking/dcn.py,sha256=dUV5GbHypBGc9vVozk6aGYfIXq23c0deX-HFnIhZueg,4208
+nextrec/models/ranking/deepfm.py,sha256=y28yJxF__TZR3O1G2ufKZVtBRLgCgmlXWqvPgLzwm3U,3510
+nextrec/models/ranking/dien.py,sha256=E6s9TDwQfGSwtzzh8hG2F5gwgVxzVZPcptYvHLNzOLA,8475
+nextrec/models/ranking/din.py,sha256=j5tkT5k91CbsMlMr5vJOySrcY2_rFGxmEgJJ0McW7-Q,7196
+nextrec/models/ranking/fibinet.py,sha256=X6CbQbritvq5jql_Tvs4bn_tRla2zpWPplftZv8k6f0,4853
+nextrec/models/ranking/fm.py,sha256=3Qx_Fgowegr6UPQtEeTmHtOrbWzkvqH94ZTjOqRLu-E,2961
+nextrec/models/ranking/masknet.py,sha256=Bu0mZl2vKqcGnqCuUjPHjPRd1f-cDTeVwFj8Y_6v3C8,4639
+nextrec/models/ranking/pnn.py,sha256=5RxIKdxD0XcGq-b_QDdwGRwk6b_5BQjyMvCw3Ibv2Kk,4957
+nextrec/models/ranking/widedeep.py,sha256=b6ctElaZPv5WSYDA4piYUBo3je0eJpWpWECwcuWavM4,3716
+nextrec/models/ranking/xdeepfm.py,sha256=I00J5tfE4tPluqeW-qrNtE4V_9fC7-rgFvA0Fxqka7o,4274
+nextrec/utils/__init__.py,sha256=6x3OZbqks2gtgJd00y_-Y8QiAT42x5t14ARHQ-ULQDo,350
+nextrec/utils/embedding.py,sha256=yxYSdFx0cJITh3Gf-K4SdhwRtKGcI0jOsyBgZ0NLa_c,465
+nextrec/utils/initializer.py,sha256=ffYOs5QuIns_d_-5e40iNtg6s1ftgREJN-ueq_NbDQE,1647
+nextrec/utils/optimizer.py,sha256=85ifoy2IQgjPHOqLqr1ho7XBGE_0ry1yEB9efS6C2lM,2446
+nextrec-0.2.2.dist-info/METADATA,sha256=lAhtUMV17TjgvjOuKZph_it7O02Ld1gKA0mc0k44oNQ,11425
+nextrec-0.2.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+nextrec-0.2.2.dist-info/licenses/LICENSE,sha256=2fQfVKeafywkni7MYHyClC6RGGC3laLTXCNBx-ubtp0,1064
+nextrec-0.2.2.dist-info/RECORD,,

nextrec/basic/dataloader.py

@@ -1,447 +0,0 @@
-"""
-Dataloader definitions
-
-Date: create on 27/10/2025
-Author:
-    Yang Zhou,zyaztec@gmail.com
-"""
-
-import os
-import glob
-import torch
-import logging
-import numpy as np
-import pandas as pd
-import tqdm
-
-from pathlib import Path
-from typing import Iterator, Literal, Union, Optional
-from torch.utils.data import DataLoader, TensorDataset, IterableDataset
-
-from nextrec.data.preprocessor import DataProcessor
-from nextrec.data import get_column_data, collate_fn
-
-from nextrec.basic.features import DenseFeature, SparseFeature, SequenceFeature
-from nextrec.basic.loggers import colorize
-
-
-class FileDataset(IterableDataset):
-    """
-    Iterable dataset for reading multiple files in batches.
-    Supports CSV and Parquet files with chunk-based reading.
-    """
-    
-    def __init__(self, 
-                 file_paths: list[str],                      # file paths to read, containing CSV or Parquet files
-                 dense_features: list[DenseFeature],         # dense feature definitions
-                 sparse_features: list[SparseFeature],       # sparse feature definitions
-                 sequence_features: list[SequenceFeature],   # sequence feature definitions
-                 target_columns: list[str],                   # target column names
-                 chunk_size: int = 10000,
-                 file_type: Literal['csv', 'parquet'] = 'csv',
-                 processor: Optional['DataProcessor'] = None): # optional DataProcessor for transformation
-
-        self.file_paths = file_paths
-        self.dense_features = dense_features
-        self.sparse_features = sparse_features
-        self.sequence_features = sequence_features
-        self.target_columns = target_columns
-        self.chunk_size = chunk_size
-        self.file_type = file_type
-        self.processor = processor
-        
-        self.all_features = dense_features + sparse_features + sequence_features
-        self.feature_names = [f.name for f in self.all_features]
-        self.current_file_index = 0
-        self.total_files = len(file_paths)
-    
-    def __iter__(self) -> Iterator[tuple]:
-        self.current_file_index = 0
-        self._file_pbar = None
-        
-        # Create progress bar for file processing when multiple files
-        if self.total_files > 1:
-            self._file_pbar = tqdm.tqdm(
-                total=self.total_files, 
-                desc="Files", 
-                unit="file",
-                position=0,
-                leave=True,
-                bar_format='{desc}: {percentage:3.0f}%|{bar}| {n_fmt}/{total_fmt} [{elapsed}<{remaining}]'
-            )
-        
-        for file_path in self.file_paths:
-            self.current_file_index += 1
-            
-            # Update file progress bar
-            if self._file_pbar is not None:
-                self._file_pbar.update(1)
-            elif self.total_files == 1:
-                # For single file, log the file name
-                file_name = os.path.basename(file_path)
-                logging.info(colorize(f"Processing file: {file_name}", color="cyan"))
-            
-            if self.file_type == 'csv':
-                yield from self._read_csv_chunks(file_path)
-            elif self.file_type == 'parquet':
-                yield from self._read_parquet_chunks(file_path)
-        
-        # Close file progress bar
-        if self._file_pbar is not None:
-            self._file_pbar.close()
-    
-    def _read_csv_chunks(self, file_path: str) -> Iterator[tuple]:
-        chunk_iterator = pd.read_csv(file_path, chunksize=self.chunk_size)
-        
-        for chunk in chunk_iterator:
-            tensors = self._dataframe_to_tensors(chunk)
-            if tensors:
-                yield tensors
-    
-    def _read_parquet_chunks(self, file_path: str) -> Iterator[tuple]:
-        """
-        Read parquet file in chunks to reduce memory footprint.
-        Uses pyarrow's batch reading for true streaming.
-        """
-        import pyarrow.parquet as pq
-        parquet_file = pq.ParquetFile(file_path)
-        for batch in parquet_file.iter_batches(batch_size=self.chunk_size):
-            chunk = batch.to_pandas()            
-            tensors = self._dataframe_to_tensors(chunk)
-            if tensors:
-                yield tensors
-            del chunk
-                
-    def _dataframe_to_tensors(self, df: pd.DataFrame) -> tuple | None:
-        if self.processor is not None:
-            if not self.processor.is_fitted:
-                raise ValueError("DataProcessor must be fitted before using in streaming mode")
-            transformed_data = self.processor.transform(df, return_dict=True)
-        else:
-            transformed_data = df
-        
-        tensors = []
-        
-        # Process features
-        for feature in self.all_features:
-            if self.processor is not None:
-                column_data = transformed_data.get(feature.name)
-                if column_data is None:
-                    continue
-            else:
-                # Get data from original dataframe
-                if feature.name not in df.columns:
-                    logging.warning(colorize(f"Feature column '{feature.name}' not found in DataFrame", "yellow"))
-                    continue
-                column_data = df[feature.name].values
-            
-            # Handle sequence features: convert to 2D array of shape (batch_size, seq_length)
-            if isinstance(feature, SequenceFeature):
-                if isinstance(column_data, np.ndarray) and column_data.dtype == object:
-                    try:
-                        column_data = np.stack([np.asarray(seq, dtype=np.int64) for seq in column_data])  # type: ignore
-                    except (ValueError, TypeError) as e:
-                        # Fallback: handle variable-length sequences by padding
-                        sequences = []
-                        max_len = feature.max_len if hasattr(feature, 'max_len') else 0
-                        for seq in column_data:
-                            if isinstance(seq, (list, tuple, np.ndarray)):
-                                seq_arr = np.asarray(seq, dtype=np.int64)
-                            else:
-                                seq_arr = np.array([], dtype=np.int64)
-                            sequences.append(seq_arr)
-                        
-                        # Pad sequences to same length
-                        if max_len == 0:
-                            max_len = max(len(seq) for seq in sequences) if sequences else 1
-                        
-                        padded = []
-                        for seq in sequences:
-                            if len(seq) > max_len:
-                                padded.append(seq[:max_len])
-                            else:
-                                pad_width = max_len - len(seq)
-                                padded.append(np.pad(seq, (0, pad_width), constant_values=0))
-                        column_data = np.stack(padded)
-                else:
-                    column_data = np.asarray(column_data, dtype=np.int64)
-                tensor = torch.from_numpy(column_data)
-            elif isinstance(feature, DenseFeature):
-                tensor = torch.from_numpy(np.asarray(column_data, dtype=np.float32))
-            else:  # SparseFeature
-                tensor = torch.from_numpy(np.asarray(column_data, dtype=np.int64))
-            
-            tensors.append(tensor)
-        
-        # Process targets
-        target_tensors = []
-        for target_name in self.target_columns:
-            if self.processor is not None:
-                target_data = transformed_data.get(target_name)
-                if target_data is None: 
-                    continue
-            else:
-                if target_name not in df.columns:
-                    continue
-                target_data = df[target_name].values
-            
-            target_tensor = torch.from_numpy(np.asarray(target_data, dtype=np.float32))
-            
-            if target_tensor.dim() == 1:
-                target_tensor = target_tensor.view(-1, 1)
-            
-            target_tensors.append(target_tensor)
-        
-        # Combine target tensors
-        if target_tensors:
-            if len(target_tensors) == 1 and target_tensors[0].shape[1] > 1:
-                y_tensor = target_tensors[0]
-            else:
-                y_tensor = torch.cat(target_tensors, dim=1)
-            
-            if y_tensor.shape[1] == 1:
-                y_tensor = y_tensor.squeeze(1)
-            
-            tensors.append(y_tensor)
-        
-        if not tensors:
-            return None
-        
-        return tuple(tensors)
-
-
-class RecDataLoader:
-    """
-    Custom DataLoader for recommendation models.
-    Supports multiple input formats: dict, DataFrame, CSV files, Parquet files, and directories.
-    Optionally supports DataProcessor for on-the-fly data transformation.
-
-    Examples:
-        >>> # 创建RecDataLoader
-        >>> dataloader = RecDataLoader(
-        >>>     dense_features=dense_features,
-        >>>     sparse_features=sparse_features,
-        >>>     sequence_features=sequence_features,
-        >>>     target_columns=target_columns,
-        >>>     processor=processor
-        >>> )
-    """
-    
-    def __init__(self,
-                 dense_features: list[DenseFeature] | None = None,
-                 sparse_features: list[SparseFeature] | None = None,
-                 sequence_features: list[SequenceFeature] | None = None,
-                 target: list[str] | None | str = None,
-                 processor: Optional['DataProcessor'] = None):
-
-        self.dense_features = dense_features if dense_features else []
-        self.sparse_features = sparse_features if sparse_features else []
-        self.sequence_features = sequence_features if sequence_features else []
-        if isinstance(target, str):
-            self.target_columns = [target]
-        elif isinstance(target, list):
-            self.target_columns = target
-        else:
-            self.target_columns = []
-        self.processor = processor
-        
-        self.all_features = self.dense_features + self.sparse_features + self.sequence_features
-    
-    def create_dataloader(self,
-                         data: Union[dict, pd.DataFrame, str, DataLoader],
-                         batch_size: int = 32,
-                         shuffle: bool = True,
-                         load_full: bool = True,
-                         chunk_size: int = 10000) -> DataLoader:
-        """
-        Create DataLoader from various data sources.
-        """
-        if isinstance(data, DataLoader):
-            return data
-        
-        if isinstance(data, (str, os.PathLike)):
-            return self._create_from_path(data, batch_size, shuffle, load_full, chunk_size)
-        
-        if isinstance(data, (dict, pd.DataFrame)):
-            return self._create_from_memory(data, batch_size, shuffle)
-
-        raise ValueError(f"Unsupported data type: {type(data)}")
-    
-    def _create_from_memory(self, 
-                           data: Union[dict, pd.DataFrame],
-                           batch_size: int,
-                           shuffle: bool) -> DataLoader:
-
-        if self.processor is not None:
-            if not self.processor.is_fitted:
-                raise ValueError("DataProcessor must be fitted before using in RecDataLoader")
-            data = self.processor.transform(data, return_dict=True)
-        
-        tensors = []
-        
-        # Process features
-        for feature in self.all_features:
-            column = get_column_data(data, feature.name)
-            if column is None:
-                raise KeyError(f"Feature {feature.name} not found in provided data.")
-
-            if isinstance(feature, SequenceFeature):
-                if isinstance(column, pd.Series):
-                    column = column.values
-                
-                # Handle different input formats for sequence features
-                if isinstance(column, np.ndarray):
-                    # Check if elements are actually sequences (not just object dtype scalars)
-                    if column.dtype == object and len(column) > 0 and isinstance(column[0], (list, tuple, np.ndarray)):
-                        # Each element is a sequence (array/list), stack them into 2D array
-                        try:
-                            column = np.stack([np.asarray(seq, dtype=np.int64) for seq in column])  # type: ignore
-                        except (ValueError, TypeError) as e:
-                            # Fallback: handle variable-length sequences by padding
-                            sequences = []
-                            max_len = feature.max_len if hasattr(feature, 'max_len') else 0
-                            for seq in column:
-                                if isinstance(seq, (list, tuple, np.ndarray)):
-                                    seq_arr = np.asarray(seq, dtype=np.int64)
-                                else:
-                                    seq_arr = np.array([], dtype=np.int64)
-                                sequences.append(seq_arr)
-                            
-                            # Pad sequences to same length
-                            if max_len == 0:
-                                max_len = max(len(seq) for seq in sequences) if sequences else 1
-                            
-                            padded = []
-                            for seq in sequences:
-                                if len(seq) > max_len:
-                                    padded.append(seq[:max_len])
-                                else:
-                                    pad_width = max_len - len(seq)
-                                    padded.append(np.pad(seq, (0, pad_width), constant_values=0))
-                            column = np.stack(padded)
-                    elif column.ndim == 1:
-                        # 1D array, need to reshape or handle appropriately
-                        # Assuming each element should be treated as a single-item sequence
-                        column = column.reshape(-1, 1)
-                    # else: already a 2D array
-                
-                column = np.asarray(column, dtype=np.int64)
-                tensor = torch.from_numpy(column)
-                
-            elif isinstance(feature, DenseFeature):
-                tensor = torch.from_numpy(np.asarray(column, dtype=np.float32))
-            else:  # SparseFeature
-                tensor = torch.from_numpy(np.asarray(column, dtype=np.int64))
-            
-            tensors.append(tensor)
-        
-        # Process targets
-        label_tensors = []
-        for target_name in self.target_columns:
-            column = get_column_data(data, target_name)
-            if column is None:
-                continue
-            
-            label_tensor = torch.from_numpy(np.asarray(column, dtype=np.float32))
-            
-            if label_tensor.dim() == 1:
-                label_tensor = label_tensor.view(-1, 1)
-            elif label_tensor.dim() == 2:
-                if label_tensor.shape[0] == 1 and label_tensor.shape[1] > 1:
-                    label_tensor = label_tensor.t()
-            
-            label_tensors.append(label_tensor)
-        
-        # Combine target tensors
-        if label_tensors:
-            if len(label_tensors) == 1 and label_tensors[0].shape[1] > 1:
-                y_tensor = label_tensors[0]
-            else:
-                y_tensor = torch.cat(label_tensors, dim=1)
-            
-            if y_tensor.shape[1] == 1:
-                y_tensor = y_tensor.squeeze(1)
-            
-            tensors.append(y_tensor)
-        
-        dataset = TensorDataset(*tensors)
-        return DataLoader(dataset, batch_size=batch_size, shuffle=shuffle)
-    
-    def _create_from_path(self,
-                         path: str,
-                         batch_size: int,
-                         shuffle: bool,
-                         load_full: bool,
-                         chunk_size: int) -> DataLoader:
-        """
-        Create DataLoader from a file path, supporting CSV and Parquet formats, with options for full loading or streaming.
-        """
-
-        path_obj = Path(path)
-        
-        # Determine if it's a file or directory
-        if path_obj.is_file():
-            file_paths = [str(path_obj)]
-            file_type = self._get_file_type(str(path_obj))
-        elif path_obj.is_dir():
-            # Find all CSV and Parquet files in directory
-            csv_files = glob.glob(os.path.join(path, "*.csv"))
-            parquet_files = glob.glob(os.path.join(path, "*.parquet"))
-            
-            if csv_files and parquet_files:
-                raise ValueError("Directory contains both CSV and Parquet files. Please use a single format.")
-            
-            file_paths = csv_files if csv_files else parquet_files
-            
-            if not file_paths:
-                raise ValueError(f"No CSV or Parquet files found in directory: {path}")
-            
-            file_type = 'csv' if csv_files else 'parquet'
-            file_paths.sort()  # Sort for consistent ordering
-        else:
-            raise ValueError(f"Invalid path: {path}")
-        
-        # Load full data into memory or use streaming
-        if load_full:
-            dfs = []
-            for file_path in file_paths:
-                if file_type == 'csv':
-                    df = pd.read_csv(file_path)
-                else:  # parquet
-                    df = pd.read_parquet(file_path)
-                dfs.append(df)
-            
-            combined_df = pd.concat(dfs, ignore_index=True)
-            return self._create_from_memory(combined_df, batch_size, shuffle)
-        else:
-            return self._load_files_streaming(file_paths, file_type, batch_size, chunk_size)
-    
-
-    def _load_files_streaming(self,
-                             file_paths: list[str],
-                             file_type: Literal['csv', 'parquet'],
-                             batch_size: int,
-                             chunk_size: int) -> DataLoader:
-        # Create FileDataset for streaming
-        dataset = FileDataset(
-            file_paths=file_paths,
-            dense_features=self.dense_features,
-            sparse_features=self.sparse_features,
-            sequence_features=self.sequence_features,
-            target_columns=self.target_columns,
-            chunk_size=chunk_size,
-            file_type=file_type,
-            processor=self.processor
-        )
-        
-        return DataLoader(dataset, batch_size=batch_size, collate_fn=collate_fn)
-    
-    def _get_file_type(self, file_path: str) -> Literal['csv', 'parquet']:
-        ext = os.path.splitext(file_path)[1].lower()
-        if ext == '.csv':
-            return 'csv'
-        elif ext == '.parquet':
-            return 'parquet'
-        else:
-            raise ValueError(f"Unsupported file type: {ext}")
-