nextrec 0.3.3__py3-none-any.whl → 0.3.5__py3-none-any.whl

nextrec/data/data_utils.py

@@ -1,268 +1,35 @@
-"""Data processing utilities for NextRec."""
-
-import torch
-import numpy as np
-import pandas as pd
-import pyarrow.parquet as pq
-from pathlib import Path
-from typing import Any, Mapping, Sequence
-
-def stack_section(batch: list[dict], section: str):
-    """Stack one section of the batch (features/labels/ids)."""
-    entries = [item.get(section) for item in batch if item.get(section) is not None]
-    if not entries:
-        return None
-    merged: dict = {}
-    for name in entries[0]: # type: ignore
-        tensors = [item[section][name] for item in batch if item.get(section) is not None and name in item[section]]
-        merged[name] = torch.stack(tensors, dim=0)
-    return merged
-
-def collate_fn(batch):
-    """
-    Collate a list of sample dicts into the unified batch format:
-    {
-        "features": {name: Tensor(B, ...)},
-        "labels": {target: Tensor(B, ...)} or None,
-        "ids": {id_name: Tensor(B, ...)} or None,
-    }
-    """
-    if not batch:
-        return {"features": {}, "labels": None, "ids": None}
-
-    first = batch[0]
-    if isinstance(first, dict) and "features" in first:
-        # Streaming dataset yields already-batched chunks; avoid adding an extra dim.
-        if first.get("_already_batched") and len(batch) == 1:
-            return {
-                "features": first.get("features", {}),
-                "labels": first.get("labels"),
-                "ids": first.get("ids"),
-            }
-        return {
-            "features": stack_section(batch, "features") or {},
-            "labels": stack_section(batch, "labels"),
-            "ids": stack_section(batch, "ids"),
-        }
-
-    # Fallback: stack tuples/lists of tensors
-    num_tensors = len(first)
-    result = []
-    for i in range(num_tensors):
-        tensor_list = [item[i] for item in batch]
-        first_item = tensor_list[0]
-        if isinstance(first_item, torch.Tensor):
-            stacked = torch.cat(tensor_list, dim=0)
-        elif isinstance(first_item, np.ndarray):
-            stacked = np.concatenate(tensor_list, axis=0)
-        elif isinstance(first_item, list):
-            combined = []
-            for entry in tensor_list:
-                combined.extend(entry)
-            stacked = combined
-        else:
-            stacked = tensor_list
-        result.append(stacked)
-    return tuple(result)
-
-def get_column_data(data: dict | pd.DataFrame, name: str):
-    """Extract column data from various data structures."""
-    if isinstance(data, dict):
-        return data[name] if name in data else None
-    elif isinstance(data, pd.DataFrame):
-        if name not in data.columns:
-            return None
-        return data[name].values
-    else:
-        if hasattr(data, name):
-            return getattr(data, name)
-        raise KeyError(f"Unsupported data type for extracting column {name}")
-
-def resolve_file_paths(path: str) -> tuple[list[str], str]:
-    """Resolve file or directory path into a sorted list of files and file type."""
-    path_obj = Path(path)
-
-    if path_obj.is_file():
-        file_type = path_obj.suffix.lower().lstrip(".")
-        assert file_type in ["csv", "parquet"], f"Unsupported file extension: {file_type}"
-        return [str(path_obj)], file_type
-
-    if path_obj.is_dir():
-        collected_files = [p for p in path_obj.iterdir() if p.is_file()]
-        csv_files = [str(p) for p in collected_files if p.suffix.lower() == ".csv"]
-        parquet_files = [str(p) for p in collected_files if p.suffix.lower() == ".parquet"]
-
-        if csv_files and parquet_files:
-            raise ValueError("Directory contains both CSV and Parquet files. Please keep a single format.")
-        file_paths = csv_files if csv_files else parquet_files
-        if not file_paths:
-            raise ValueError(f"No CSV or Parquet files found in directory: {path}")
-        file_paths.sort()
-        file_type = "csv" if csv_files else "parquet"
-        return file_paths, file_type
-
-    raise ValueError(f"Invalid path: {path}")
-
-def iter_file_chunks(file_path: str, file_type: str, chunk_size: int):
-    """Yield DataFrame chunks for CSV/Parquet without loading the whole file."""
-    if file_type == "csv":
-        yield from pd.read_csv(file_path, chunksize=chunk_size)
-        return
-    parquet_file = pq.ParquetFile(file_path)
-    for batch in parquet_file.iter_batches(batch_size=chunk_size):
-        yield batch.to_pandas()
-
-def read_table(file_path: str, file_type: str) -> pd.DataFrame:
-    """Read a single CSV/Parquet file."""
-    if file_type == "csv":
-        return pd.read_csv(file_path)
-    return pd.read_parquet(file_path)
-
-def load_dataframes(file_paths: list[str], file_type: str) -> list[pd.DataFrame]:
-    """Load multiple files of the same type into DataFrames."""
-    return [read_table(fp, file_type) for fp in file_paths]
-
-def default_output_dir(path: str) -> Path:
-    """Generate a default output directory path based on the input path."""
-    path_obj = Path(path)
-    if path_obj.is_file():
-        return path_obj.parent / f"{path_obj.stem}_preprocessed"
-    return path_obj.with_name(f"{path_obj.name}_preprocessed")
-
-def split_dict_random(data_dict: dict, test_size: float = 0.2, random_state: int | None = None):
-    """Randomly split a dictionary of data into training and testing sets."""
-    lengths = [len(v) for v in data_dict.values()]
-    if len(set(lengths)) != 1:
-        raise ValueError(f"Length mismatch: {lengths}")
-    n = lengths[0]
-    rng = np.random.default_rng(random_state)
-    perm = rng.permutation(n)
-    cut = int(round(n * (1 - test_size)))
-    train_idx, test_idx = perm[:cut], perm[cut:]
-    def take(v, idx):
-        if isinstance(v, np.ndarray):
-            return v[idx]
-        elif isinstance(v, pd.Series):
-            return v.iloc[idx].to_numpy()  
-        else:
-            v_arr = np.asarray(v, dtype=object)  
-            return v_arr[idx]
-    train_dict = {k: take(v, train_idx) for k, v in data_dict.items()}
-    test_dict  = {k: take(v, test_idx)  for k, v in data_dict.items()}
-    return train_dict, test_dict
-
-def build_eval_candidates(
-    df_all: pd.DataFrame,
-    user_col: str,
-    item_col: str,
-    label_col: str,
-    user_features: pd.DataFrame,
-    item_features: pd.DataFrame,
-    num_pos_per_user: int = 5,
-    num_neg_per_pos: int = 50,
-    random_seed: int = 2025,
-) -> pd.DataFrame:
-    """Build evaluation candidates with positive and negative samples for each user.   """
-    rng = np.random.default_rng(random_seed)
-
-    users = df_all[user_col].unique()
-    all_items = item_features[item_col].unique()
-    rows = []
-    user_hist_items = {u: df_all[df_all[user_col] == u][item_col].unique() for u in users}
-    for u in users:
-        df_user = df_all[df_all[user_col] == u]
-        pos_items = df_user[df_user[label_col] == 1][item_col].unique()
-        if len(pos_items) == 0:
-            continue
-        pos_items = pos_items[:num_pos_per_user]
-        seen_items = set(user_hist_items[u])
-        neg_pool = np.setdiff1d(all_items, np.fromiter(seen_items, dtype=all_items.dtype))
-        if len(neg_pool) == 0:
-            continue
-        for pos in pos_items:
-            if len(neg_pool) <= num_neg_per_pos:
-                neg_items = neg_pool
-            else:
-                neg_items = rng.choice(neg_pool, size=num_neg_per_pos, replace=False)
-            rows.append((u, pos, 1))
-            for ni in neg_items:
-                rows.append((u, ni, 0))
-    eval_df = pd.DataFrame(rows, columns=[user_col, item_col, label_col])
-    eval_df = eval_df.merge(user_features, on=user_col, how='left')
-    eval_df = eval_df.merge(item_features, on=item_col, how='left')
-    return eval_df
-
-def batch_to_dict(batch_data: Any, include_ids: bool = True) -> dict:
-    """Standardize a dataloader batch into a dict of features, labels, and ids."""
-    if not (isinstance(batch_data, Mapping) and "features" in batch_data):
-        raise TypeError(
-            "[BaseModel-batch_to_dict Error] Batch data must be a dict with 'features' produced by the current DataLoader."
-        )
-    return {
-        "features": batch_data.get("features", {}),
-        "labels": batch_data.get("labels"),
-        "ids": batch_data.get("ids") if include_ids else None,
-    }
-
-
-# def get_user_ids(
-#     data: dict | pd.DataFrame | None, user_id_column: str = "user_id"
-# ) -> np.ndarray | None:
-#     """Extract user IDs from a dataset dict or DataFrame."""
-#     if data is None:
-#         return None
-#     if isinstance(data, pd.DataFrame) and user_id_column in data.columns:
-#         return np.asarray(data[user_id_column].values)
-#     if isinstance(data, dict) and user_id_column in data:
-#         return np.asarray(data[user_id_column])
-#     return None
-
-
-# def get_user_ids_from_batch(
-#     batch_dict: Mapping[str, Any], id_columns: Sequence[str] | None = None
-# ) -> np.ndarray | None:
-#     """Extract the prioritized user id column from a batch dict."""
-#     ids_container = batch_dict.get("ids") if isinstance(batch_dict, Mapping) else None
-#     if not ids_container:
-#         return None
-
-#     batch_user_id = None
-#     if id_columns:
-#         for id_name in id_columns:
-#             if id_name in ids_container:
-#                 batch_user_id = ids_container[id_name]
-#                 break
-#     if batch_user_id is None:
-#         batch_user_id = next(iter(ids_container.values()), None)
-#     if batch_user_id is None:
-#         return None
-
-#     if isinstance(batch_user_id, torch.Tensor):
-#         ids_np = batch_user_id.detach().cpu().numpy()
-#     else:
-#         ids_np = np.asarray(batch_user_id)
-#     if ids_np.ndim == 0:
-#         ids_np = ids_np.reshape(1)
-#     return ids_np.reshape(ids_np.shape[0])
-
-
-def get_user_ids(data, id_columns: list[str] | str | None = None) -> np.ndarray | None:
-    id_columns = id_columns if isinstance(id_columns, list) else [id_columns] if isinstance(id_columns, str) else []
-    if not id_columns:
-        return None
-
-    main_id = id_columns[0]  
-    if isinstance(data, pd.DataFrame) and main_id in data.columns:
-        arr = np.asarray(data[main_id].values)
-        return arr.reshape(arr.shape[0])
-    if isinstance(data, dict):
-        ids_container = data.get("ids")
-        if isinstance(ids_container, dict) and main_id in ids_container:
-            val = ids_container[main_id]
-            val = val.detach().cpu().numpy() if isinstance(val, torch.Tensor) else np.asarray(val)
-            return val.reshape(val.shape[0])
-        if main_id in data:
-            arr = np.asarray(data[main_id])
-            return arr.reshape(arr.shape[0])
-
-    return None
+"""
+Data processing utilities for NextRec (Refactored)
+
+This module now re-exports functions from specialized submodules:
+- batch_utils: collate_fn, batch_to_dict
+- data_processing: get_column_data, split_dict_random, build_eval_candidates, get_user_ids
+- nextrec.utils.file_utils: resolve_file_paths, iter_file_chunks, read_table, load_dataframes, default_output_dir
+
+Date: create on 27/10/2025
+Last update: 03/12/2025 (refactored)
+Author: Yang Zhou, zyaztec@gmail.com
+"""
+
+# Import from new organized modules
+from nextrec.data.batch_utils import collate_fn, batch_to_dict, stack_section
+from nextrec.data.data_processing import get_column_data, split_dict_random, build_eval_candidates, get_user_ids
+from nextrec.utils.file import resolve_file_paths, iter_file_chunks, read_table, load_dataframes, default_output_dir
+
+__all__ = [
+    # Batch utilities
+    'collate_fn',
+    'batch_to_dict',
+    'stack_section',
+    # Data processing
+    'get_column_data',
+    'split_dict_random',
+    'build_eval_candidates',
+    'get_user_ids',
+    # File utilities
+    'resolve_file_paths',
+    'iter_file_chunks',
+    'read_table',
+    'load_dataframes',
+    'default_output_dir',
+]

nextrec/data/dataloader.py

@@ -20,8 +20,10 @@ from nextrec.data.preprocessor import DataProcessor
 from nextrec.basic.features import DenseFeature, SparseFeature, SequenceFeature, FeatureSet
 
 from nextrec.basic.loggers import colorize
-from nextrec.data import get_column_data, collate_fn, resolve_file_paths, read_table
-from nextrec.utils import to_tensor
+from nextrec.data.data_processing import get_column_data
+from nextrec.data.batch_utils import collate_fn
+from nextrec.utils.file import resolve_file_paths, read_table
+from nextrec.utils.tensor import to_tensor
 
 class TensorDictDataset(Dataset):
     """Dataset returning sample-level dicts matching the unified batch schema."""
@@ -185,9 +187,9 @@ class RecDataLoader(FeatureSet):
                              chunk_size: int,
                              shuffle: bool) -> DataLoader:
         if shuffle:
-            logging.warning("[RecDataLoader Warning] Shuffle is ignored in streaming mode (IterableDataset).")
+            logging.info("[RecDataLoader Info] Shuffle is ignored in streaming mode (IterableDataset).")
         if batch_size != 1:
-            logging.warning("[RecDataLoader Warning] Streaming mode enforces batch_size=1; tune chunk_size to control memory/throughput.")
+            logging.info("[RecDataLoader Info] Streaming mode enforces batch_size=1; tune chunk_size to control memory/throughput.")
         dataset = FileDataset(file_paths=file_paths, dense_features=self.dense_features, sparse_features=self.sparse_features, sequence_features=self.sequence_features, target_columns=self.target_columns, id_columns=self.id_columns, chunk_size=chunk_size, file_type=file_type, processor=self.processor)
         return DataLoader(dataset, batch_size=1, collate_fn=collate_fn)
 

nextrec/data/preprocessor.py

@@ -16,48 +16,18 @@ import pandas as pd
 import tqdm
 from pathlib import Path
 from typing import Dict, Union, Optional, Literal, Any
-from sklearn.preprocessing import (
-    StandardScaler, 
-    MinMaxScaler, 
-    RobustScaler, 
-    MaxAbsScaler,
-    LabelEncoder
-)
+from sklearn.preprocessing import StandardScaler, MinMaxScaler, RobustScaler, MaxAbsScaler, LabelEncoder
+
 
-from nextrec.basic.loggers import setup_logger, colorize
-from nextrec.data.data_utils import (
-    resolve_file_paths,
-    iter_file_chunks,
-    read_table,
-    load_dataframes,
-    default_output_dir,
-)
-from nextrec.basic.session import resolve_save_path
 from nextrec.basic.features import FeatureSet
+from nextrec.basic.loggers import colorize
+from nextrec.basic.session import resolve_save_path
+from nextrec.utils.file import resolve_file_paths, iter_file_chunks, read_table, load_dataframes, default_output_dir
+
 from nextrec.__version__ import __version__
 
 
 class DataProcessor(FeatureSet):
-    """DataProcessor for data preprocessing including numeric, sparse, sequence features and target processing.
-    
-    Examples:
-        >>> processor = DataProcessor()
-        >>> processor.add_numeric_feature('age', scaler='standard')
-        >>> processor.add_sparse_feature('user_id', encode_method='hash', hash_size=10000)
-        >>> processor.add_sequence_feature('item_history', encode_method='label', max_len=50, pad_value=0)
-        >>> processor.add_target('label', target_type='binary')
-        >>> 
-        >>> # Fit and transform data
-        >>> processor.fit(train_df)
-        >>> processed_data = processor.transform(test_df)  # Returns dict of numpy arrays
-        >>> 
-        >>> # Save and load processor
-        >>> processor.save('processor.pkl')
-        >>> loaded_processor = DataProcessor.load('processor.pkl')
-        >>> 
-        >>> # Get vocabulary sizes for embedding layers
-        >>> vocab_sizes = processor.get_vocab_sizes()
-    """
     def __init__(self):
         self.numeric_features: Dict[str, Dict[str, Any]] = {}
         self.sparse_features: Dict[str, Dict[str, Any]] = {}
@@ -132,10 +102,10 @@ class DataProcessor(FeatureSet):
         }
         self.set_target_id(list(self.target_features.keys()), [])
         
-    def _hash_string(self, s: str, hash_size: int) -> int:
+    def hash_string(self, s: str, hash_size: int) -> int:
         return int(hashlib.md5(str(s).encode()).hexdigest(), 16) % hash_size
         
-    def _process_numeric_feature_fit(self, data: pd.Series, config: Dict[str, Any]):
+    def process_numeric_feature_fit(self, data: pd.Series, config: Dict[str, Any]):
         name = str(data.name)
         scaler_type = config['scaler']
         fill_na = config['fill_na']
@@ -164,7 +134,7 @@ class DataProcessor(FeatureSet):
             scaler.fit(values)
             self.scalers[name] = scaler
             
-    def _process_numeric_feature_transform(self, data: pd.Series, config: Dict[str, Any]) -> np.ndarray:
+    def process_numeric_feature_transform(self, data: pd.Series, config: Dict[str, Any]) -> np.ndarray:
         logger = logging.getLogger()
         name = str(data.name)
         scaler_type = config['scaler']
@@ -184,7 +154,7 @@ class DataProcessor(FeatureSet):
                 result = scaler.transform(values.reshape(-1, 1)).ravel()
         return result
         
-    def _process_sparse_feature_fit(self, data: pd.Series, config: Dict[str, Any]):
+    def process_sparse_feature_fit(self, data: pd.Series, config: Dict[str, Any]):
         name = str(data.name)
         encode_method = config['encode_method']
         fill_na = config['fill_na'] # <UNK>
@@ -197,7 +167,7 @@ class DataProcessor(FeatureSet):
         elif encode_method == 'hash':
             config['vocab_size'] = config['hash_size']
             
-    def _process_sparse_feature_transform(self, data: pd.Series, config: Dict[str, Any]) -> np.ndarray:
+    def process_sparse_feature_transform(self, data: pd.Series, config: Dict[str, Any]) -> np.ndarray:
         name = str(data.name)
         encode_method = config['encode_method']
         fill_na = config['fill_na']
@@ -215,11 +185,11 @@ class DataProcessor(FeatureSet):
             return encoded.to_numpy()
         if encode_method == 'hash':
             hash_size = config['hash_size']
-            hash_fn = self._hash_string
+            hash_fn = self.hash_string
             return np.fromiter((hash_fn(v, hash_size) for v in sparse_series.to_numpy()), dtype=np.int64, count=sparse_series.size,)
         return np.array([], dtype=np.int64)
             
-    def _process_sequence_feature_fit(self, data: pd.Series, config: Dict[str, Any]):
+    def process_sequence_feature_fit(self, data: pd.Series, config: Dict[str, Any]):
         name = str(data.name)
         encode_method = config['encode_method']
         separator = config['separator']
@@ -252,7 +222,7 @@ class DataProcessor(FeatureSet):
         elif encode_method == 'hash':
             config['vocab_size'] = config['hash_size']
             
-    def _process_sequence_feature_transform(self, data: pd.Series, config: Dict[str, Any]) -> np.ndarray:
+    def process_sequence_feature_transform(self, data: pd.Series, config: Dict[str, Any]) -> np.ndarray:
         """Optimized sequence transform with preallocation and cached vocab map."""
         name = str(data.name)
         encode_method = config['encode_method']
@@ -276,7 +246,7 @@ class DataProcessor(FeatureSet):
                 config['_class_to_idx'] = class_to_idx
         else:
             class_to_idx = None  # type: ignore
-        hash_fn = self._hash_string
+        hash_fn = self.hash_string
         hash_size = config.get('hash_size')
         for i, seq in enumerate(arr):
             # normalize sequence to a list of strings
@@ -301,11 +271,7 @@ class DataProcessor(FeatureSet):
             elif encode_method == 'hash':
                 if hash_size is None:
                     raise ValueError("hash_size must be set for hash encoding")
-                encoded = [
-                    hash_fn(str(token), hash_size)
-                    for token in tokens
-                    if str(token).strip()
-                ]
+                encoded = [hash_fn(str(token), hash_size) for token in tokens if str(token).strip()]
             else:
                 encoded = []
             if not encoded:
@@ -315,7 +281,7 @@ class DataProcessor(FeatureSet):
             output[i, : len(encoded)] = encoded
         return output
         
-    def _process_target_fit(self, data: pd.Series, config: Dict[str, Any]):
+    def process_target_fit(self, data: pd.Series, config: Dict[str, Any]):
         name = str(data.name)
         target_type = config['target_type']
         label_map = config.get('label_map')
@@ -334,7 +300,7 @@ class DataProcessor(FeatureSet):
                 config['label_map'] = label_map    
             self.target_encoders[name] = label_map
             
-    def _process_target_transform(self, data: pd.Series, config: Dict[str, Any]) -> np.ndarray:
+    def process_target_transform(self, data: pd.Series, config: Dict[str, Any]) -> np.ndarray:
         logger = logging.getLogger()
         name = str(data.name)
         target_type = config.get('target_type')
@@ -355,13 +321,13 @@ class DataProcessor(FeatureSet):
                     result.append(0)
             return np.array(result, dtype=np.int64 if target_type == 'multiclass' else np.float32)
     
-    def _load_dataframe_from_path(self, path: str) -> pd.DataFrame:
+    def load_dataframe_from_path(self, path: str) -> pd.DataFrame:
         """Load all data from a file or directory path into a single DataFrame."""
         file_paths, file_type = resolve_file_paths(path)
         frames = load_dataframes(file_paths, file_type)
         return pd.concat(frames, ignore_index=True) if len(frames) > 1 else frames[0]
 
-    def _extract_sequence_tokens(self, value: Any, separator: str) -> list[str]:
+    def extract_sequence_tokens(self, value: Any, separator: str) -> list[str]:
         """Extract sequence tokens from a single value."""
         if value is None:
             return []
@@ -374,7 +340,7 @@ class DataProcessor(FeatureSet):
             return [str(v) for v in value]
         return [str(value)]
 
-    def _fit_from_path(self, path: str, chunk_size: int) -> 'DataProcessor':
+    def fit_from_path(self, path: str, chunk_size: int) -> 'DataProcessor':
         """Fit processor statistics by streaming files to reduce memory usage."""
         logger = logging.getLogger()
         logger.info(colorize("Fitting DataProcessor (streaming path mode)...", color="cyan", bold=True))
@@ -433,7 +399,7 @@ class DataProcessor(FeatureSet):
                     series = chunk[name]
                     tokens = []
                     for val in series:
-                        tokens.extend(self._extract_sequence_tokens(val, separator))
+                        tokens.extend(self.extract_sequence_tokens(val, separator))
                     seq_vocab[name].update(tokens)
 
                 # target features
@@ -548,7 +514,7 @@ class DataProcessor(FeatureSet):
         logger.info(colorize("DataProcessor fitted successfully (streaming path mode)", color="green", bold=True))
         return self
 
-    def _transform_in_memory(
+    def transform_in_memory(
         self,
         data: Union[pd.DataFrame, Dict[str, Any]],
         return_dict: bool,
@@ -581,7 +547,7 @@ class DataProcessor(FeatureSet):
                 continue
             # Convert to Series for processing
             series_data = pd.Series(data_dict[name], name=name)
-            processed = self._process_numeric_feature_transform(series_data, config)
+            processed = self.process_numeric_feature_transform(series_data, config)
             result_dict[name] = processed
 
         # process sparse features
@@ -590,7 +556,7 @@ class DataProcessor(FeatureSet):
                 logger.warning(f"Sparse feature {name} not found in data")
                 continue
             series_data = pd.Series(data_dict[name], name=name)
-            processed = self._process_sparse_feature_transform(series_data, config)
+            processed = self.process_sparse_feature_transform(series_data, config)
             result_dict[name] = processed
 
         # process sequence features
@@ -599,7 +565,7 @@ class DataProcessor(FeatureSet):
                 logger.warning(f"Sequence feature {name} not found in data")
                 continue
             series_data = pd.Series(data_dict[name], name=name)
-            processed = self._process_sequence_feature_transform(series_data, config)
+            processed = self.process_sequence_feature_transform(series_data, config)
             result_dict[name] = processed
 
         # process target features
@@ -608,10 +574,10 @@ class DataProcessor(FeatureSet):
                 logger.warning(f"Target {name} not found in data")
                 continue
             series_data = pd.Series(data_dict[name], name=name)
-            processed = self._process_target_transform(series_data, config)
+            processed = self.process_target_transform(series_data, config)
             result_dict[name] = processed
 
-        def _dict_to_dataframe(result: Dict[str, np.ndarray]) -> pd.DataFrame:
+        def dict_to_dataframe(result: Dict[str, np.ndarray]) -> pd.DataFrame:
             # Convert all arrays to Series/lists at once to avoid fragmentation
             columns_dict = {}
             for key, value in result.items():
@@ -629,7 +595,7 @@ class DataProcessor(FeatureSet):
             effective_format = save_format or "parquet"
         result_df = None
         if (not return_dict) or persist:
-            result_df = _dict_to_dataframe(result_dict)
+            result_df = dict_to_dataframe(result_dict)
         if persist:
             if output_path is None:
                 raise ValueError("output_path must be provided when persisting transformed data.")
@@ -649,7 +615,7 @@ class DataProcessor(FeatureSet):
         assert result_df is not None, "DataFrame is None after transform"
         return result_df
 
-    def _transform_path(
+    def transform_path(
         self,
         input_path: str,
         output_path: Optional[str],
@@ -669,13 +635,7 @@ class DataProcessor(FeatureSet):
         saved_paths = []
         for file_path in tqdm.tqdm(file_paths, desc="Transforming files", unit="file"):
             df = read_table(file_path, file_type)
-            transformed_df = self._transform_in_memory(
-                df,
-                return_dict=False,
-                persist=False,
-                save_format=None,
-                output_path=None,
-            )
+            transformed_df = self.transform_in_memory(df, return_dict=False, persist=False, save_format=None, output_path=None)
             assert isinstance(transformed_df, pd.DataFrame), "Expected DataFrame when return_dict=False"
             source_path = Path(file_path)
             target_file = output_root / f"{source_path.stem}.{target_format}"
@@ -695,9 +655,9 @@ class DataProcessor(FeatureSet):
             uses_robust = any(cfg.get("scaler") == "robust" for cfg in self.numeric_features.values())
             if uses_robust:
                 logger.warning("Robust scaler requires full data; loading all files into memory. Consider smaller chunk_size or different scaler if memory is limited.")
-                data = self._load_dataframe_from_path(path_str)
+                data = self.load_dataframe_from_path(path_str)
             else:
-                return self._fit_from_path(path_str, chunk_size)
+                return self.fit_from_path(path_str, chunk_size)
         if isinstance(data, dict):
             data = pd.DataFrame(data)
         logger.info(colorize("Fitting DataProcessor...", color="cyan", bold=True))
@@ -705,22 +665,22 @@ class DataProcessor(FeatureSet):
             if name not in data.columns:
                 logger.warning(f"Numeric feature {name} not found in data")
                 continue
-            self._process_numeric_feature_fit(data[name], config)
+            self.process_numeric_feature_fit(data[name], config)
         for name, config in self.sparse_features.items():
             if name not in data.columns:
                 logger.warning(f"Sparse feature {name} not found in data")
                 continue
-            self._process_sparse_feature_fit(data[name], config)
+            self.process_sparse_feature_fit(data[name], config)
         for name, config in self.sequence_features.items():
             if name not in data.columns:
                 logger.warning(f"Sequence feature {name} not found in data")
                 continue
-            self._process_sequence_feature_fit(data[name], config)
+            self.process_sequence_feature_fit(data[name], config)
         for name, config in self.target_features.items():
             if name not in data.columns:
                 logger.warning(f"Target {name} not found in data")
                 continue
-            self._process_target_fit(data[name], config)
+            self.process_target_fit(data[name], config)
         self.is_fitted = True
         return self
         
@@ -736,14 +696,8 @@ class DataProcessor(FeatureSet):
         if isinstance(data, (str, os.PathLike)):
             if return_dict:
                 raise ValueError("Path transform writes files only; set return_dict=False when passing a path.")
-            return self._transform_path(str(data), output_path, save_format)
-        return self._transform_in_memory(
-            data=data,
-            return_dict=return_dict,
-            persist=output_path is not None,
-            save_format=save_format,
-            output_path=output_path,
-        )
+            return self.transform_path(str(data), output_path, save_format)
+        return self.transform_in_memory(data=data, return_dict=return_dict, persist=output_path is not None, save_format=save_format, output_path=output_path)
             
     def fit_transform(
         self, 

nextrec/models/multi_task/poso.py

@@ -46,7 +46,7 @@ from nextrec.basic.features import DenseFeature, SequenceFeature, SparseFeature
 from nextrec.basic.layers import EmbeddingLayer, MLP, PredictionLayer
 from nextrec.basic.activation import activation_layer
 from nextrec.basic.model import BaseModel
-from nextrec.utils.common import merge_features
+from nextrec.utils.model import merge_features
 
 
 class POSOGate(nn.Module):