nextrec 0.4.8__py3-none-any.whl → 0.4.9__py3-none-any.whl

nextrec/utils/__init__.py

@@ -1,71 +1,84 @@
 """
 Utilities package for NextRec
 
-This package provides various utility functions organized by category:
-- optimizer: Optimizer and scheduler utilities
-- initializer: Weight initialization utilities
-- embedding: Embedding dimension calculation
-- device_utils: Device management and selection
-- tensor_utils: Tensor operations and conversions
-- file_utils: File I/O operations
-- model_utils: Model-related utilities
-- feature_utils: Feature processing utilities
-- config_utils: Configuration loading and processing utilities
-
 Date: create on 13/11/2025
-Last update: 06/12/2025
+Last update: 19/12/2025
 Author: Yang Zhou, zyaztec@gmail.com
 """
 
-from . import optimizer, initializer, embedding
-from .optimizer import get_optimizer, get_scheduler
-from .initializer import get_initializer
-from .embedding import get_auto_embedding_dim
-from .device import resolve_device, get_device_info
-from .tensor import to_tensor, stack_tensors, concat_tensors, pad_sequence_tensors
-from .file import (
-    resolve_file_paths,
-    read_table,
-    load_dataframes,
-    iter_file_chunks,
-    default_output_dir,
-    read_yaml,
-)
-from .model import merge_features, get_mlp_output_dim
-from .feature import normalize_to_list
-from .synthetic_data import (
-    generate_match_data,
-    generate_ranking_data,
-    generate_multitask_data,
-    generate_distributed_ranking_data,
-)
+from . import console, data, embedding, torch_utils
 from .config import (
-    resolve_path,
-    select_features,
-    register_processor_features,
     build_feature_objects,
+    build_model_instance,
     extract_feature_groups,
     load_model_class,
-    build_model_instance,
+    register_processor_features,
+    resolve_path,
+    select_features,
+)
+from .console import (
+    display_metrics_table,
+    get_nextrec_version,
+    log_startup_info,
+    progress,
+)
+from .data import (
+    default_output_dir,
+    generate_distributed_ranking_data,
+    generate_match_data,
+    generate_multitask_data,
+    generate_ranking_data,
+    iter_file_chunks,
+    load_dataframes,
+    read_table,
+    read_yaml,
+    resolve_file_paths,
+)
+from .embedding import get_auto_embedding_dim
+from .feature import normalize_to_list
+from .model import get_mlp_output_dim, merge_features
+from .torch_utils import (
+    add_distributed_sampler,
+    concat_tensors,
+    configure_device,
+    gather_numpy,
+    get_device_info,
+    get_initializer,
+    get_optimizer,
+    get_scheduler,
+    init_process_group,
+    pad_sequence_tensors,
+    resolve_device,
+    stack_tensors,
+    to_tensor,
 )
 
 __all__ = [
-    # Optimizer & Scheduler
+    # Console utilities
+    "get_nextrec_version",
+    "log_startup_info",
+    "progress",
+    "display_metrics_table",
+    # Optimizer & Scheduler (torch utils)
     "get_optimizer",
     "get_scheduler",
-    # Initializer
+    # Initializer (torch utils)
     "get_initializer",
-    # Embedding
+    # Embedding utilities
     "get_auto_embedding_dim",
-    # Device utilities
+    # Device utilities (torch utils)
     "resolve_device",
     "get_device_info",
+    "configure_device",
+    "init_process_group",
+    "gather_numpy",
+    "add_distributed_sampler",
     # Tensor utilities
     "to_tensor",
     "stack_tensors",
     "concat_tensors",
     "pad_sequence_tensors",
-    # File utilities
+    # Data utilities
     "resolve_file_paths",
     "read_table",
     "read_yaml",
@@ -79,10 +92,10 @@ __all__ = [
     "normalize_to_list",
     # Config utilities
     "resolve_path",
-    "select_features",
     "register_processor_features",
     "build_feature_objects",
     "extract_feature_groups",
+    "select_features",
     "load_model_class",
     "build_model_instance",
     # Synthetic data utilities
@@ -91,7 +104,8 @@ __all__ = [
     "generate_multitask_data",
     "generate_distributed_ranking_data",
     # Module exports
-    "optimizer",
-    "initializer",
+    "console",
+    "data",
     "embedding",
+    "torch_utils",
 ]

nextrec/utils/config.py

@@ -4,7 +4,8 @@ Configuration utilities for NextRec
 This module provides utilities for loading and processing configuration files,
 including feature configuration, model configuration, and training configuration.
 
-Date: create on 06/12/2025
+Date: create on 27/10/2025
+Checkpoint: edit on 19/12/2025
 Author: Yang Zhou, zyaztec@gmail.com
 """
 
@@ -23,7 +24,7 @@ import torch
 from nextrec.utils.feature import normalize_to_list
 
 if TYPE_CHECKING:
-    from nextrec.basic.features import DenseFeature, SparseFeature, SequenceFeature
+    from nextrec.basic.features import DenseFeature, SequenceFeature, SparseFeature
     from nextrec.data.preprocessor import DataProcessor
 
 
@@ -52,7 +53,7 @@ def select_features(
         names = [name for name in cfg.keys() if name in columns]
         missing = [name for name in cfg.keys() if name not in columns]
         if missing:
-            print(f"[feature_config] skipped missing {group} columns: {missing}")
+            print(f"[Feature Config] skipped missing {group} columns: {missing}")
         return names
 
     dense_names = pick("dense")
@@ -129,7 +130,7 @@ def build_feature_objects(
         sparse_names: List of sparse feature names
         sequence_names: List of sequence feature names
     """
-    from nextrec.basic.features import DenseFeature, SparseFeature, SequenceFeature
+    from nextrec.basic.features import DenseFeature, SequenceFeature, SparseFeature
 
     dense_cfg = feature_cfg.get("dense", {}) or {}
     sparse_cfg = feature_cfg.get("sparse", {}) or {}
@@ -236,7 +237,7 @@ def extract_feature_groups(
 
         if missing_defined:
             print(
-                f"[feature_config] feature_groups.{group_name} contains features not defined in dense/sparse/sequence: {missing_defined}"
+                f"[Feature Config] feature_groups.{group_name} contains features not defined in dense/sparse/sequence: {missing_defined}"
             )
 
         for n in name_list:
@@ -249,7 +250,7 @@ def extract_feature_groups(
 
         if missing_cols:
             print(
-                f"[feature_config] feature_groups.{group_name} missing data columns: {missing_cols}"
+                f"[Feature Config] feature_groups.{group_name} missing data columns: {missing_cols}"
             )
 
         resolved[group_name] = filtered
@@ -442,7 +443,7 @@ def build_model_instance(
 
         if group_key not in feature_groups:
             print(
-                f"[feature_config] feature_bindings refers to unknown group '{group_key}', skipped"
+                f"[Feature Config] feature_bindings refers to unknown group '{group_key}', skipped"
             )
             continue
 

nextrec/utils/console.py

@@ -0,0 +1,371 @@
+"""
+Console and CLI utilities for NextRec.
+
+This module centralizes CLI logging helpers, progress display, and metric tables.
+
+Date: create on 19/12/2025
+Checkpoint: edit on 19/12/2025
+Author: Yang Zhou, zyaztec@gmail.com
+"""
+
+from __future__ import annotations
+
+import io
+import logging
+import numbers
+import os
+import platform
+import sys
+from datetime import datetime, timedelta
+from typing import Any, Callable, Iterable, Mapping, TypeVar
+
+import numpy as np
+from rich import box
+from rich.console import Console
+from rich.progress import (
+    BarColumn,
+    MofNCompleteColumn,
+    Progress,
+    SpinnerColumn,
+    TaskProgressColumn,
+    TextColumn,
+    TimeElapsedColumn,
+    TimeRemainingColumn,
+)
+from rich.table import Table
+from rich.text import Text
+
+from nextrec.utils.feature import as_float, normalize_to_list
+
+T = TypeVar("T")
+
+
+def get_nextrec_version() -> str:
+    """
+    Best-effort version resolver for NextRec.
+
+    Prefer in-repo `nextrec.__version__`, fall back to installed package metadata.
+    """
+    try:
+        from nextrec import __version__  # type: ignore
+
+        if __version__:
+            return str(__version__)
+    except Exception:
+        pass
+
+    try:
+        from importlib.metadata import version
+
+        return version("nextrec")
+    except Exception:
+        return "unknown"
+
+
+def log_startup_info(
+    logger: logging.Logger, *, mode: str, config_path: str | None
+) -> None:
+    """Log a short, user-friendly startup banner."""
+    version = get_nextrec_version()
+    now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
+    lines = [
+        "NextRec CLI",
+        f"- Version: {version}",
+        f"- Time: {now}",
+        f"- Mode: {mode}",
+        f"- Config: {config_path or '(not set)'}",
+        f"- Python: {platform.python_version()} ({sys.executable})",
+        f"- Platform: {platform.system()} {platform.release()} ({platform.machine()})",
+        f"- Workdir: {os.getcwd()}",
+        f"- Command: {' '.join(sys.argv)}",
+    ]
+    for line in lines:
+        logger.info(line)
+
+
+class BlackTimeElapsedColumn(TimeElapsedColumn):
+    def render(self, task) -> Text:
+        elapsed = task.finished_time if task.finished else task.elapsed
+        if elapsed is None:
+            return Text("-:--:--", style="black")
+        delta = timedelta(seconds=max(0, int(elapsed)))
+        return Text(str(delta), style="black")
+
+
+class BlackTimeRemainingColumn(TimeRemainingColumn):
+    def render(self, task) -> Text:
+        if self.elapsed_when_finished and task.finished:
+            task_time = task.finished_time
+        else:
+            task_time = task.time_remaining
+
+        if task.total is None:
+            return Text("", style="black")
+
+        if task_time is None:
+            return Text("--:--" if self.compact else "-:--:--", style="black")
+
+        minutes, seconds = divmod(int(task_time), 60)
+        hours, minutes = divmod(minutes, 60)
+
+        if self.compact and not hours:
+            formatted = f"{minutes:02d}:{seconds:02d}"
+        else:
+            formatted = f"{hours:d}:{minutes:02d}:{seconds:02d}"
+
+        return Text(formatted, style="black")
+
+
+class BlackMofNCompleteColumn(MofNCompleteColumn):
+    def render(self, task) -> Text:
+        completed = int(task.completed)
+        total = int(task.total) if task.total is not None else "?"
+        total_width = len(str(total))
+        return Text(
+            f"{completed:{total_width}d}{self.separator}{total}",
+            style="black",
+        )
+
+
+def progress(
+    iterable: Iterable[T],
+    *,
+    description: str | None = None,
+    total: int | None = None,
+    disable: bool = False,
+) -> Iterable[T]:
+    if disable:
+        for item in iterable:
+            yield item
+        return
+    resolved_total = total
+    if resolved_total is None:
+        try:
+            resolved_total = len(iterable)  # type: ignore[arg-type]
+        except TypeError:
+            resolved_total = None
+
+    progress_bar = Progress(
+        SpinnerColumn(style="black"),
+        TextColumn("{task.description}", style="black"),
+        BarColumn(
+            bar_width=36, style="black", complete_style="black", finished_style="black"
+        ),
+        TaskProgressColumn(style="black"),
+        BlackMofNCompleteColumn(),
+        BlackTimeElapsedColumn(),
+        BlackTimeRemainingColumn(),
+        refresh_per_second=12,
+    )
+
+    task_id = progress_bar.add_task(description or "Working", total=resolved_total)
+    progress_bar.start()
+    try:
+        for item in iterable:
+            yield item
+            progress_bar.advance(task_id, 1)
+    finally:
+        progress_bar.stop()
+
+
+def group_metrics_by_task(
+    metrics: Mapping[str, Any] | None,
+    target_names: list[str] | str | None,
+    default_task_name: str = "overall",
+) -> tuple[list[str], dict[str, dict[str, float]]]:
+    if not metrics:
+        return [], {}
+
+    if isinstance(target_names, str):
+        target_names = [target_names]
+    if not isinstance(target_names, list) or not target_names:
+        target_names = [default_task_name]
+
+    targets_by_len = sorted(target_names, key=len, reverse=True)
+    grouped: dict[str, dict[str, float]] = {}
+    for key, raw_value in metrics.items():
+        value = as_float(raw_value)
+        if value is None:
+            continue
+
+        matched_target: str | None = None
+        metric_name = key
+        for target in targets_by_len:
+            suffix = f"_{target}"
+            if key.endswith(suffix):
+                metric_name = key[: -len(suffix)]
+                matched_target = target
+                break
+
+        if matched_target is None:
+            matched_target = (
+                target_names[0] if len(target_names) == 1 else default_task_name
+            )
+        grouped.setdefault(matched_target, {})[metric_name] = value
+
+    task_order: list[str] = []
+    for target in target_names:
+        if target in grouped:
+            task_order.append(target)
+    for task_name in grouped:
+        if task_name not in task_order:
+            task_order.append(task_name)
+    return task_order, grouped
+
+
+def display_metrics_table(
+    epoch: int,
+    epochs: int,
+    split: str,
+    loss: float | np.ndarray | None,
+    metrics: Mapping[str, Any] | None,
+    target_names: list[str] | str | None,
+    base_metrics: list[str] | None = None,
+    is_main_process: bool = True,
+    colorize: Callable[[str], str] | None = None,
+) -> None:
+    if not is_main_process:
+        return
+
+    target_list = normalize_to_list(target_names)
+    task_order, grouped = group_metrics_by_task(metrics, target_names=target_names)
+
+    if isinstance(loss, np.ndarray) and target_list:
+        # Ensure tasks with losses are shown even when metrics are missing for some targets.
+        normalized_order: list[str] = []
+        for name in target_list:
+            if name not in normalized_order:
+                normalized_order.append(name)
+        for name in task_order:
+            if name not in normalized_order:
+                normalized_order.append(name)
+        task_order = normalized_order
+
+    if Console is None or Table is None or box is None:
+        prefix = f"Epoch {epoch}/{epochs} - {split}:"
+        segments: list[str] = []
+        if isinstance(loss, numbers.Number):
+            segments.append(f"loss={float(loss):.4f}")
+        if task_order and grouped:
+            task_strs: list[str] = []
+            for task_name in task_order:
+                metric_items = grouped.get(task_name, {})
+                if not metric_items:
+                    continue
+                metric_str = ", ".join(
+                    f"{k}={float(v):.4f}" for k, v in metric_items.items()
+                )
+                task_strs.append(f"{task_name}[{metric_str}]")
+            if task_strs:
+                segments.append(", ".join(task_strs))
+        elif metrics:
+            metric_str = ", ".join(
+                f"{k}={float(v):.4f}"
+                for k, v in metrics.items()
+                if as_float(v) is not None
+            )
+            if metric_str:
+                segments.append(metric_str)
+        if not segments:
+            return
+        msg = f"{prefix} " + ", ".join(segments)
+        if colorize is not None:
+            msg = colorize(msg)
+        logging.info(msg)
+        return
+
+    title = f"Epoch {epoch}/{epochs} - {split}"
+    if isinstance(loss, numbers.Number):
+        title += f" (loss={float(loss):.4f})"
+
+    table = Table(
+        title=title,
+        box=box.ROUNDED,
+        header_style="bold",
+        title_style="bold",
+    )
+    table.add_column("Task", style="bold")
+
+    include_loss = isinstance(loss, np.ndarray)
+    if include_loss:
+        table.add_column("loss", justify="right")
+
+    metric_names: list[str] = []
+    for task_name in task_order:
+        for metric_name in grouped.get(task_name, {}):
+            if metric_name not in metric_names:
+                metric_names.append(metric_name)
+
+    preferred_order: list[str] = []
+    if isinstance(base_metrics, list):
+        preferred_order = [m for m in base_metrics if m in metric_names]
+    remaining = [m for m in metric_names if m not in preferred_order]
+    metric_names = preferred_order + sorted(remaining)
+
+    for metric_name in metric_names:
+        table.add_column(metric_name, justify="right")
+
+    def fmt(value: float | None) -> str:
+        if value is None:
+            return "-"
+        if np.isnan(value):
+            return "nan"
+        if np.isinf(value):
+            return "inf" if value > 0 else "-inf"
+        return f"{value:.4f}"
+
+    loss_by_task: dict[str, float] = {}
+    if isinstance(loss, np.ndarray):
+        if target_list:
+            for i, task_name in enumerate(target_list):
+                if i < loss.shape[0]:
+                    loss_by_task[task_name] = float(loss[i])
+            if "overall" in task_order and "overall" not in loss_by_task:
+                loss_by_task["overall"] = float(np.sum(loss))
+        elif task_order:
+            for i, task_name in enumerate(task_order):
+                if i < loss.shape[0]:
+                    loss_by_task[task_name] = float(loss[i])
+        else:
+            task_order = ["overall"]
+            loss_by_task["overall"] = float(np.sum(loss))
+
+    if not task_order:
+        task_order = ["__overall__"]
+
+    for task_name in task_order:
+        row: list[str] = [str(task_name)]
+        if include_loss:
+            row.append(fmt(loss_by_task.get(task_name)))
+        for metric_name in metric_names:
+            row.append(fmt(grouped.get(task_name, {}).get(metric_name)))
+        table.add_row(*row)
+
+    Console().print(table)
+
+    record_console = Console(file=io.StringIO(), record=True, width=120)
+    record_console.print(table)
+    table_text = record_console.export_text(styles=False).rstrip()
+
+    root_logger = logging.getLogger()
+    record = root_logger.makeRecord(
+        root_logger.name,
+        logging.INFO,
+        __file__,
+        0,
+        "[MetricsTable]\n" + table_text,
+        args=(),
+        exc_info=None,
+        extra=None,
+    )
+
+    emitted = False
+    for handler in root_logger.handlers:
+        if isinstance(handler, logging.FileHandler):
+            handler.emit(record)
+            emitted = True
+
+    if not emitted:
+        # Fallback: no file handlers configured, use standard logging.
+        root_logger.log(logging.INFO, "[MetricsTable]\n" + table_text)