nextrec 0.4.8__py3-none-any.whl → 0.4.10__py3-none-any.whl

nextrec/utils/console.py

@@ -0,0 +1,371 @@
+"""
+Console and CLI utilities for NextRec.
+
+This module centralizes CLI logging helpers, progress display, and metric tables.
+
+Date: create on 19/12/2025
+Checkpoint: edit on 19/12/2025
+Author: Yang Zhou, zyaztec@gmail.com
+"""
+
+from __future__ import annotations
+
+import io
+import logging
+import numbers
+import os
+import platform
+import sys
+from datetime import datetime, timedelta
+from typing import Any, Callable, Iterable, Mapping, TypeVar
+
+import numpy as np
+from rich import box
+from rich.console import Console
+from rich.progress import (
+    BarColumn,
+    MofNCompleteColumn,
+    Progress,
+    SpinnerColumn,
+    TaskProgressColumn,
+    TextColumn,
+    TimeElapsedColumn,
+    TimeRemainingColumn,
+)
+from rich.table import Table
+from rich.text import Text
+
+from nextrec.utils.feature import as_float, normalize_to_list
+
+T = TypeVar("T")
+
+
+def get_nextrec_version() -> str:
+    """
+    Best-effort version resolver for NextRec.
+
+    Prefer in-repo `nextrec.__version__`, fall back to installed package metadata.
+    """
+    try:
+        from nextrec import __version__  # type: ignore
+
+        if __version__:
+            return str(__version__)
+    except Exception:
+        pass
+
+    try:
+        from importlib.metadata import version
+
+        return version("nextrec")
+    except Exception:
+        return "unknown"
+
+
+def log_startup_info(
+    logger: logging.Logger, *, mode: str, config_path: str | None
+) -> None:
+    """Log a short, user-friendly startup banner."""
+    version = get_nextrec_version()
+    now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
+    lines = [
+        "NextRec CLI",
+        f"- Version: {version}",
+        f"- Time: {now}",
+        f"- Mode: {mode}",
+        f"- Config: {config_path or '(not set)'}",
+        f"- Python: {platform.python_version()} ({sys.executable})",
+        f"- Platform: {platform.system()} {platform.release()} ({platform.machine()})",
+        f"- Workdir: {os.getcwd()}",
+        f"- Command: {' '.join(sys.argv)}",
+    ]
+    for line in lines:
+        logger.info(line)
+
+
+class BlackTimeElapsedColumn(TimeElapsedColumn):
+    def render(self, task) -> Text:
+        elapsed = task.finished_time if task.finished else task.elapsed
+        if elapsed is None:
+            return Text("-:--:--", style="black")
+        delta = timedelta(seconds=max(0, int(elapsed)))
+        return Text(str(delta), style="black")
+
+
+class BlackTimeRemainingColumn(TimeRemainingColumn):
+    def render(self, task) -> Text:
+        if self.elapsed_when_finished and task.finished:
+            task_time = task.finished_time
+        else:
+            task_time = task.time_remaining
+
+        if task.total is None:
+            return Text("", style="black")
+
+        if task_time is None:
+            return Text("--:--" if self.compact else "-:--:--", style="black")
+
+        minutes, seconds = divmod(int(task_time), 60)
+        hours, minutes = divmod(minutes, 60)
+
+        if self.compact and not hours:
+            formatted = f"{minutes:02d}:{seconds:02d}"
+        else:
+            formatted = f"{hours:d}:{minutes:02d}:{seconds:02d}"
+
+        return Text(formatted, style="black")
+
+
+class BlackMofNCompleteColumn(MofNCompleteColumn):
+    def render(self, task) -> Text:
+        completed = int(task.completed)
+        total = int(task.total) if task.total is not None else "?"
+        total_width = len(str(total))
+        return Text(
+            f"{completed:{total_width}d}{self.separator}{total}",
+            style="black",
+        )
+
+
+def progress(
+    iterable: Iterable[T],
+    *,
+    description: str | None = None,
+    total: int | None = None,
+    disable: bool = False,
+) -> Iterable[T]:
+    if disable:
+        for item in iterable:
+            yield item
+        return
+    resolved_total = total
+    if resolved_total is None:
+        try:
+            resolved_total = len(iterable)  # type: ignore[arg-type]
+        except TypeError:
+            resolved_total = None
+
+    progress_bar = Progress(
+        SpinnerColumn(style="black"),
+        TextColumn("{task.description}", style="black"),
+        BarColumn(
+            bar_width=36, style="black", complete_style="black", finished_style="black"
+        ),
+        TaskProgressColumn(style="black"),
+        BlackMofNCompleteColumn(),
+        BlackTimeElapsedColumn(),
+        BlackTimeRemainingColumn(),
+        refresh_per_second=12,
+    )
+
+    task_id = progress_bar.add_task(description or "Working", total=resolved_total)
+    progress_bar.start()
+    try:
+        for item in iterable:
+            yield item
+            progress_bar.advance(task_id, 1)
+    finally:
+        progress_bar.stop()
+
+
+def group_metrics_by_task(
+    metrics: Mapping[str, Any] | None,
+    target_names: list[str] | str | None,
+    default_task_name: str = "overall",
+) -> tuple[list[str], dict[str, dict[str, float]]]:
+    if not metrics:
+        return [], {}
+
+    if isinstance(target_names, str):
+        target_names = [target_names]
+    if not isinstance(target_names, list) or not target_names:
+        target_names = [default_task_name]
+
+    targets_by_len = sorted(target_names, key=len, reverse=True)
+    grouped: dict[str, dict[str, float]] = {}
+    for key, raw_value in metrics.items():
+        value = as_float(raw_value)
+        if value is None:
+            continue
+
+        matched_target: str | None = None
+        metric_name = key
+        for target in targets_by_len:
+            suffix = f"_{target}"
+            if key.endswith(suffix):
+                metric_name = key[: -len(suffix)]
+                matched_target = target
+                break
+
+        if matched_target is None:
+            matched_target = (
+                target_names[0] if len(target_names) == 1 else default_task_name
+            )
+        grouped.setdefault(matched_target, {})[metric_name] = value
+
+    task_order: list[str] = []
+    for target in target_names:
+        if target in grouped:
+            task_order.append(target)
+    for task_name in grouped:
+        if task_name not in task_order:
+            task_order.append(task_name)
+    return task_order, grouped
+
+
+def display_metrics_table(
+    epoch: int,
+    epochs: int,
+    split: str,
+    loss: float | np.ndarray | None,
+    metrics: Mapping[str, Any] | None,
+    target_names: list[str] | str | None,
+    base_metrics: list[str] | None = None,
+    is_main_process: bool = True,
+    colorize: Callable[[str], str] | None = None,
+) -> None:
+    if not is_main_process:
+        return
+
+    target_list = normalize_to_list(target_names)
+    task_order, grouped = group_metrics_by_task(metrics, target_names=target_names)
+
+    if isinstance(loss, np.ndarray) and target_list:
+        # Ensure tasks with losses are shown even when metrics are missing for some targets.
+        normalized_order: list[str] = []
+        for name in target_list:
+            if name not in normalized_order:
+                normalized_order.append(name)
+        for name in task_order:
+            if name not in normalized_order:
+                normalized_order.append(name)
+        task_order = normalized_order
+
+    if Console is None or Table is None or box is None:
+        prefix = f"Epoch {epoch}/{epochs} - {split}:"
+        segments: list[str] = []
+        if isinstance(loss, numbers.Number):
+            segments.append(f"loss={float(loss):.4f}")
+        if task_order and grouped:
+            task_strs: list[str] = []
+            for task_name in task_order:
+                metric_items = grouped.get(task_name, {})
+                if not metric_items:
+                    continue
+                metric_str = ", ".join(
+                    f"{k}={float(v):.4f}" for k, v in metric_items.items()
+                )
+                task_strs.append(f"{task_name}[{metric_str}]")
+            if task_strs:
+                segments.append(", ".join(task_strs))
+        elif metrics:
+            metric_str = ", ".join(
+                f"{k}={float(v):.4f}"
+                for k, v in metrics.items()
+                if as_float(v) is not None
+            )
+            if metric_str:
+                segments.append(metric_str)
+        if not segments:
+            return
+        msg = f"{prefix} " + ", ".join(segments)
+        if colorize is not None:
+            msg = colorize(msg)
+        logging.info(msg)
+        return
+
+    title = f"Epoch {epoch}/{epochs} - {split}"
+    if isinstance(loss, numbers.Number):
+        title += f" (loss={float(loss):.4f})"
+
+    table = Table(
+        title=title,
+        box=box.ROUNDED,
+        header_style="bold",
+        title_style="bold",
+    )
+    table.add_column("Task", style="bold")
+
+    include_loss = isinstance(loss, np.ndarray)
+    if include_loss:
+        table.add_column("loss", justify="right")
+
+    metric_names: list[str] = []
+    for task_name in task_order:
+        for metric_name in grouped.get(task_name, {}):
+            if metric_name not in metric_names:
+                metric_names.append(metric_name)
+
+    preferred_order: list[str] = []
+    if isinstance(base_metrics, list):
+        preferred_order = [m for m in base_metrics if m in metric_names]
+    remaining = [m for m in metric_names if m not in preferred_order]
+    metric_names = preferred_order + sorted(remaining)
+
+    for metric_name in metric_names:
+        table.add_column(metric_name, justify="right")
+
+    def fmt(value: float | None) -> str:
+        if value is None:
+            return "-"
+        if np.isnan(value):
+            return "nan"
+        if np.isinf(value):
+            return "inf" if value > 0 else "-inf"
+        return f"{value:.4f}"
+
+    loss_by_task: dict[str, float] = {}
+    if isinstance(loss, np.ndarray):
+        if target_list:
+            for i, task_name in enumerate(target_list):
+                if i < loss.shape[0]:
+                    loss_by_task[task_name] = float(loss[i])
+            if "overall" in task_order and "overall" not in loss_by_task:
+                loss_by_task["overall"] = float(np.sum(loss))
+        elif task_order:
+            for i, task_name in enumerate(task_order):
+                if i < loss.shape[0]:
+                    loss_by_task[task_name] = float(loss[i])
+        else:
+            task_order = ["overall"]
+            loss_by_task["overall"] = float(np.sum(loss))
+
+    if not task_order:
+        task_order = ["__overall__"]
+
+    for task_name in task_order:
+        row: list[str] = [str(task_name)]
+        if include_loss:
+            row.append(fmt(loss_by_task.get(task_name)))
+        for metric_name in metric_names:
+            row.append(fmt(grouped.get(task_name, {}).get(metric_name)))
+        table.add_row(*row)
+
+    Console().print(table)
+
+    record_console = Console(file=io.StringIO(), record=True, width=120)
+    record_console.print(table)
+    table_text = record_console.export_text(styles=False).rstrip()
+
+    root_logger = logging.getLogger()
+    record = root_logger.makeRecord(
+        root_logger.name,
+        logging.INFO,
+        __file__,
+        0,
+        "[MetricsTable]\n" + table_text,
+        args=(),
+        exc_info=None,
+        extra=None,
+    )
+
+    emitted = False
+    for handler in root_logger.handlers:
+        if isinstance(handler, logging.FileHandler):
+            handler.emit(record)
+            emitted = True
+
+    if not emitted:
+        # Fallback: no file handlers configured, use standard logging.
+        root_logger.log(logging.INFO, "[MetricsTable]\n" + table_text)

nextrec/utils/{synthetic_data.py → data.py}

@@ -1,17 +1,101 @@
 """
-Synthetic Data Generation Utilities
+Data utilities for NextRec.
 
-This module provides utilities for generating synthetic datasets for testing
-and tutorial purposes in the NextRec framework.
+This module provides file I/O helpers and synthetic data generation.
 
-Date: create on 06/12/2025
+Date: create on 19/12/2025
+Checkpoint: edit on 19/12/2025
 Author: Yang Zhou, zyaztec@gmail.com
 """
 
-import torch
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Dict, Generator, List, Optional, Tuple
+
 import numpy as np
 import pandas as pd
-from typing import Optional, Dict, List, Tuple
+import pyarrow.parquet as pq
+import torch
+import yaml
+
+
+def resolve_file_paths(path: str) -> tuple[list[str], str]:
+    """
+    Resolve file or directory path into a sorted list of files and file type.
+
+    Args: path: Path to a file or directory
+    Returns: tuple: (list of file paths, file type)
+    """
+    path_obj = Path(path)
+
+    if path_obj.is_file():
+        file_type = path_obj.suffix.lower().lstrip(".")
+        assert file_type in [
+            "csv",
+            "parquet",
+        ], f"Unsupported file extension: {file_type}"
+        return [str(path_obj)], file_type
+
+    if path_obj.is_dir():
+        collected_files = [p for p in path_obj.iterdir() if p.is_file()]
+        csv_files = [str(p) for p in collected_files if p.suffix.lower() == ".csv"]
+        parquet_files = [
+            str(p) for p in collected_files if p.suffix.lower() == ".parquet"
+        ]
+
+        if csv_files and parquet_files:
+            raise ValueError(
+                "Directory contains both CSV and Parquet files. Please keep a single format."
+            )
+        file_paths = csv_files if csv_files else parquet_files
+        if not file_paths:
+            raise ValueError(f"No CSV or Parquet files found in directory: {path}")
+        file_paths.sort()
+        file_type = "csv" if csv_files else "parquet"
+        return file_paths, file_type
+
+    raise ValueError(f"Invalid path: {path}")
+
+
+def read_table(path: str | Path, data_format: str | None = None) -> pd.DataFrame:
+    data_path = Path(path)
+    fmt = data_format.lower() if data_format else data_path.suffix.lower().lstrip(".")
+    if data_path.is_dir() and not fmt:
+        fmt = "parquet"
+    if fmt in {"parquet", ""}:
+        return pd.read_parquet(data_path)
+    if fmt in {"csv", "txt"}:
+        # Use low_memory=False to avoid mixed-type DtypeWarning on wide CSVs
+        return pd.read_csv(data_path, low_memory=False)
+    raise ValueError(f"Unsupported data format: {data_path}")
+
+
+def load_dataframes(file_paths: list[str], file_type: str) -> list[pd.DataFrame]:
+    return [read_table(fp, file_type) for fp in file_paths]
+
+
+def iter_file_chunks(
+    file_path: str, file_type: str, chunk_size: int
+) -> Generator[pd.DataFrame, None, None]:
+    if file_type == "csv":
+        yield from pd.read_csv(file_path, chunksize=chunk_size)
+        return
+    parquet_file = pq.ParquetFile(file_path)
+    for batch in parquet_file.iter_batches(batch_size=chunk_size):
+        yield batch.to_pandas()
+
+
+def default_output_dir(path: str) -> Path:
+    path_obj = Path(path)
+    if path_obj.is_file():
+        return path_obj.parent / f"{path_obj.stem}_preprocessed"
+    return path_obj.with_name(f"{path_obj.name}_preprocessed")
+
+
+def read_yaml(path: str | Path):
+    with open(path, "r", encoding="utf-8") as file:
+        return yaml.safe_load(file) or {}
 
 
 def generate_ranking_data(
@@ -90,13 +174,14 @@ def generate_ranking_data(
         sequence_vocabs.append(seq_vocab)
 
     if "gender" in data and "dense_0" in data:
+        dense_1 = data.get("dense_1", 0)
         # Complex label generation with feature correlation
         label_probs = 1 / (
             1
             + np.exp(
                 -(
                     data["dense_0"] * 0.3
-                    + data["dense_1"] * 0.2
+                    + dense_1 * 0.2
                     + (data["gender"] - 0.5) * 0.5
                     + np.random.randn(n_samples) * 0.1
                 )
@@ -112,7 +197,7 @@ def generate_ranking_data(
         print(f"Positive rate: {data['label'].mean():.4f}")
 
     # Import here to avoid circular import
-    from nextrec.basic.features import DenseFeature, SparseFeature, SequenceFeature
+    from nextrec.basic.features import DenseFeature, SequenceFeature, SparseFeature
 
     # Create feature definitions
     # Use input_dim for dense features to be compatible with both simple and complex scenarios
@@ -273,7 +358,7 @@ def generate_match_data(
     print(f"Positive rate: {data['label'].mean():.4f}")
 
     # Import here to avoid circular import
-    from nextrec.basic.features import DenseFeature, SparseFeature, SequenceFeature
+    from nextrec.basic.features import DenseFeature, SequenceFeature, SparseFeature
 
     # User dense features
     user_dense_features = [DenseFeature(name="user_age", input_dim=1)]
@@ -413,15 +498,17 @@ def generate_multitask_data(
 
     # Generate multi-task labels with correlation
     # CTR (click) is relatively easier to predict
-    ctr_logits = (
-        data["dense_0"] * 0.3 + data["dense_1"] * 0.2 + np.random.randn(n_samples) * 0.5
-    )
+    dense_0 = data.get("dense_0", 0)
+    dense_1 = data.get("dense_1", 0)
+    dense_2 = data.get("dense_2", 0)
+    dense_3 = data.get("dense_3", 0)
+    ctr_logits = dense_0 * 0.3 + dense_1 * 0.2 + np.random.randn(n_samples) * 0.5
     data["click"] = (1 / (1 + np.exp(-ctr_logits)) > 0.5).astype(np.float32)
 
     # CVR (conversion) depends on click and is harder
     cvr_logits = (
-        data["dense_2"] * 0.2
-        + data["dense_3"] * 0.15
+        dense_2 * 0.2
+        + dense_3 * 0.15
         + data["click"] * 1.5  # Strong dependency on click
         + np.random.randn(n_samples) * 0.8
     )
@@ -441,7 +528,7 @@ def generate_multitask_data(
     print(f"CTCVR rate: {data['ctcvr'].mean():.4f}")
 
     # Import here to avoid circular import
-    from nextrec.basic.features import DenseFeature, SparseFeature, SequenceFeature
+    from nextrec.basic.features import DenseFeature, SequenceFeature, SparseFeature
 
     # Create feature definitions
     dense_features = [

nextrec/utils/feature.py

@@ -2,9 +2,13 @@
 Feature processing utilities for NextRec
 
 Date: create on 03/12/2025
+Checkpoint: edit on 19/12/2025
 Author: Yang Zhou, zyaztec@gmail.com
 """
 
+import numbers
+from typing import Any
+
 
 def normalize_to_list(value: str | list[str] | None) -> list[str]:
     if value is None:
@@ -12,3 +16,14 @@ def normalize_to_list(value: str | list[str] | None) -> list[str]:
     if isinstance(value, str):
         return [value]
     return list(value)
+
+
+def as_float(value: Any) -> float | None:
+    if isinstance(value, numbers.Number):
+        return float(value)
+    if hasattr(value, "item"):
+        try:
+            return float(value.item())
+        except Exception:
+            return None
+    return None