model-eval-toolkit 0.1.0__py3-none-any.whl

evalreport/core/task_inference.py

@@ -0,0 +1,180 @@
+from __future__ import annotations
+from typing import Any, Iterable, Optional
+
+import numpy as np
+
+
+def infer_task(
+    *,
+    y_true: Optional[Iterable[Any]] = None,
+    y_pred: Optional[Iterable[Any]] = None,
+    X: Optional[Iterable[Any]] = None,
+    timestamps: Optional[Iterable[Any]] = None,
+    embeddings: Optional[Iterable[Any]] = None,
+    y_prob: Optional[Iterable[Any]] = None,
+) -> str:
+    """Infer ML task type from provided inputs.
+
+    Notes
+    -----
+    This is a heuristic v0.1 inference intended to be *good enough* for common
+    scenarios. Power users should still prefer `task="classification"` /
+    `task="regression"` / etc. when they know the correct task.
+    """
+    if timestamps is not None:
+        return "timeseries"
+    if embeddings is not None:
+        return "clustering"
+
+    def _is_seq_of_strings(x: Any) -> bool:
+        if x is None:
+            return False
+        try:
+            seq = list(x)
+            if not seq:
+                return False
+            return all(isinstance(v, str) for v in seq)
+        except Exception:
+            return False
+
+    def _avg_tokens(seq: Iterable[Any]) -> float:
+        vals = list(seq)
+        if not vals:
+            return 0.0
+        total = 0
+        n = 0
+        for v in vals:
+            if v is None:
+                continue
+            s = str(v)
+            total += len(s.split())
+            n += 1
+        return (total / max(1, n)) if n else 0.0
+
+    def _looks_like_text_generation(y_t: Any, y_p: Any) -> bool:
+        # Both sides are strings; generation outputs tend to be longer.
+        if not (_is_seq_of_strings(y_t) and _is_seq_of_strings(y_p)):
+            return False
+        avg_t = _avg_tokens(y_t)
+        avg_p = _avg_tokens(y_p)
+        # Conservative threshold: labels like "pos"/"neg" usually have ~1 token.
+        return (avg_t > 3.0) or (avg_p > 3.0)
+
+    def _looks_like_text_classification(y_t: Any, y_p: Any) -> bool:
+        if not (_is_seq_of_strings(y_t) and _is_seq_of_strings(y_p)):
+            return False
+        # If they are strings but short, treat as classification labels.
+        avg_t = _avg_tokens(y_t)
+        avg_p = _avg_tokens(y_p)
+        return (avg_t <= 3.0) and (avg_p <= 3.0)
+
+    def _looks_like_detection(y_t: Any, y_p: Any) -> bool:
+        # Expect: list of images -> list of dict boxes with 'bbox'
+        try:
+            gt = list(y_t) if y_t is not None else None
+            pr = list(y_p) if y_p is not None else None
+            if not gt or not pr:
+                return False
+            if not isinstance(gt[0], (list, tuple)) or not isinstance(pr[0], (list, tuple)):
+                return False
+            # Check first non-empty box dict
+            for boxes in (gt[0], pr[0]):
+                if not boxes:
+                    continue
+                first = boxes[0]
+                if isinstance(first, dict) and "bbox" in first:
+                    return True
+            return False
+        except Exception:
+            return False
+
+    def _looks_like_segmentation(y_t: Any, y_p: Any) -> bool:
+        # Expect: masks with at least 2D structure (H,W) or (N,H,W)
+        try:
+            gt = np.asarray(list(y_t)) if y_t is not None else None
+            pr = np.asarray(list(y_p)) if y_p is not None else None
+            if gt is None or pr is None:
+                return False
+            # classification labels are 1D; segmentation should be 2D or 3D
+            return gt.ndim >= 2 and pr.ndim >= 2
+        except Exception:
+            return False
+
+    def _looks_like_recommendation(y_t: Any, y_p: Any) -> bool:
+        # Expect per-user relevant items + ranked lists:
+        # y_true: list[ list[item] ] (or set)
+        # y_pred: list[ list[item] ]
+        try:
+            gt = list(y_t) if y_t is not None else None
+            pr = list(y_p) if y_p is not None else None
+            if not gt or not pr:
+                return False
+            if not isinstance(gt[0], (list, tuple, set)):
+                return False
+            if not isinstance(pr[0], (list, tuple)):
+                return False
+            return True
+        except Exception:
+            return False
+
+    def _looks_like_discrete_labels(arr: np.ndarray) -> bool:
+        # If labels are integer-like (or small number of unique values),
+        # likely classification/clustering.
+        if arr.size == 0:
+            return False
+        if arr.dtype.kind in {"i", "u", "b"}:
+            return True
+        if arr.dtype.kind == "f":
+            # float but close to integers?
+            rounded = np.rint(arr)
+            if np.allclose(arr, rounded, atol=1e-8):
+                return True
+        # otherwise: not enough evidence
+        return False
+
+    # Vision --------------------------------------------------------------
+    if y_true is not None and y_pred is not None:
+        if _looks_like_detection(y_true, y_pred):
+            return "detection"
+        if _looks_like_segmentation(y_true, y_pred):
+            return "segmentation"
+        if _looks_like_recommendation(y_true, y_pred):
+            return "recommendation"
+
+        # NLP --------------------------------------------------------------
+        if _looks_like_text_generation(y_true, y_pred):
+            return "text_generation"
+        if _looks_like_text_classification(y_true, y_pred):
+            return "text_classification"
+
+    # Clustering ----------------------------------------------------------
+    if X is not None and y_true is None and y_pred is not None:
+        try:
+            labels = np.asarray(list(y_pred))
+            if _looks_like_discrete_labels(labels) and labels.ndim == 1:
+                return "clustering"
+        except Exception:
+            pass
+
+    # Regression vs classification ---------------------------------------
+    if y_true is not None and y_pred is not None:
+        try:
+            y_true_arr = np.asarray(list(y_true))
+            if y_true_arr.ndim == 0:
+                return "classification"
+
+            if y_true_arr.dtype.kind == "f":
+                # If float targets look integer-like, treat as classification labels
+                # only when predictions are also integer-like (otherwise it's regression).
+                y_pred_arr = np.asarray(list(y_pred))
+                if _looks_like_discrete_labels(y_true_arr) and _looks_like_discrete_labels(y_pred_arr):
+                    return "classification"
+                return "regression"
+        except Exception:  # pragma: no cover - heuristic fallback
+            pass
+
+        # Non-float targets default to classification.
+        return "classification"
+
+    return "classification"
+

evalreport/nlp/__init__.py

@@ -0,0 +1,5 @@
+from .text_classification import TextClassificationReport
+from .text_generation import TextGenerationReport
+
+__all__ = ["TextClassificationReport", "TextGenerationReport"]
+

evalreport/nlp/text_classification.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Iterable, Optional, Sequence
+
+from ..classification.report import ClassificationReport
+
+
+@dataclass
+class TextClassificationReport(ClassificationReport):
+    """Text classification report.
+
+    Currently reuses the core ClassificationReport logic (metrics, plots, insights)
+    and can be extended later with token-level error analysis.
+    """
+
+    y_true: Optional[Iterable[Any]] = None
+    y_pred: Optional[Iterable[Any]] = None
+    y_prob: Optional[Iterable[Any]] = None
+    labels: Optional[Sequence[Any]] = None
+

evalreport/nlp/text_generation.py

@@ -0,0 +1,202 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Iterable, List, Optional, Sequence
+
+import matplotlib
+
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+import numpy as np
+
+from ..core.base_report import BaseReport
+
+
+def _as_str_list(x: Optional[Iterable[Any]]) -> Optional[List[str]]:
+    if x is None:
+        return None
+    return ["" if v is None else str(v) for v in x]
+
+
+def _tokenize(text: str) -> List[str]:
+    # Minimal tokenizer: whitespace split. Keeps v0.1 dependency-light.
+    return [t for t in text.strip().split() if t]
+
+
+def _ngram_counts(tokens: List[str], n: int) -> dict[tuple[str, ...], int]:
+    counts: dict[tuple[str, ...], int] = {}
+    if n <= 0:
+        return counts
+    for i in range(0, max(0, len(tokens) - n + 1)):
+        ng = tuple(tokens[i : i + n])
+        counts[ng] = counts.get(ng, 0) + 1
+    return counts
+
+
+def _bleu_like(references: Sequence[str], predictions: Sequence[str], max_n: int = 4) -> float:
+    """A lightweight BLEU-like score (not sacreBLEU).
+
+    Computes clipped n-gram precisions with a brevity penalty and uniform weights.
+    Intended as a reasonable v0.1 metric without extra dependencies.
+    """
+    # corpus n-gram stats
+    clipped_hits = np.zeros(max_n, dtype=float)
+    total_preds = np.zeros(max_n, dtype=float)
+    ref_len = 0
+    pred_len = 0
+
+    for ref, pred in zip(references, predictions):
+        ref_toks = _tokenize(ref)
+        pred_toks = _tokenize(pred)
+        ref_len += len(ref_toks)
+        pred_len += len(pred_toks)
+        for n in range(1, max_n + 1):
+            ref_counts = _ngram_counts(ref_toks, n)
+            pred_counts = _ngram_counts(pred_toks, n)
+            total_preds[n - 1] += sum(pred_counts.values())
+            for ng, c in pred_counts.items():
+                clipped_hits[n - 1] += min(c, ref_counts.get(ng, 0))
+
+    precisions = []
+    for n in range(max_n):
+        if total_preds[n] == 0:
+            precisions.append(0.0)
+        else:
+            precisions.append(float(clipped_hits[n] / total_preds[n]))
+
+    # geometric mean with smoothing for zeros
+    eps = 1e-12
+    log_p = np.mean([np.log(max(p, eps)) for p in precisions])
+    geo_mean = float(np.exp(log_p))
+
+    # brevity penalty
+    if pred_len == 0:
+        return 0.0
+    bp = 1.0 if pred_len > ref_len else float(np.exp(1.0 - (ref_len / max(1, pred_len))))
+    return float(bp * geo_mean)
+
+
+def _rouge_l_like(references: Sequence[str], predictions: Sequence[str]) -> float:
+    """Lightweight ROUGE-L F1 over whitespace tokens (corpus average)."""
+
+    def lcs(a: List[str], b: List[str]) -> int:
+        # DP LCS length
+        dp = np.zeros((len(a) + 1, len(b) + 1), dtype=int)
+        for i in range(1, len(a) + 1):
+            for j in range(1, len(b) + 1):
+                if a[i - 1] == b[j - 1]:
+                    dp[i, j] = dp[i - 1, j - 1] + 1
+                else:
+                    dp[i, j] = max(dp[i - 1, j], dp[i, j - 1])
+        return int(dp[len(a), len(b)])
+
+    scores = []
+    for ref, pred in zip(references, predictions):
+        r = _tokenize(ref)
+        p = _tokenize(pred)
+        if not r and not p:
+            scores.append(1.0)
+            continue
+        if not r or not p:
+            scores.append(0.0)
+            continue
+        l = lcs(r, p)
+        prec = l / max(1, len(p))
+        rec = l / max(1, len(r))
+        if prec + rec == 0:
+            scores.append(0.0)
+        else:
+            scores.append(2 * prec * rec / (prec + rec))
+    return float(np.mean(scores)) if scores else 0.0
+
+
+@dataclass
+class TextGenerationReport(BaseReport):
+    references: Optional[Iterable[Any]] = None
+    predictions: Optional[Iterable[Any]] = None
+
+    def _compute_metrics(self) -> None:
+        refs = _as_str_list(self.references)
+        preds = _as_str_list(self.predictions)
+        if refs is None or preds is None:
+            raise ValueError("TextGenerationReport requires references and predictions.")
+        if len(refs) != len(preds):
+            raise ValueError("TextGenerationReport requires references and predictions of equal length.")
+
+        bleu = _bleu_like(refs, preds)
+        rouge_l = _rouge_l_like(refs, preds)
+
+        # Simple lexical overlap ratio (Jaccard over tokens) as a semantic-gap proxy
+        overlaps = []
+        for r, p in zip(refs, preds):
+            rt = set(_tokenize(r))
+            pt = set(_tokenize(p))
+            denom = len(rt | pt)
+            overlaps.append(1.0 if denom == 0 else (len(rt & pt) / denom))
+        jaccard = float(np.mean(overlaps)) if overlaps else 0.0
+
+        self.metrics.update(
+            {
+                "bleu_like": float(bleu),
+                "rouge_l_f1_like": float(rouge_l),
+                "token_jaccard": float(jaccard),
+                "num_samples": int(len(refs)),
+            }
+        )
+
+
+        self._cached_scores = {
+            "bleu_like": [bleu],  # corpus-level; keep structure for plots
+            "rouge_l_f1_like": [rouge_l],
+            "token_jaccard": overlaps,
+        }
+
+    def _generate_plots(self) -> None:
+        root = self.output_dir or Path("reports")
+        plot_dir = root / "evalreport_plots"
+        plot_dir.mkdir(parents=True, exist_ok=True)
+
+        plots: dict[str, str] = {}
+
+        # Distribution of token_jaccard per sample
+        try:
+            vals = np.asarray(self._cached_scores.get("token_jaccard", []), dtype=float)
+            if vals.size > 0:
+                plt.figure(figsize=(5, 3.5))
+                plt.hist(vals, bins=20, alpha=0.85)
+                plt.xlabel("Token Jaccard overlap")
+                plt.ylabel("Count")
+                plt.title("Generation overlap distribution")
+                path = plot_dir / "nlp_generation_overlap_distribution.png"
+                plt.tight_layout()
+                plt.savefig(path)
+                plt.close()
+                plots["overlap_distribution"] = str(path)
+        except Exception:
+            pass
+
+        self.plots = plots
+
+    def _generate_insights(self) -> None:
+        insights: List[str] = []
+        j = self.metrics.get("token_jaccard")
+        b = self.metrics.get("bleu_like")
+        r = self.metrics.get("rouge_l_f1_like")
+
+        if isinstance(j, (int, float)) and j < 0.2:
+            insights.append("Low lexical overlap; model outputs may be paraphrasing heavily or drifting semantically.")
+        if isinstance(b, (int, float)) and b < 0.1 and isinstance(r, (int, float)) and r < 0.2:
+            insights.append("Very low overlap-based scores; check formatting, tokenization, or reference/prediction alignment.")
+
+        self.insights = insights
+
+        self.metric_descriptions.update(
+            {
+                "bleu_like": "N-gram overlap score with brevity penalty (lightweight BLEU-like). Higher is better.",
+                "rouge_l_f1_like": "Sequence overlap based on LCS (lightweight ROUGE-L F1). Higher is better.",
+                "token_jaccard": "Token-set Jaccard overlap between prediction and reference (lexical overlap proxy).",
+                "num_samples": "Number of evaluated reference/prediction pairs.",
+            }
+        )
+

evalreport/ranking/__init__.py

@@ -0,0 +1,3 @@
+from .report import RankingReport
+
+__all__ = ["RankingReport"]

evalreport/ranking/report.py

@@ -0,0 +1,274 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from math import log2
+from pathlib import Path
+from typing import Any, Iterable, List, Optional, Sequence, Set, Tuple
+
+import matplotlib
+
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+import numpy as np
+
+from ..core.base_report import BaseReport
+
+
+def _to_set(x: Iterable[Any]) -> Set[Any]:
+    return set(x)
+
+
+def _to_list(x: Iterable[Any]) -> List[Any]:
+    return list(x)
+
+
+def precision_at_k(relevant: Set[Any], ranked: Sequence[Any], k: int) -> float:
+    if k <= 0:
+        return 0.0
+    top = ranked[:k]
+    if not top:
+        return 0.0
+    hits = sum(1 for item in top if item in relevant)
+    return hits / float(k)
+
+
+def recall_at_k(relevant: Set[Any], ranked: Sequence[Any], k: int) -> float:
+    if not relevant:
+        return 0.0
+    top = ranked[:k]
+    hits = sum(1 for item in top if item in relevant)
+    return hits / float(len(relevant))
+
+
+def hit_rate_at_k(relevant: Set[Any], ranked: Sequence[Any], k: int) -> float:
+    if k <= 0:
+        return 0.0
+    top = ranked[:k]
+    return 1.0 if any(item in relevant for item in top) else 0.0
+
+
+def average_precision(relevant: Set[Any], ranked: Sequence[Any]) -> float:
+    """Mean average precision for one user (binary relevance)."""
+    if not relevant:
+        return 0.0
+    hits = 0
+    prec_sum = 0.0
+    for i, item in enumerate(ranked, start=1):
+        if item in relevant:
+            hits += 1
+            prec_sum += hits / float(i)
+    return prec_sum / float(len(relevant))
+
+
+def dcg_at_k(relevant: Set[Any], ranked: Sequence[Any], k: int) -> float:
+    dcg = 0.0
+    for i, item in enumerate(ranked[:k], start=1):
+        rel = 1.0 if item in relevant else 0.0
+        dcg += rel / log2(i + 1)
+    return dcg
+
+
+def idcg_at_k(relevant: Set[Any], k: int) -> float:
+    """Ideal DCG when all |R| relevant items are at the top (binary)."""
+    n_rel = min(len(relevant), k)
+    if n_rel == 0:
+        return 0.0
+    return sum(1.0 / log2(i + 1) for i in range(1, n_rel + 1))
+
+
+def ndcg_at_k(relevant: Set[Any], ranked: Sequence[Any], k: int) -> float:
+    ideal = idcg_at_k(relevant, k)
+    if ideal <= 0:
+        return 0.0
+    return dcg_at_k(relevant, ranked, k) / ideal
+
+
+@dataclass
+class RankingReport(BaseReport):
+    """Recommendation / ranking evaluation (list-wise per user or query).
+
+    Parameters
+    ----------
+    relevant :
+        Per-user (or per-query) ground-truth relevant items. Iterable of iterables.
+    ranked :
+        Per-user ranked recommendation lists (best first). Same length as ``relevant``.
+    k_values :
+        K cutoffs for P@K, R@K, NDCG@K, Hit@K.
+    """
+
+    relevant: Optional[Iterable[Iterable[Any]]] = None
+    ranked: Optional[Iterable[Iterable[Any]]] = None
+    k_values: Tuple[int, ...] = (1, 5, 10)
+
+    def _normalize_inputs(self) -> Tuple[List[Set[Any]], List[List[Any]]]:
+        if self.relevant is None or self.ranked is None:
+            raise ValueError("RankingReport requires `relevant` and `ranked` (same number of users/queries).")
+        rel_list = [_to_set(r) for r in self.relevant]
+        rank_list = [_to_list(r) for r in self.ranked]
+        if len(rel_list) != len(rank_list):
+            raise ValueError("`relevant` and `ranked` must have the same length (one list per user/query).")
+        if len(rel_list) == 0:
+            raise ValueError("RankingReport requires at least one user/query.")
+        return rel_list, rank_list
+
+    def _compute_metrics(self) -> None:
+        rel_list, rank_list = self._normalize_inputs()
+        n_users = len(rel_list)
+
+        # Per-K aggregates
+        ks = sorted(set(max(1, int(k)) for k in self.k_values))
+        max_k = max(ks)
+
+        mean_p: dict[str, float] = {}
+        mean_r: dict[str, float] = {}
+        mean_ndcg: dict[str, float] = {}
+        mean_hit: dict[str, float] = {}
+
+        for k in ks:
+            ps, rs, nds, hs = [], [], [], []
+            for rel, ranked in zip(rel_list, rank_list):
+                ps.append(precision_at_k(rel, ranked, k))
+                rs.append(recall_at_k(rel, ranked, k))
+                nds.append(ndcg_at_k(rel, ranked, k))
+                hs.append(hit_rate_at_k(rel, ranked, k))
+            mean_p[f"precision_at_{k}"] = float(np.mean(ps))
+            mean_r[f"recall_at_{k}"] = float(np.mean(rs))
+            mean_ndcg[f"ndcg_at_{k}"] = float(np.mean(nds))
+            mean_hit[f"hit_rate_at_{k}"] = float(np.mean(hs))
+
+        # MAP (mean of per-user AP)
+        aps = [average_precision(rel, ranked) for rel, ranked in zip(rel_list, rank_list)]
+        map_score = float(np.mean(aps))
+
+        self.metrics.update(
+            {
+                "map": map_score,
+                "num_users": n_users,
+                **mean_p,
+                **mean_r,
+                **mean_ndcg,
+                **mean_hit,
+            }
+        )
+
+        # Descriptions for every metric key (HTML/PDF “Metric reference” section).
+        self.metric_descriptions.update(
+            {
+                "map": (
+                    "Mean Average Precision (MAP): average of per-user Average Precision, "
+                    "where AP rewards placing relevant items higher in the ranked list (binary relevance per item)."
+                ),
+                "num_users": "Number of users or queries in the evaluation set (one ranked list per row).",
+            }
+        )
+        for k in ks:
+            self.metric_descriptions[f"precision_at_{k}"] = (
+                f"Mean Precision@{k}: averaged across users—of the top-{k} recommendations, "
+                f"what fraction are relevant (|relevant ∩ top-{k}| / {k})."
+            )
+            self.metric_descriptions[f"recall_at_{k}"] = (
+                f"Mean Recall@{k}: averaged across users—what fraction of that user’s relevant items "
+                f"appear in the top-{k} list (|relevant ∩ top-{k}| / |relevant|; 0 if no relevant items)."
+            )
+            self.metric_descriptions[f"ndcg_at_{k}"] = (
+                f"Mean NDCG@{k}: averaged across users—normalized Discounted Cumulative Gain at rank {k}, "
+                f"comparing the ranked list to an ideal that puts all relevant items first (binary relevance)."
+            )
+            self.metric_descriptions[f"hit_rate_at_{k}"] = (
+                f"Mean Hit Rate@{k}: averaged across users—fraction of users who have at least one relevant item "
+                f"in their top-{k} recommendations."
+            )
+
+        self._cached_ks = ks
+        self._cached_max_k = max_k
+        self._cached_rel = rel_list
+        self._cached_rank = rank_list
+
+    def _generate_plots(self) -> None:
+        root = self.output_dir or Path("reports")
+        plot_dir = root / "evalreport_plots"
+        plot_dir.mkdir(parents=True, exist_ok=True)
+        plots: dict[str, str] = {}
+
+        ks = getattr(self, "_cached_ks", [])
+        rel_list = getattr(self, "_cached_rel", None)
+        rank_list = getattr(self, "_cached_rank", None)
+        if not ks or rel_list is None or rank_list is None:
+            self.plots = plots
+            return
+
+        # Precision@K curve
+        try:
+            precs = [self.metrics.get(f"precision_at_{k}", 0.0) for k in ks]
+            plt.figure(figsize=(5.5, 4))
+            plt.plot(ks, precs, marker="o", linewidth=2)
+            plt.xlabel("K")
+            plt.ylabel("Mean Precision@K")
+            plt.title("Precision@K curve")
+            plt.ylim(0, 1.05)
+            plt.grid(True, alpha=0.3)
+            path = plot_dir / "ranking_precision_at_k.png"
+            plt.tight_layout()
+            plt.savefig(path)
+            plt.close()
+            plots["precision_at_k_curve"] = str(path)
+        except Exception:
+            pass
+
+        # Cumulative gain: mean cumulative relevant count vs rank (up to max_k)
+        try:
+            max_k = int(getattr(self, "_cached_max_k", max(ks)))
+            cum = np.zeros(max_k, dtype=float)
+            for rel, ranked in zip(rel_list, rank_list):
+                for pos in range(max_k):
+                    if pos < len(ranked) and ranked[pos] in rel:
+                        cum[pos] += 1.0
+            cum_mean = np.cumsum(cum) / float(len(rel_list))
+            ranks = np.arange(1, max_k + 1)
+            plt.figure(figsize=(5.5, 4))
+            plt.plot(ranks, cum_mean, marker="s", linewidth=2, color="#2ca02c")
+            plt.xlabel("Rank cutoff")
+            plt.ylabel("Mean cumulative relevant retrieved")
+            plt.title("Cumulative gain (mean over users)")
+            plt.grid(True, alpha=0.3)
+            path = plot_dir / "ranking_cumulative_gain.png"
+            plt.tight_layout()
+            plt.savefig(path)
+            plt.close()
+            plots["cumulative_gain"] = str(path)
+        except Exception:
+            pass
+
+        self.plots = plots
+
+    def _generate_insights(self) -> None:
+        insights: List[str] = []
+        ks = getattr(self, "_cached_ks", [])
+        if len(ks) >= 2:
+            k_small, k_large = ks[0], ks[-1]
+            p_small = self.metrics.get(f"precision_at_{k_small}")
+            p_large = self.metrics.get(f"precision_at_{k_large}")
+            if isinstance(p_small, (int, float)) and isinstance(p_large, (int, float)):
+                if p_large + 1e-6 < p_small * 0.85:
+                    insights.append(
+                        f"Precision@K drops from {k_small} to {k_large}; quality may degrade in deeper ranks."
+                    )
+
+        # Long-tail: users with many relevant items vs few
+        rel_list = getattr(self, "_cached_rel", [])
+        if rel_list:
+            sizes = [len(r) for r in rel_list]
+            if sizes:
+                med = float(np.median(sizes))
+                mx = max(sizes)
+                if mx > 0 and med > 0 and mx / med >= 5:
+                    insights.append(
+                        "Large spread in number of relevant items per user; check long-tail users separately."
+                    )
+
+        map_score = self.metrics.get("map")
+        if isinstance(map_score, (int, float)) and map_score < 0.1:
+            insights.append("Low MAP; consider improving candidate generation, re-ranking, or relevance signals.")
+
+        self.insights = insights

evalreport/regression/__init__.py

@@ -0,0 +1,4 @@
+from .report import RegressionReport
+
+__all__ = ["RegressionReport"]
+