scorebook 0.0.13__py3-none-any.whl → 0.0.15__py3-none-any.whl

scorebook/metrics/f1.py

@@ -0,0 +1,96 @@
+"""F1 metric implementation for Scorebook."""
+
+from typing import Any, Dict, List, Tuple, Union
+
+from sklearn.metrics import f1_score
+
+from scorebook.metrics import MetricBase, scorebook_metric
+
+
+@scorebook_metric
+class F1(MetricBase):
+    """F1 score metric for evaluating model predictions using scikit-learn.
+
+    F1 = 2 * (Precision * Recall) / (Precision + Recall)
+    where:
+    - Precision = TP / (TP + FP)
+    - Recall = TP / (TP + FN)
+
+    This metric can handle both binary and multi-class classification tasks.
+
+    Args:
+        average: The averaging method(s) for multi-class classification.
+            Can be a single string or list of strings:
+            - 'macro': Unweighted mean across labels
+            - 'micro': Global calculation counting total TP, FP, FN
+            - 'weighted': Weighted mean by support
+            - 'all': All three methods simultaneously
+            - List of methods: Calculate multiple methods
+            Defaults to 'macro'.
+    """
+
+    def __init__(self, average: Union[str, List[str]] = "macro", **kwargs: Any) -> None:
+        """Initialize F1 metric with specified averaging method(s).
+
+        Args:
+            average: Averaging method(s) - string or list of strings.
+                Options: 'macro', 'micro', 'weighted', 'all', or a list of methods.
+                Defaults to 'macro'.
+            **kwargs: Additional keyword arguments passed to scikit-learn's f1_score function.
+
+        Raises:
+            ValueError: If average contains invalid methods or combines 'all' with others.
+        """
+        # Normalize to list for validation
+        averages = [average] if isinstance(average, str) else average
+
+        # Validate
+        valid = {"macro", "micro", "weighted", "all"}
+        if not all(a in valid for a in averages):
+            raise ValueError(f"Invalid average method(s). Must be from {valid}.")
+        if len(averages) > 1 and "all" in averages:
+            raise ValueError("'all' cannot be combined with other methods.")
+
+        self.average = average
+        self.kwargs = kwargs
+
+    def score(self, outputs: List[Any], labels: List[Any]) -> Tuple[Dict[str, Any], List[Any]]:
+        """Calculate F1 score between predictions and references using scikit-learn.
+
+        Args:
+            outputs: A list of inference outputs.
+            labels: A list of ground truth labels.
+
+        Returns:
+            Tuple containing:
+                - aggregate_scores (Dict[str, float]): Dictionary with F1 scores
+                  keyed by averaging method (e.g., {"F1 (macro)": 0.85} or
+                  {"F1 (macro)": 0.85, "F1 (micro)": 0.82}).
+                - item_scores (List[bool]): True/False list indicating correct
+                  predictions.
+
+        """
+        # Normalize to list of methods to calculate
+        if isinstance(self.average, str):
+            methods = ["macro", "micro", "weighted"] if self.average == "all" else [self.average]
+        else:
+            methods = self.average
+
+        # Handle empty lists
+        if not outputs:
+            return {f"F1 ({method})": 0.0 for method in methods}, []
+
+        # Calculate F1 score using scikit-learn with configured averaging method
+        # Default zero_division=0 unless overridden in kwargs
+        kwargs = {"zero_division": 0, **self.kwargs}
+
+        # Calculate item scores (correctness of each prediction)
+        item_scores = [output == label for output, label in zip(outputs, labels)]
+
+        # Calculate F1 for each method
+        aggregate_scores = {
+            f"F1 ({method})": f1_score(labels, outputs, average=method, **kwargs)
+            for method in methods
+        }
+
+        return aggregate_scores, item_scores

scorebook/metrics/precision.py

@@ -1,19 +1,94 @@
 """Precision metric implementation for Scorebook."""
 
-from typing import Any, Dict, List, Tuple
+from typing import Any, Dict, List, Tuple, Union
 
-from scorebook.metrics.metric_base import MetricBase
-from scorebook.metrics.metric_registry import MetricRegistry
+from sklearn.metrics import precision_score
 
+from scorebook.metrics import MetricBase, scorebook_metric
 
-@MetricRegistry.register()
+
+@scorebook_metric
 class Precision(MetricBase):
-    """Precision metric for binary classification.
+    """Precision score metric for evaluating model predictions using scikit-learn.
 
     Precision = TP / (TP + FP)
+
+    This metric can handle both binary and multi-class classification tasks.
+
+    Args:
+        average: The averaging method(s) for multi-class classification.
+            Can be a single string or list of strings:
+            - 'macro': Unweighted mean across labels
+            - 'micro': Global calculation counting total TP, FP
+            - 'weighted': Weighted mean by support
+            - 'all': All three methods simultaneously
+            - List of methods: Calculate multiple methods
+            Defaults to 'macro'.
     """
 
-    @staticmethod
-    def score(outputs: List[Any], labels: List[Any]) -> Tuple[Dict[str, Any], List[Any]]:
-        """Not implemented."""
-        raise NotImplementedError("Precision not implemented")
+    def __init__(self, average: Union[str, List[str]] = "macro", **kwargs: Any) -> None:
+        """Initialize Precision metric with specified averaging method(s).
+
+        Args:
+            average: Averaging method(s) - string or list of strings.
+                Options: 'macro', 'micro', 'weighted', 'all', or a list of methods.
+                Defaults to 'macro'.
+            **kwargs: Additional keyword arguments passed to sklearn's precision_score.
+
+        Raises:
+            ValueError: If average contains invalid methods or combines 'all' with others.
+        """
+        # Normalize to list for validation
+        averages = [average] if isinstance(average, str) else average
+
+        # Validate
+        valid = {"macro", "micro", "weighted", "all"}
+        if not all(a in valid for a in averages):
+            raise ValueError(f"Invalid average method(s). Must be from {valid}.")
+        if len(averages) > 1 and "all" in averages:
+            raise ValueError("'all' cannot be combined with other methods.")
+
+        self.average = average
+        self.kwargs = kwargs
+
+    def score(self, outputs: List[Any], labels: List[Any]) -> Tuple[Dict[str, Any], List[Any]]:
+        """Calculate Precision score between predictions and references using scikit-learn.
+
+        Args:
+            outputs: A list of inference outputs.
+            labels: A list of ground truth labels.
+
+        Returns:
+            Tuple containing:
+                - aggregate_scores (Dict[str, float]): Dictionary with Precision scores
+                  keyed by averaging method (e.g., {"Precision (macro)": 0.85} or
+                  {"Precision (macro)": 0.85, "Precision (micro)": 0.82}).
+                - item_scores (List[bool]): True/False list indicating correct
+                  predictions.
+
+        """
+
+        # Normalize to list of methods to calculate
+        if isinstance(self.average, str):
+            methods = ["macro", "micro", "weighted"] if self.average == "all" else [self.average]
+        else:
+            methods = self.average
+
+        # Handle empty lists
+        if not outputs:
+            return {f"Precision ({method})": 0.0 for method in methods}, []
+
+        # Calculate Precision score using scikit-learn with configured averaging method
+        # Default zero_division=0 unless overridden in kwargs
+        kwargs = {"zero_division": 0, **self.kwargs}
+
+        # Calculate item scores (correctness of each prediction)
+        item_scores = [output == label for output, label in zip(outputs, labels)]
+
+        # Calculate Precision for each method
+        aggregate_scores = {
+            f"Precision ({method})": precision_score(labels, outputs, average=method, **kwargs)
+            for method in methods
+        }
+
+        return aggregate_scores, item_scores

scorebook/metrics/recall.py

@@ -0,0 +1,94 @@
+"""Recall metric implementation for Scorebook."""
+
+from typing import Any, Dict, List, Tuple, Union
+
+from sklearn.metrics import recall_score
+
+from scorebook.metrics import MetricBase, scorebook_metric
+
+
+@scorebook_metric
+class Recall(MetricBase):
+    """Recall score metric for evaluating model predictions using scikit-learn.
+
+    Recall = TP / (TP + FN)
+
+    This metric can handle both binary and multi-class classification tasks.
+
+    Args:
+        average: The averaging method(s) for multi-class classification.
+            Can be a single string or list of strings:
+            - 'macro': Unweighted mean across labels
+            - 'micro': Global calculation counting total TP, FP, FN
+            - 'weighted': Weighted mean by support
+            - 'all': All three methods simultaneously
+            - List of methods: Calculate multiple methods
+            Defaults to 'macro'.
+    """
+
+    def __init__(self, average: Union[str, List[str]] = "macro", **kwargs: Any) -> None:
+        """Initialize Recall metric with specified averaging method(s).
+
+        Args:
+            average: Averaging method(s) - string or list of strings.
+                Options: 'macro', 'micro', 'weighted', 'all', or a list of methods.
+                Defaults to 'macro'.
+            **kwargs: Additional keyword arguments passed to sklearn's recall_score.
+
+        Raises:
+            ValueError: If average contains invalid methods or combines 'all' with others.
+        """
+        # Normalize to list for validation
+        averages = [average] if isinstance(average, str) else average
+
+        # Validate
+        valid = {"macro", "micro", "weighted", "all"}
+        if not all(a in valid for a in averages):
+            raise ValueError(f"Invalid average method(s). Must be from {valid}.")
+        if len(averages) > 1 and "all" in averages:
+            raise ValueError("'all' cannot be combined with other methods.")
+
+        self.average = average
+        self.kwargs = kwargs
+
+    def score(self, outputs: List[Any], labels: List[Any]) -> Tuple[Dict[str, Any], List[Any]]:
+        """Calculate Recall score between predictions and references using scikit-learn.
+
+        Args:
+            outputs: A list of inference outputs.
+            labels: A list of ground truth labels.
+
+        Returns:
+            Tuple containing:
+                - aggregate_scores (Dict[str, float]): Dictionary with Recall scores
+                  keyed by averaging method (e.g., {"Recall (macro)": 0.85} or
+                  {"Recall (macro)": 0.85, "Recall (micro)": 0.82}).
+                - item_scores (List[bool]): True/False list indicating correct
+                  predictions.
+
+        """
+
+        # Normalize to list of methods to calculate
+        if isinstance(self.average, str):
+            methods = ["macro", "micro", "weighted"] if self.average == "all" else [self.average]
+        else:
+            methods = self.average
+
+        # Handle empty lists
+        if not outputs:
+            return {f"Recall ({method})": 0.0 for method in methods}, []
+
+        # Calculate Recall score using scikit-learn with configured averaging method
+        # Default zero_division=0 unless overridden in kwargs
+        kwargs = {"zero_division": 0, **self.kwargs}
+
+        # Calculate item scores (correctness of each prediction)
+        item_scores = [output == label for output, label in zip(outputs, labels)]
+
+        # Calculate Recall for each method
+        aggregate_scores = {
+            f"Recall ({method})": recall_score(labels, outputs, average=method, **kwargs)
+            for method in methods
+        }
+
+        return aggregate_scores, item_scores

scorebook/metrics/rouge.py

@@ -0,0 +1,85 @@
+"""ROUGE metric implementation for Scorebook."""
+
+import warnings
+from typing import Any, Dict, List, Optional, Tuple
+
+from rouge_score import rouge_scorer
+
+from scorebook.metrics import MetricBase, scorebook_metric
+
+
+@scorebook_metric
+class ROUGE(MetricBase):
+    """ROUGE metric for evaluating text generation quality.
+
+    ROUGE (Recall-Oriented Understudy for Gisting Evaluation) measures
+    the overlap between generated text and reference text.
+    Returns ROUGE-1 and ROUGE-L F1 scores.
+    """
+
+    def __init__(self, rouge_types: Optional[List[str]] = None, **kwargs: Any) -> None:
+        """Initialize the ROUGE metric.
+
+        Args:
+            rouge_types: List of ROUGE types to calculate (e.g., ["rouge1", "rouge2", "rougeL"]).
+                        Defaults to ["rouge1", "rougeL"].
+            **kwargs: Additional keyword arguments to pass to RougeScorer
+                     (e.g., use_stemmer, split_summaries, tokenizer).
+                     Defaults to use_stemmer=True if not provided.
+        """
+        if rouge_types is None:
+            warnings.warn(
+                "No rouge_types specified, defaulting to ['rouge1', 'rougeL']",
+                UserWarning,
+                stacklevel=2,
+            )
+            rouge_types = ["rouge1", "rougeL"]
+        if "use_stemmer" not in kwargs:
+            warnings.warn(
+                "use_stemmer not specified, defaulting to True",
+                UserWarning,
+                stacklevel=2,
+            )
+            kwargs["use_stemmer"] = True
+        self.rouge_types = rouge_types
+        self.scorer = rouge_scorer.RougeScorer(rouge_types, **kwargs)
+
+    def score(self, outputs: List[Any], labels: List[Any]) -> Tuple[Dict[str, Any], List[Any]]:
+        """Calculate ROUGE scores between predictions and references.
+
+        Args:
+            outputs: A list of generated text outputs.
+            labels: A list of reference text labels.
+
+        Returns:
+            A tuple containing:
+            - aggregate_scores: Dict with average F1 scores for each configured ROUGE type
+            - item_scores: List of dicts with F1 scores for each configured ROUGE type
+        """
+
+        if not outputs:  # Handle empty lists
+            return {rouge_type: 0.0 for rouge_type in self.rouge_types}, []
+
+        # Calculate item scores
+        item_scores = []
+        for output, label in zip(outputs, labels):
+            # Convert to strings if needed
+            output_str = str(output) if output is not None else ""
+            label_str = str(label) if label is not None else ""
+
+            # Calculate ROUGE scores
+            scores = self.scorer.score(output_str, label_str)
+
+            # Extract F1 scores (fmeasure) for all configured rouge types
+            item_score = {
+                rouge_type: scores[rouge_type].fmeasure for rouge_type in self.rouge_types
+            }
+            item_scores.append(item_score)
+
+        # Calculate aggregate scores (average of all items for each rouge type)
+        aggregate_scores = {
+            rouge_type: sum(item[rouge_type] for item in item_scores) / len(item_scores)
+            for rouge_type in self.rouge_types
+        }
+
+        return aggregate_scores, item_scores

scorebook/score/__init__.py

@@ -1,6 +1 @@
 """Score module for computing metrics on pre-computed outputs."""
-
-from scorebook.score._async.score_async import score_async
-from scorebook.score._sync.score import score
-
-__all__ = ["score", "score_async"]

scorebook/score/_async/score_async.py

@@ -1,6 +1,7 @@
 import logging
 from typing import Any, Dict, List, Literal, Optional, Union, cast
 
+from scorebook.dashboard.upload_results import upload_result_async
 from scorebook.exceptions import DataMismatchError, ParameterValidationError
 from scorebook.score.score_helpers import (
     calculate_metric_scores_async,
@@ -8,9 +9,9 @@ from scorebook.score.score_helpers import (
     resolve_metrics,
     validate_items,
 )
-from scorebook.trismik.upload_results import upload_result_async
 from scorebook.types import Metrics
-from scorebook.utils import resolve_show_progress, resolve_upload_results, scoring_progress_context
+from scorebook.utils.common_helpers import resolve_show_progress, resolve_upload_results
+from scorebook.utils.progress_bars import scoring_progress_context
 
 logger = logging.getLogger(__name__)
 

scorebook/score/_sync/score.py

@@ -1,6 +1,7 @@
 import logging
 from typing import Any, Dict, List, Literal, Optional, Union, cast
 
+from scorebook.dashboard.upload_results import upload_result
 from scorebook.exceptions import DataMismatchError, ParameterValidationError
 from scorebook.score.score_helpers import (
     calculate_metric_scores,
@@ -8,9 +9,9 @@ from scorebook.score.score_helpers import (
     resolve_metrics,
     validate_items,
 )
-from scorebook.trismik.upload_results import upload_result
 from scorebook.types import Metrics
-from scorebook.utils import resolve_show_progress, resolve_upload_results, scoring_progress_context
+from scorebook.utils.common_helpers import resolve_show_progress, resolve_upload_results
+from scorebook.utils.progress_bars import scoring_progress_context
 
 logger = logging.getLogger(__name__)
 

scorebook/score/score_helpers.py

@@ -4,10 +4,10 @@ import logging
 from typing import Any, Dict, List, Mapping, Optional, Type, Union
 
 from scorebook.exceptions import DataMismatchError, ParameterValidationError
-from scorebook.metrics.metric_base import MetricBase
-from scorebook.metrics.metric_registry import MetricRegistry
+from scorebook.metrics.core.metric_base import MetricBase
+from scorebook.metrics.core.metric_registry import MetricRegistry
 from scorebook.types import MetricScore
-from scorebook.utils import is_awaitable
+from scorebook.utils.async_utils import is_awaitable
 
 logger = logging.getLogger(__name__)
 
@@ -89,7 +89,7 @@ async def calculate_metric_scores_async(
     for metric in metrics:
 
         if progress_bar is not None:
-            progress_bar.set_current_metric(metric.name)
+            progress_bar.set_postfix(metric=metric.name)
 
         if is_awaitable(metric.score):
             aggregate_scores, item_scores = await metric.score(outputs, labels)
@@ -134,7 +134,7 @@ def calculate_metric_scores(
     for metric in metrics:
 
         if progress_bar is not None:
-            progress_bar.set_current_metric(metric.name)
+            progress_bar.set_postfix(metric=metric.name)
 
         if is_awaitable(metric.score):
             raise ParameterValidationError(
@@ -164,18 +164,27 @@ def format_results(
     hyperparameters = hyperparameters or {}
     dataset_name = dataset_name or "scored_items"
 
+    # Detect key collisions across all metrics (for both aggregate and item scores)
+    all_keys: Dict[str, set] = {}
+    for metric_score in metric_scores:
+        for key in metric_score.aggregate_scores.keys():
+            all_keys.setdefault(key, set()).add(metric_score.metric_name)
+        # Also check item_scores keys if they are dicts
+        if metric_score.item_scores and isinstance(metric_score.item_scores[0], dict):
+            for key in metric_score.item_scores[0].keys():
+                all_keys.setdefault(key, set()).add(metric_score.metric_name)
+    colliding_keys = {k for k, metrics in all_keys.items() if len(metrics) > 1}
+
     # Build aggregate results
-    aggregate_result = {
+    aggregate_result: Dict[str, Any] = {
         "dataset": dataset_name,
         **hyperparameters,
     }
 
-    # Add aggregate scores from metrics
+    # Add aggregate scores from metrics (flat, with suffix on collision)
     for metric_score in metric_scores:
         for key, value in metric_score.aggregate_scores.items():
-            score_key = (
-                key if key == metric_score.metric_name else f"{metric_score.metric_name}_{key}"
-            )
+            score_key = f"{key}_{metric_score.metric_name}" if key in colliding_keys else key
             aggregate_result[score_key] = value
 
     # Build item results
@@ -193,10 +202,18 @@ def format_results(
         if inputs is not None and inputs[idx] is not None:
             item_result["input"] = inputs[idx]
 
-        # Add item-level metric scores
+        # Add item-level metric scores (flat, with suffix on collision)
         for metric_score in metric_scores:
             if idx < len(metric_score.item_scores):
-                item_result[metric_score.metric_name] = metric_score.item_scores[idx]
+                item_scores = metric_score.item_scores[idx]
+                if isinstance(item_scores, dict):
+                    for key, value in item_scores.items():
+                        score_key = (
+                            f"{key}_{metric_score.metric_name}" if key in colliding_keys else key
+                        )
+                        item_result[score_key] = value
+                else:
+                    item_result[metric_score.metric_name] = item_scores
 
         item_results.append(item_result)
 

scorebook/types.py

@@ -3,12 +3,12 @@
 from dataclasses import dataclass
 from typing import Any, Dict, List, Optional, Sequence, Type, Union
 
-from scorebook.eval_datasets import EvalDataset
-from scorebook.metrics.metric_base import MetricBase
+from scorebook.eval_datasets.eval_dataset import EvalDataset
+from scorebook.metrics.core.metric_base import MetricBase
 
 # Type alias for metrics parameter
 Metrics = Union[
-    str, "MetricBase", Type["MetricBase"], Sequence[Union[str, "MetricBase", Type["MetricBase"]]]
+    str, MetricBase, Type[MetricBase], Sequence[Union[str, MetricBase, Type[MetricBase]]]
 ]
 
 

scorebook/utils/__init__.py

@@ -1,23 +1 @@
 """Utility functions and common helpers for the Scorebook framework."""
-
-from contextlib import nullcontext
-
-from scorebook.utils.async_utils import async_nullcontext, is_awaitable
-from scorebook.utils.common_helpers import resolve_show_progress, resolve_upload_results
-from scorebook.utils.io_helpers import validate_path
-from scorebook.utils.progress_bars import evaluation_progress_context, scoring_progress_context
-from scorebook.utils.render_template import render_template
-from scorebook.utils.transform_helpers import expand_dict
-
-__all__ = [
-    "async_nullcontext",
-    "nullcontext",
-    "is_awaitable",
-    "resolve_show_progress",
-    "resolve_upload_results",
-    "validate_path",
-    "expand_dict",
-    "evaluation_progress_context",
-    "scoring_progress_context",
-    "render_template",
-]

scorebook/utils/common_helpers.py

@@ -17,7 +17,7 @@ def resolve_upload_results(upload_results: Union[Literal["auto"], bool]) -> bool
         bool: Whether to upload results to Trismik
     """
     if upload_results == "auto":
-        from scorebook.trismik.credentials import get_token
+        from scorebook.dashboard.credentials import get_token
 
         upload_results = get_token() is not None
         logger.debug("Auto upload results resolved to: %s", upload_results)

scorebook/utils/mock_llm/__init__.py

@@ -0,0 +1,41 @@
+"""Mock LLM utilities for testing and demonstrations."""
+
+import json
+import random
+from pathlib import Path
+from typing import Any, List
+
+# Load the mock data once at module initialization
+_DATA_PATH = Path(__file__).parent / "data" / "mock_llm_data.json"
+with open(_DATA_PATH, "r", encoding="utf-8") as f:
+    _MOCK_DATA = json.load(f)
+
+
+def mock_llm(inputs: List[Any], **hyperparameters: Any) -> List[str]:
+    """Mock LLM that returns answers based on pre-recorded accuracy data."""
+
+    results = []
+    all_choices = ["A", "B", "C", "D", "E"]
+
+    for item in inputs:
+        item_id = item["id"]
+
+        # Look up the item in our mock data
+        if item_id not in _MOCK_DATA:
+            # If item not found, return random answer
+            results.append(random.choice(all_choices))
+            continue
+
+        item_data = _MOCK_DATA[item_id]
+        correct_answer = item_data["answer"]
+        was_accurate = item_data["accuracy"]
+
+        if was_accurate:
+            # Return the correct answer
+            results.append(correct_answer)
+        else:
+            # Return a random incorrect answer
+            incorrect_choices = [choice for choice in all_choices if choice != correct_answer]
+            results.append(random.choice(incorrect_choices))
+
+    return results