scorebook 0.0.13__py3-none-any.whl → 0.0.15__py3-none-any.whl

scorebook/metrics/README.md

@@ -0,0 +1,121 @@
+# Adding Metrics to Scorebook
+
+This guide explains how to add new metrics to Scorebook.
+
+## Quick Start
+
+1. Create a metric file: `src/scorebook/metrics/yourmetric.py`
+2. Implement the metric class
+3. Add tests
+4. Submit PR for review
+
+### Where to Put Tests
+
+Tests go in one of two directories:
+
+- **`tests/unit/test_metrics/`** - For fast tests using mocked data. These run on every commit.
+- **`tests/extended/test_metrics/`** - For tests that require external dependencies, large datasets, or are computationally expensive.
+
+Most metrics only need unit tests. Use extended tests when your metric relies on external APIs, models, or takes significant time to run.
+
+See [CONTRIBUTING.md](../../../CONTRIBUTING.md) for instructions on running tests.
+
+---
+
+## Requirements
+
+Your metric must:
+
+- Use the `@scorebook_metric` decorator
+- Inherit from `MetricBase`
+- Implement the `score()` static method
+
+The `score()` method returns a tuple of `(aggregate_scores, item_scores)`:
+
+- **aggregate_scores**: A `Dict[str, float]` with overall metric values (e.g., `{"accuracy": 0.85}`)
+- **item_scores**: A `List` of per-item scores. For metrics that produce a single value per item, use `int`, `float`, `bool`, or `str`. For metrics that produce multiple values per item, use a `Dict[str, Union[int, float, bool, str]]` where keys are metric names.
+
+---
+
+## File Naming
+
+Metric files must use normalized names (lowercase, no underscores/spaces). This naming convention is required for the registry's lazy loading system to work.
+
+1. User requests a metric by name (e.g., `"f1_score"`, `"F1Score"`, or `"f1 score"`)
+2. The registry normalizes the input → `"f1score"`
+3. The registry imports `scorebook.metrics.f1score`
+4. The `@scorebook_metric` decorator registers the class
+
+**Examples:**
+- Class: `F1Score` → File: `f1score.py` → User can request: `"f1score"`, `"F1Score"`, `"f1_score"`, `"f1 score"`
+- Class: `MeanSquaredError` → File: `meansquarederror.py` → User can request: `"MeanSquaredError"`, `"mean_squared_error"`, etc.
+
+**Collision detection:** Class names that normalize to the same key will raise an error at registration time. For example, `F1Score` and `F1_Score` both normalize to `"f1score"` and cannot coexist.
+
+---
+
+## Implementation Template
+
+Create your metric file in `src/scorebook/metrics/yourmetric.py`:
+
+```python
+"""Brief description of the metric."""
+
+from typing import Any, Dict, List, Tuple
+
+from scorebook.metrics import MetricBase, scorebook_metric
+
+
+@scorebook_metric
+class YourMetric(MetricBase):
+    """One-line description of what this metric measures.
+
+    Formula or explanation (e.g., Accuracy = correct / total).
+    """
+
+    def score(outputs: List[Any], labels: List[Any]) -> Tuple[Dict[str, Any], List[Any]]:
+        """Calculate metric score between outputs and labels.
+
+        Args:
+            outputs: A list of model inference outputs.
+            labels: A list of ground truth labels.
+
+        Returns:
+            Tuple containing:
+                - Aggregate scores dict (e.g., {"your_metric": 0.85})
+                - List of per-item scores
+
+        Raises:
+            ValueError: If outputs and labels have different lengths.
+        """
+        # Input validation
+        if len(outputs) != len(labels):
+            raise ValueError("Number of outputs must match number of labels")
+
+        if not outputs:
+            return {"your_metric": 0.0}, []
+
+        # Calculate per-item scores
+        item_scores = [calculate_score(out, lab) for out, lab in zip(outputs, labels)]
+
+        # Calculate aggregate score
+        aggregate_score = sum(item_scores) / len(item_scores)
+
+        return {"your_metric": aggregate_score}, item_scores
+```
+
+---
+
+## Documentation
+
+Each metric should have:
+
+1. **Module-level docstring**: Brief description at the top of the file
+2. **Class docstring**: What the metric measures, formula, and any limitations
+3. **Method docstring**: Args, Returns, and Raises sections
+
+---
+
+## Example
+
+See `src/scorebook/metrics/accuracy.py` for a complete reference implementation.

scorebook/metrics/__init__.py

@@ -1,18 +1,9 @@
-"""
-Metrics for evaluating model predictions.
+"""Metrics for evaluating model predictions."""
 
-This module provides a collection of evaluation metrics for comparing model outputs
-against ground truth labels. Available metrics include standard classification and
-generation metrics like accuracy, precision, recall, F1-score, etc.
+from scorebook.metrics.core.metric_base import MetricBase
+from scorebook.metrics.core.metric_registry import scorebook_metric
 
-Metrics can be accessed by name through the `get_metrics()` function or used
-directly by instantiating specific metric classes. All metrics implement a
-common interface for scoring predictions against references.
-"""
-
-from scorebook.metrics.accuracy import Accuracy
-from scorebook.metrics.metric_base import MetricBase
-from scorebook.metrics.metric_registry import MetricRegistry
-from scorebook.metrics.precision import Precision
-
-__all__ = ["MetricBase", "Precision", "Accuracy", "MetricRegistry"]
+__all__ = [
+    "MetricBase",
+    "scorebook_metric",
+]

scorebook/metrics/accuracy.py

@@ -2,11 +2,10 @@
 
 from typing import Any, Dict, List, Tuple
 
-from scorebook.metrics.metric_base import MetricBase
-from scorebook.metrics.metric_registry import MetricRegistry
+from scorebook.metrics import MetricBase, scorebook_metric
 
 
-@MetricRegistry.register()
+@scorebook_metric
 class Accuracy(MetricBase):
     """Accuracy metric for evaluating model predictions of any type.
 
@@ -25,9 +24,6 @@ class Accuracy(MetricBase):
             The aggregate accuracy score for all items (correct predictions / total predictions).
             The item scores for each output-label pair (true/false).
         """
-        if len(outputs) != len(labels):
-            raise ValueError("Number of outputs must match number of labels")
-
         if not outputs:  # Handle empty lists
             return {"accuracy": 0.0}, []
 

scorebook/metrics/bertscore.py

@@ -0,0 +1,50 @@
+"""BertScore implementation for Scorebook."""
+
+from typing import Any, Dict, List, Tuple
+
+import bert_score
+
+from scorebook.metrics import scorebook_metric
+from scorebook.metrics.core.metric_base import MetricBase
+
+
+@scorebook_metric
+class BertScore(MetricBase):
+    """BertScore metric for evaluating model predictions against reference text."""
+
+    def __init__(self, **kwargs: Any) -> None:
+        """Initialize BertScore metric."""
+        defaults = {"lang": "en", "verbose": False}
+        self.kwargs = {**defaults, **kwargs}  # User kwargs override defaults
+
+    def score(self, outputs: List[Any], labels: List[Any]) -> Tuple[Dict[str, Any], List[Any]]:
+        """Calculate bert score between predictions and references.
+
+        Args:
+            outputs: A list of inference outputs.
+            labels: A list of ground truth labels.
+
+        Returns:
+            A tuple containing:
+                - aggregate_scores (Dict[str, float]): Dictionary with average precision,
+                  recall, and F1 scores for all items.
+                - item_scores (List[Dict[str, float]]): List of dictionaries with precision,
+                  recall, and F1 scores for each output-label pair.
+        """
+        if not outputs:  # Handle empty lists
+            return {"precision": 0.0, "recall": 0.0, "F1": 0.0}, []
+
+        # Calculate item scores
+        p_scores, r_scores, f1_scores = bert_score.score(outputs, labels, **self.kwargs)
+
+        item_scores = [
+            {"precision": p, "recall": r, "F1": f1}
+            for p, r, f1 in zip(p_scores.tolist(), r_scores.tolist(), f1_scores.tolist())
+        ]
+        aggregate_scores = {
+            "precision": p_scores.mean().item(),
+            "recall": r_scores.mean().item(),
+            "F1": f1_scores.mean().item(),
+        }
+
+        return aggregate_scores, item_scores

scorebook/metrics/bleu.py

@@ -0,0 +1,82 @@
+"""BLEU metric implementation for Scorebook, based on sacrebleu."""
+
+from typing import Any, Dict, List, Tuple
+
+import sacrebleu
+
+from scorebook.metrics import MetricBase, scorebook_metric
+
+
+@scorebook_metric
+class BLEU(MetricBase):
+    """BLEU metric implementation for Scorebook, based on sacrebleu."""
+
+    def __init__(self, compact: bool = True, **kwargs: Any) -> None:
+        """
+        Generate BLEU metric.
+
+        :param compact: if True, returns only the BLEU metric; if False,
+        returns the full signature of BLEU.
+        :param kwargs: additional arguments passed to BLEU.
+        """
+
+        self.compact = compact
+        self.corpus_bleu = sacrebleu.metrics.BLEU(**kwargs)
+
+        # Overwrite effective order for sentence level scores
+        kwargs["effective_order"] = True
+        self.sentence_bleu = sacrebleu.metrics.BLEU(**kwargs)
+
+    def score(self, outputs: List[Any], labels: List[Any]) -> Tuple[Dict[str, Any], List[Any]]:
+        """Calculate accuracy score between predictions and references.
+
+        Args:
+            outputs: A list of inference outputs.
+            labels: A list of ground truth labels.
+
+        Returns:
+            The aggregate accuracy score for all items (correct predictions / total predictions).
+            The item scores for each output-label pair (true/false).
+        """
+
+        if not outputs:  # Handle empty lists
+            return {"BLEU": 0.0}, []
+
+        item_scores = []
+        # Calculate item scores
+        for output, label in zip(outputs, labels):
+            item_bleu: sacrebleu.metrics.BLEUScore = self.sentence_bleu.sentence_score(
+                output, [label]
+            )
+            item_score = {
+                "BLEU": item_bleu.score,
+            }
+
+            if not self.compact:
+                item_score["1-gram"] = item_bleu.precisions[0]
+                item_score["2-gram"] = item_bleu.precisions[1]
+                item_score["3-gram"] = item_bleu.precisions[2]
+                item_score["4-gram"] = item_bleu.precisions[3]
+                item_score["BP"] = item_bleu.bp
+                item_score["ratio"] = item_bleu.ratio
+                item_score["hyp_len"] = item_bleu.sys_len
+                item_score["ref_len"] = item_bleu.ref_len
+
+            item_scores.append(item_score)
+
+        # Calculate aggregate score
+
+        corpus_bleu: sacrebleu.metrics.BLEUScore = self.corpus_bleu.corpus_score(outputs, [labels])
+        aggregate_scores = {"BLEU": corpus_bleu.score}
+
+        if not self.compact:
+            aggregate_scores["1-gram"] = corpus_bleu.precisions[0]
+            aggregate_scores["2-gram"] = corpus_bleu.precisions[1]
+            aggregate_scores["3-gram"] = corpus_bleu.precisions[2]
+            aggregate_scores["4-gram"] = corpus_bleu.precisions[3]
+            aggregate_scores["BP"] = corpus_bleu.bp
+            aggregate_scores["ratio"] = corpus_bleu.ratio
+            aggregate_scores["hyp_len"] = corpus_bleu.sys_len
+            aggregate_scores["ref_len"] = corpus_bleu.ref_len
+
+        return aggregate_scores, item_scores

scorebook/metrics/core/__init__.py

@@ -0,0 +1 @@
+"""Core metric framework components."""

scorebook/metrics/{metric_base.py → core/metric_base.py}

@@ -12,9 +12,8 @@ class MetricBase(ABC):
         """Return the metric name based on the class name."""
         return self.__class__.__name__.lower()
 
-    @staticmethod
     @abstractmethod
-    def score(outputs: List[Any], labels: List[Any]) -> Tuple[Dict[str, Any], List[Any]]:
+    def score(self, outputs: List[Any], labels: List[Any]) -> Tuple[Dict[str, Any], List[Any]]:
         """Calculate the metric score for a list of outputs and labels.
 
         Args:

scorebook/metrics/core/metric_registry.py

@@ -0,0 +1,195 @@
+"""
+Registry module for evaluation metrics.
+
+This module maintains a centralized registry of available evaluation metrics
+that can be used to assess model performance. It provides a single access point
+to retrieve all implemented metric classes.
+"""
+
+import importlib
+from typing import Any, Callable, Dict, List, Type, Union
+
+from scorebook.metrics.core.metric_base import MetricBase
+
+
+class MetricRegistry:
+    """A registry for evaluation metrics.
+
+    This class provides a central registry for all evaluation metrics in the system.
+    Metrics are lazily loaded on demand - when you request a metric by name, it will
+    be automatically imported from the metrics directory using a naming convention.
+
+    Naming Convention:
+        All metric names are normalized by:
+        - Converting to lowercase
+        - Removing all underscores and spaces
+
+        Module files must follow this normalized naming (lowercase, no underscores/spaces):
+            Examples:
+                Class "Accuracy" → module "accuracy.py"
+                Class "F1Score" → module "f1score.py"
+                Class "MeanSquaredError" → module "meansquarederror.py"
+
+        User input is also normalized, so all variations work:
+            "f1_score", "F1Score", "f1 score" → all resolve to "f1score"
+
+    Collision Detection:
+        Class names that normalize to the same key will raise an error:
+            "F1Score" and "F1_Score" both → "f1score" (COLLISION)
+            "MetricName" and "Metric_Name" both → "metricname" (COLLISION)
+
+    Security:
+        Lazy loading is restricted to modules in the "scorebook.metrics." namespace.
+        Python's import system validates module names.
+    """
+
+    _registry: Dict[str, Type[MetricBase]] = {}
+
+    @staticmethod
+    def _normalize_name(name: str) -> str:
+        """Normalize a metric name to a registry key."""
+        return name.lower().replace("_", "").replace(" ", "")
+
+    @classmethod
+    def register(cls) -> Callable[[Type[MetricBase]], Type[MetricBase]]:
+        """Register a metric class in the registry.
+
+        Returns:
+            A decorator that registers the class and returns it.
+
+        Raises:
+            ValueError: If a metric with the given name is already registered.
+        """
+
+        def decorator(metric_cls: Type[MetricBase]) -> Type[MetricBase]:
+
+            # Normalize the class name to a registry key
+            key = cls._normalize_name(metric_cls.__name__)
+            if key in cls._registry:
+                raise ValueError(
+                    f"Metric '{key}' is already registered. "
+                    f"Class names '{metric_cls.__name__}' and "
+                    f"'{cls._registry[key].__name__}' both normalize to '{key}'."
+                )
+            cls._registry[key] = metric_cls
+            return metric_cls
+
+        return decorator
+
+    @classmethod
+    def _lazy_load_metric(cls, normalized_key: str) -> None:
+        """Attempt to lazily load a metric module using naming convention.
+
+        Module files must be named using the normalized key (lowercase, no underscores/spaces).
+
+        Args:
+            normalized_key: The normalized metric name (lowercase, no underscores/spaces): "f1score"
+
+        Raises:
+            ValueError: If the module doesn't exist or fails to register
+            ImportError: If the module exists but has import errors
+        """
+        # Check if already registered
+        if normalized_key in cls._registry:
+            return
+
+        # Try to import the module using the normalized key
+        try:
+            importlib.import_module(f"scorebook.metrics.{normalized_key}")
+        except ModuleNotFoundError:
+            # Module doesn't exist - provide helpful error
+            error_msg = (
+                f"Metric '{normalized_key}' could not be found. "
+                f"Attempted to import from 'scorebook.metrics.{normalized_key}'."
+            )
+            if cls._registry:
+                registered = ", ".join(sorted(cls._registry.keys()))
+                error_msg += f" Currently registered metrics: {registered}"
+            else:
+                error_msg += " No metrics are currently registered."
+            raise ValueError(error_msg)
+        except ImportError as e:
+            # Module exists but has import errors - re-raise with context
+            raise ImportError(
+                f"Failed to load metric '{normalized_key}' from module "
+                f"'scorebook.metrics.{normalized_key}': {e}"
+            ) from e
+
+    @classmethod
+    def get(cls, name_or_class: Union[str, Type[MetricBase]], **kwargs: Any) -> MetricBase:
+        """
+        Get an instance of a registered metric by name or class.
+
+        Args:
+            name_or_class: The metric name (string) or class (subclass of MetricBase).
+            **kwargs: Additional arguments to pass to the metric's constructor.
+
+        Returns:
+            An instance of the requested metric.
+
+        Raises:
+            ValueError: If the metric cannot be found or loaded.
+            ImportError: If lazy loading fails due to import errors.
+        """
+        # If input is a class that's a subclass of MetricBase, instantiate it directly
+        if isinstance(name_or_class, type) and issubclass(name_or_class, MetricBase):
+            return name_or_class(**kwargs)
+
+        # If input is a string, look up the class in the registry
+        if isinstance(name_or_class, str):
+            # Normalize the input to a registry key
+            normalized_key = cls._normalize_name(name_or_class)
+
+            # Try lazy loading if not already registered
+            if normalized_key not in cls._registry:
+                cls._lazy_load_metric(normalized_key)
+
+            # After lazy loading attempt, check registry
+            if normalized_key not in cls._registry:
+                raise ValueError(
+                    f"Metric '{name_or_class}' module was loaded but failed to register. "
+                    f"Ensure the metric class has the @scorebook_metric decorator."
+                )
+
+            return cls._registry[normalized_key](**kwargs)
+
+        raise ValueError(
+            f"Invalid metric type: {type(name_or_class)}. "
+            f"Must be string name or MetricBase subclass"
+        )
+
+    @classmethod
+    def list_metrics(cls) -> List[str]:
+        """
+        List all registered metrics.
+
+        Returns:
+            A list of metric names.
+        """
+        return list(cls._registry.keys())
+
+
+def scorebook_metric(cls: Type[MetricBase]) -> Type[MetricBase]:
+    """Register a custom metric with Scorebook.
+
+    Args:
+        cls: A metric class that inherits from MetricBase
+
+    Returns:
+        The same class, now registered with Scorebook
+
+    Example:
+        ```python
+        from scorebook import scorebook_metric
+        from scorebook.metrics import MetricBase
+
+        @scorebook_metric
+        class MyCustomMetric(MetricBase):
+            def score(self, outputs, labels):
+                # Your metric implementation
+                return aggregate_scores, item_scores
+        ```
+    Raises:
+        ValueError: If a metric with the same normalized name is already registered
+    """
+    return MetricRegistry.register()(cls)

scorebook/metrics/exactmatch.py

@@ -0,0 +1,95 @@
+"""Exact Match metric implementation for Scorebook."""
+
+import string
+from typing import Any, Dict, List, Tuple
+
+from scorebook.metrics import MetricBase, scorebook_metric
+
+
+@scorebook_metric
+class ExactMatch(MetricBase):
+    """Exact Match metric for evaluating string predictions.
+
+    Compares strings for exact equality with optional preprocessing.
+
+    Args:
+        case_insensitive: If True, convert strings to lowercase before comparison.
+            Defaults to True.
+        strip: If True, strip leading and trailing whitespace before comparison.
+            Defaults to True.
+        strip_punctuation: If True, strip leading and trailing punctuation before
+            comparison. Defaults to False.
+    """
+
+    @property
+    def name(self) -> str:
+        """Return the metric name."""
+        return "exact_match"
+
+    def __init__(
+        self,
+        case_insensitive: bool = True,
+        strip: bool = True,
+        strip_punctuation: bool = False,
+    ) -> None:
+        """Initialize ExactMatch metric with preprocessing options.
+
+        Args:
+            case_insensitive: If True, convert strings to lowercase before comparison.
+                Defaults to True.
+            strip: If True, strip leading and trailing whitespace before comparison.
+                Defaults to True.
+            strip_punctuation: If True, strip leading and trailing punctuation before
+                comparison. Defaults to False.
+        """
+        self.case_insensitive = case_insensitive
+        self.strip = strip
+        self.strip_punctuation = strip_punctuation
+
+    def _preprocess(self, value: Any) -> Any:
+        """Apply preprocessing to a value if it's a string.
+
+        Args:
+            value: The value to preprocess.
+
+        Returns:
+            The preprocessed value (string) or original value (non-string).
+        """
+        if not isinstance(value, str):
+            return value
+
+        result = value
+        if self.strip:
+            result = result.strip()
+        if self.strip_punctuation:
+            result = result.strip(string.punctuation)
+        if self.case_insensitive:
+            result = result.lower()
+        return result
+
+    def score(self, outputs: List[Any], labels: List[Any]) -> Tuple[Dict[str, Any], List[Any]]:
+        """Calculate the exact match score between predictions and references.
+
+        Args:
+            outputs: A list of inference outputs.
+            labels: A list of ground truth labels.
+
+        Returns:
+            The aggregate exact match score for all items (matches / total).
+            The item scores for each output-label pair (true/false).
+        """
+        if not outputs:
+            return {"exact_match": 0.0}, []
+
+        # Calculate item scores with preprocessing
+        item_scores = [
+            self._preprocess(output) == self._preprocess(label)
+            for output, label in zip(outputs, labels)
+        ]
+
+        # Calculate aggregate score
+        matches = sum(item_scores)
+        total = len(outputs)
+        aggregate_scores = {"exact_match": matches / total}
+
+        return aggregate_scores, item_scores