scorebook 0.0.14__py3-none-any.whl → 0.0.15__py3-none-any.whl

scorebook/metrics/core/metric_registry.py

@@ -0,0 +1,195 @@
+"""
+Registry module for evaluation metrics.
+
+This module maintains a centralized registry of available evaluation metrics
+that can be used to assess model performance. It provides a single access point
+to retrieve all implemented metric classes.
+"""
+
+import importlib
+from typing import Any, Callable, Dict, List, Type, Union
+
+from scorebook.metrics.core.metric_base import MetricBase
+
+
+class MetricRegistry:
+    """A registry for evaluation metrics.
+
+    This class provides a central registry for all evaluation metrics in the system.
+    Metrics are lazily loaded on demand - when you request a metric by name, it will
+    be automatically imported from the metrics directory using a naming convention.
+
+    Naming Convention:
+        All metric names are normalized by:
+        - Converting to lowercase
+        - Removing all underscores and spaces
+
+        Module files must follow this normalized naming (lowercase, no underscores/spaces):
+            Examples:
+                Class "Accuracy" → module "accuracy.py"
+                Class "F1Score" → module "f1score.py"
+                Class "MeanSquaredError" → module "meansquarederror.py"
+
+        User input is also normalized, so all variations work:
+            "f1_score", "F1Score", "f1 score" → all resolve to "f1score"
+
+    Collision Detection:
+        Class names that normalize to the same key will raise an error:
+            "F1Score" and "F1_Score" both → "f1score" (COLLISION)
+            "MetricName" and "Metric_Name" both → "metricname" (COLLISION)
+
+    Security:
+        Lazy loading is restricted to modules in the "scorebook.metrics." namespace.
+        Python's import system validates module names.
+    """
+
+    _registry: Dict[str, Type[MetricBase]] = {}
+
+    @staticmethod
+    def _normalize_name(name: str) -> str:
+        """Normalize a metric name to a registry key."""
+        return name.lower().replace("_", "").replace(" ", "")
+
+    @classmethod
+    def register(cls) -> Callable[[Type[MetricBase]], Type[MetricBase]]:
+        """Register a metric class in the registry.
+
+        Returns:
+            A decorator that registers the class and returns it.
+
+        Raises:
+            ValueError: If a metric with the given name is already registered.
+        """
+
+        def decorator(metric_cls: Type[MetricBase]) -> Type[MetricBase]:
+
+            # Normalize the class name to a registry key
+            key = cls._normalize_name(metric_cls.__name__)
+            if key in cls._registry:
+                raise ValueError(
+                    f"Metric '{key}' is already registered. "
+                    f"Class names '{metric_cls.__name__}' and "
+                    f"'{cls._registry[key].__name__}' both normalize to '{key}'."
+                )
+            cls._registry[key] = metric_cls
+            return metric_cls
+
+        return decorator
+
+    @classmethod
+    def _lazy_load_metric(cls, normalized_key: str) -> None:
+        """Attempt to lazily load a metric module using naming convention.
+
+        Module files must be named using the normalized key (lowercase, no underscores/spaces).
+
+        Args:
+            normalized_key: The normalized metric name (lowercase, no underscores/spaces): "f1score"
+
+        Raises:
+            ValueError: If the module doesn't exist or fails to register
+            ImportError: If the module exists but has import errors
+        """
+        # Check if already registered
+        if normalized_key in cls._registry:
+            return
+
+        # Try to import the module using the normalized key
+        try:
+            importlib.import_module(f"scorebook.metrics.{normalized_key}")
+        except ModuleNotFoundError:
+            # Module doesn't exist - provide helpful error
+            error_msg = (
+                f"Metric '{normalized_key}' could not be found. "
+                f"Attempted to import from 'scorebook.metrics.{normalized_key}'."
+            )
+            if cls._registry:
+                registered = ", ".join(sorted(cls._registry.keys()))
+                error_msg += f" Currently registered metrics: {registered}"
+            else:
+                error_msg += " No metrics are currently registered."
+            raise ValueError(error_msg)
+        except ImportError as e:
+            # Module exists but has import errors - re-raise with context
+            raise ImportError(
+                f"Failed to load metric '{normalized_key}' from module "
+                f"'scorebook.metrics.{normalized_key}': {e}"
+            ) from e
+
+    @classmethod
+    def get(cls, name_or_class: Union[str, Type[MetricBase]], **kwargs: Any) -> MetricBase:
+        """
+        Get an instance of a registered metric by name or class.
+
+        Args:
+            name_or_class: The metric name (string) or class (subclass of MetricBase).
+            **kwargs: Additional arguments to pass to the metric's constructor.
+
+        Returns:
+            An instance of the requested metric.
+
+        Raises:
+            ValueError: If the metric cannot be found or loaded.
+            ImportError: If lazy loading fails due to import errors.
+        """
+        # If input is a class that's a subclass of MetricBase, instantiate it directly
+        if isinstance(name_or_class, type) and issubclass(name_or_class, MetricBase):
+            return name_or_class(**kwargs)
+
+        # If input is a string, look up the class in the registry
+        if isinstance(name_or_class, str):
+            # Normalize the input to a registry key
+            normalized_key = cls._normalize_name(name_or_class)
+
+            # Try lazy loading if not already registered
+            if normalized_key not in cls._registry:
+                cls._lazy_load_metric(normalized_key)
+
+            # After lazy loading attempt, check registry
+            if normalized_key not in cls._registry:
+                raise ValueError(
+                    f"Metric '{name_or_class}' module was loaded but failed to register. "
+                    f"Ensure the metric class has the @scorebook_metric decorator."
+                )
+
+            return cls._registry[normalized_key](**kwargs)
+
+        raise ValueError(
+            f"Invalid metric type: {type(name_or_class)}. "
+            f"Must be string name or MetricBase subclass"
+        )
+
+    @classmethod
+    def list_metrics(cls) -> List[str]:
+        """
+        List all registered metrics.
+
+        Returns:
+            A list of metric names.
+        """
+        return list(cls._registry.keys())
+
+
+def scorebook_metric(cls: Type[MetricBase]) -> Type[MetricBase]:
+    """Register a custom metric with Scorebook.
+
+    Args:
+        cls: A metric class that inherits from MetricBase
+
+    Returns:
+        The same class, now registered with Scorebook
+
+    Example:
+        ```python
+        from scorebook import scorebook_metric
+        from scorebook.metrics import MetricBase
+
+        @scorebook_metric
+        class MyCustomMetric(MetricBase):
+            def score(self, outputs, labels):
+                # Your metric implementation
+                return aggregate_scores, item_scores
+        ```
+    Raises:
+        ValueError: If a metric with the same normalized name is already registered
+    """
+    return MetricRegistry.register()(cls)

scorebook/metrics/exactmatch.py

@@ -0,0 +1,95 @@
+"""Exact Match metric implementation for Scorebook."""
+
+import string
+from typing import Any, Dict, List, Tuple
+
+from scorebook.metrics import MetricBase, scorebook_metric
+
+
+@scorebook_metric
+class ExactMatch(MetricBase):
+    """Exact Match metric for evaluating string predictions.
+
+    Compares strings for exact equality with optional preprocessing.
+
+    Args:
+        case_insensitive: If True, convert strings to lowercase before comparison.
+            Defaults to True.
+        strip: If True, strip leading and trailing whitespace before comparison.
+            Defaults to True.
+        strip_punctuation: If True, strip leading and trailing punctuation before
+            comparison. Defaults to False.
+    """
+
+    @property
+    def name(self) -> str:
+        """Return the metric name."""
+        return "exact_match"
+
+    def __init__(
+        self,
+        case_insensitive: bool = True,
+        strip: bool = True,
+        strip_punctuation: bool = False,
+    ) -> None:
+        """Initialize ExactMatch metric with preprocessing options.
+
+        Args:
+            case_insensitive: If True, convert strings to lowercase before comparison.
+                Defaults to True.
+            strip: If True, strip leading and trailing whitespace before comparison.
+                Defaults to True.
+            strip_punctuation: If True, strip leading and trailing punctuation before
+                comparison. Defaults to False.
+        """
+        self.case_insensitive = case_insensitive
+        self.strip = strip
+        self.strip_punctuation = strip_punctuation
+
+    def _preprocess(self, value: Any) -> Any:
+        """Apply preprocessing to a value if it's a string.
+
+        Args:
+            value: The value to preprocess.
+
+        Returns:
+            The preprocessed value (string) or original value (non-string).
+        """
+        if not isinstance(value, str):
+            return value
+
+        result = value
+        if self.strip:
+            result = result.strip()
+        if self.strip_punctuation:
+            result = result.strip(string.punctuation)
+        if self.case_insensitive:
+            result = result.lower()
+        return result
+
+    def score(self, outputs: List[Any], labels: List[Any]) -> Tuple[Dict[str, Any], List[Any]]:
+        """Calculate the exact match score between predictions and references.
+
+        Args:
+            outputs: A list of inference outputs.
+            labels: A list of ground truth labels.
+
+        Returns:
+            The aggregate exact match score for all items (matches / total).
+            The item scores for each output-label pair (true/false).
+        """
+        if not outputs:
+            return {"exact_match": 0.0}, []
+
+        # Calculate item scores with preprocessing
+        item_scores = [
+            self._preprocess(output) == self._preprocess(label)
+            for output, label in zip(outputs, labels)
+        ]
+
+        # Calculate aggregate score
+        matches = sum(item_scores)
+        total = len(outputs)
+        aggregate_scores = {"exact_match": matches / total}
+
+        return aggregate_scores, item_scores

scorebook/metrics/f1.py

@@ -0,0 +1,96 @@
+"""F1 metric implementation for Scorebook."""
+
+from typing import Any, Dict, List, Tuple, Union
+
+from sklearn.metrics import f1_score
+
+from scorebook.metrics import MetricBase, scorebook_metric
+
+
+@scorebook_metric
+class F1(MetricBase):
+    """F1 score metric for evaluating model predictions using scikit-learn.
+
+    F1 = 2 * (Precision * Recall) / (Precision + Recall)
+    where:
+    - Precision = TP / (TP + FP)
+    - Recall = TP / (TP + FN)
+
+    This metric can handle both binary and multi-class classification tasks.
+
+    Args:
+        average: The averaging method(s) for multi-class classification.
+            Can be a single string or list of strings:
+            - 'macro': Unweighted mean across labels
+            - 'micro': Global calculation counting total TP, FP, FN
+            - 'weighted': Weighted mean by support
+            - 'all': All three methods simultaneously
+            - List of methods: Calculate multiple methods
+            Defaults to 'macro'.
+    """
+
+    def __init__(self, average: Union[str, List[str]] = "macro", **kwargs: Any) -> None:
+        """Initialize F1 metric with specified averaging method(s).
+
+        Args:
+            average: Averaging method(s) - string or list of strings.
+                Options: 'macro', 'micro', 'weighted', 'all', or a list of methods.
+                Defaults to 'macro'.
+            **kwargs: Additional keyword arguments passed to scikit-learn's f1_score function.
+
+        Raises:
+            ValueError: If average contains invalid methods or combines 'all' with others.
+        """
+        # Normalize to list for validation
+        averages = [average] if isinstance(average, str) else average
+
+        # Validate
+        valid = {"macro", "micro", "weighted", "all"}
+        if not all(a in valid for a in averages):
+            raise ValueError(f"Invalid average method(s). Must be from {valid}.")
+        if len(averages) > 1 and "all" in averages:
+            raise ValueError("'all' cannot be combined with other methods.")
+
+        self.average = average
+        self.kwargs = kwargs
+
+    def score(self, outputs: List[Any], labels: List[Any]) -> Tuple[Dict[str, Any], List[Any]]:
+        """Calculate F1 score between predictions and references using scikit-learn.
+
+        Args:
+            outputs: A list of inference outputs.
+            labels: A list of ground truth labels.
+
+        Returns:
+            Tuple containing:
+                - aggregate_scores (Dict[str, float]): Dictionary with F1 scores
+                  keyed by averaging method (e.g., {"F1 (macro)": 0.85} or
+                  {"F1 (macro)": 0.85, "F1 (micro)": 0.82}).
+                - item_scores (List[bool]): True/False list indicating correct
+                  predictions.
+
+        """
+        # Normalize to list of methods to calculate
+        if isinstance(self.average, str):
+            methods = ["macro", "micro", "weighted"] if self.average == "all" else [self.average]
+        else:
+            methods = self.average
+
+        # Handle empty lists
+        if not outputs:
+            return {f"F1 ({method})": 0.0 for method in methods}, []
+
+        # Calculate F1 score using scikit-learn with configured averaging method
+        # Default zero_division=0 unless overridden in kwargs
+        kwargs = {"zero_division": 0, **self.kwargs}
+
+        # Calculate item scores (correctness of each prediction)
+        item_scores = [output == label for output, label in zip(outputs, labels)]
+
+        # Calculate F1 for each method
+        aggregate_scores = {
+            f"F1 ({method})": f1_score(labels, outputs, average=method, **kwargs)
+            for method in methods
+        }
+
+        return aggregate_scores, item_scores

scorebook/metrics/precision.py

@@ -1,19 +1,94 @@
 """Precision metric implementation for Scorebook."""
 
-from typing import Any, Dict, List, Tuple
+from typing import Any, Dict, List, Tuple, Union
 
-from scorebook.metrics.metric_base import MetricBase
-from scorebook.metrics.metric_registry import MetricRegistry
+from sklearn.metrics import precision_score
 
+from scorebook.metrics import MetricBase, scorebook_metric
 
-@MetricRegistry.register()
+
+@scorebook_metric
 class Precision(MetricBase):
-    """Precision metric for binary classification.
+    """Precision score metric for evaluating model predictions using scikit-learn.
 
     Precision = TP / (TP + FP)
+
+    This metric can handle both binary and multi-class classification tasks.
+
+    Args:
+        average: The averaging method(s) for multi-class classification.
+            Can be a single string or list of strings:
+            - 'macro': Unweighted mean across labels
+            - 'micro': Global calculation counting total TP, FP
+            - 'weighted': Weighted mean by support
+            - 'all': All three methods simultaneously
+            - List of methods: Calculate multiple methods
+            Defaults to 'macro'.
     """
 
-    @staticmethod
-    def score(outputs: List[Any], labels: List[Any]) -> Tuple[Dict[str, Any], List[Any]]:
-        """Not implemented."""
-        raise NotImplementedError("Precision not implemented")
+    def __init__(self, average: Union[str, List[str]] = "macro", **kwargs: Any) -> None:
+        """Initialize Precision metric with specified averaging method(s).
+
+        Args:
+            average: Averaging method(s) - string or list of strings.
+                Options: 'macro', 'micro', 'weighted', 'all', or a list of methods.
+                Defaults to 'macro'.
+            **kwargs: Additional keyword arguments passed to sklearn's precision_score.
+
+        Raises:
+            ValueError: If average contains invalid methods or combines 'all' with others.
+        """
+        # Normalize to list for validation
+        averages = [average] if isinstance(average, str) else average
+
+        # Validate
+        valid = {"macro", "micro", "weighted", "all"}
+        if not all(a in valid for a in averages):
+            raise ValueError(f"Invalid average method(s). Must be from {valid}.")
+        if len(averages) > 1 and "all" in averages:
+            raise ValueError("'all' cannot be combined with other methods.")
+
+        self.average = average
+        self.kwargs = kwargs
+
+    def score(self, outputs: List[Any], labels: List[Any]) -> Tuple[Dict[str, Any], List[Any]]:
+        """Calculate Precision score between predictions and references using scikit-learn.
+
+        Args:
+            outputs: A list of inference outputs.
+            labels: A list of ground truth labels.
+
+        Returns:
+            Tuple containing:
+                - aggregate_scores (Dict[str, float]): Dictionary with Precision scores
+                  keyed by averaging method (e.g., {"Precision (macro)": 0.85} or
+                  {"Precision (macro)": 0.85, "Precision (micro)": 0.82}).
+                - item_scores (List[bool]): True/False list indicating correct
+                  predictions.
+
+        """
+
+        # Normalize to list of methods to calculate
+        if isinstance(self.average, str):
+            methods = ["macro", "micro", "weighted"] if self.average == "all" else [self.average]
+        else:
+            methods = self.average
+
+        # Handle empty lists
+        if not outputs:
+            return {f"Precision ({method})": 0.0 for method in methods}, []
+
+        # Calculate Precision score using scikit-learn with configured averaging method
+        # Default zero_division=0 unless overridden in kwargs
+        kwargs = {"zero_division": 0, **self.kwargs}
+
+        # Calculate item scores (correctness of each prediction)
+        item_scores = [output == label for output, label in zip(outputs, labels)]
+
+        # Calculate Precision for each method
+        aggregate_scores = {
+            f"Precision ({method})": precision_score(labels, outputs, average=method, **kwargs)
+            for method in methods
+        }
+
+        return aggregate_scores, item_scores

scorebook/metrics/recall.py

@@ -0,0 +1,94 @@
+"""Recall metric implementation for Scorebook."""
+
+from typing import Any, Dict, List, Tuple, Union
+
+from sklearn.metrics import recall_score
+
+from scorebook.metrics import MetricBase, scorebook_metric
+
+
+@scorebook_metric
+class Recall(MetricBase):
+    """Recall score metric for evaluating model predictions using scikit-learn.
+
+    Recall = TP / (TP + FN)
+
+    This metric can handle both binary and multi-class classification tasks.
+
+    Args:
+        average: The averaging method(s) for multi-class classification.
+            Can be a single string or list of strings:
+            - 'macro': Unweighted mean across labels
+            - 'micro': Global calculation counting total TP, FP, FN
+            - 'weighted': Weighted mean by support
+            - 'all': All three methods simultaneously
+            - List of methods: Calculate multiple methods
+            Defaults to 'macro'.
+    """
+
+    def __init__(self, average: Union[str, List[str]] = "macro", **kwargs: Any) -> None:
+        """Initialize Recall metric with specified averaging method(s).
+
+        Args:
+            average: Averaging method(s) - string or list of strings.
+                Options: 'macro', 'micro', 'weighted', 'all', or a list of methods.
+                Defaults to 'macro'.
+            **kwargs: Additional keyword arguments passed to sklearn's recall_score.
+
+        Raises:
+            ValueError: If average contains invalid methods or combines 'all' with others.
+        """
+        # Normalize to list for validation
+        averages = [average] if isinstance(average, str) else average
+
+        # Validate
+        valid = {"macro", "micro", "weighted", "all"}
+        if not all(a in valid for a in averages):
+            raise ValueError(f"Invalid average method(s). Must be from {valid}.")
+        if len(averages) > 1 and "all" in averages:
+            raise ValueError("'all' cannot be combined with other methods.")
+
+        self.average = average
+        self.kwargs = kwargs
+
+    def score(self, outputs: List[Any], labels: List[Any]) -> Tuple[Dict[str, Any], List[Any]]:
+        """Calculate Recall score between predictions and references using scikit-learn.
+
+        Args:
+            outputs: A list of inference outputs.
+            labels: A list of ground truth labels.
+
+        Returns:
+            Tuple containing:
+                - aggregate_scores (Dict[str, float]): Dictionary with Recall scores
+                  keyed by averaging method (e.g., {"Recall (macro)": 0.85} or
+                  {"Recall (macro)": 0.85, "Recall (micro)": 0.82}).
+                - item_scores (List[bool]): True/False list indicating correct
+                  predictions.
+
+        """
+
+        # Normalize to list of methods to calculate
+        if isinstance(self.average, str):
+            methods = ["macro", "micro", "weighted"] if self.average == "all" else [self.average]
+        else:
+            methods = self.average
+
+        # Handle empty lists
+        if not outputs:
+            return {f"Recall ({method})": 0.0 for method in methods}, []
+
+        # Calculate Recall score using scikit-learn with configured averaging method
+        # Default zero_division=0 unless overridden in kwargs
+        kwargs = {"zero_division": 0, **self.kwargs}
+
+        # Calculate item scores (correctness of each prediction)
+        item_scores = [output == label for output, label in zip(outputs, labels)]
+
+        # Calculate Recall for each method
+        aggregate_scores = {
+            f"Recall ({method})": recall_score(labels, outputs, average=method, **kwargs)
+            for method in methods
+        }
+
+        return aggregate_scores, item_scores

scorebook/metrics/rouge.py

@@ -0,0 +1,85 @@
+"""ROUGE metric implementation for Scorebook."""
+
+import warnings
+from typing import Any, Dict, List, Optional, Tuple
+
+from rouge_score import rouge_scorer
+
+from scorebook.metrics import MetricBase, scorebook_metric
+
+
+@scorebook_metric
+class ROUGE(MetricBase):
+    """ROUGE metric for evaluating text generation quality.
+
+    ROUGE (Recall-Oriented Understudy for Gisting Evaluation) measures
+    the overlap between generated text and reference text.
+    Returns ROUGE-1 and ROUGE-L F1 scores.
+    """
+
+    def __init__(self, rouge_types: Optional[List[str]] = None, **kwargs: Any) -> None:
+        """Initialize the ROUGE metric.
+
+        Args:
+            rouge_types: List of ROUGE types to calculate (e.g., ["rouge1", "rouge2", "rougeL"]).
+                        Defaults to ["rouge1", "rougeL"].
+            **kwargs: Additional keyword arguments to pass to RougeScorer
+                     (e.g., use_stemmer, split_summaries, tokenizer).
+                     Defaults to use_stemmer=True if not provided.
+        """
+        if rouge_types is None:
+            warnings.warn(
+                "No rouge_types specified, defaulting to ['rouge1', 'rougeL']",
+                UserWarning,
+                stacklevel=2,
+            )
+            rouge_types = ["rouge1", "rougeL"]
+        if "use_stemmer" not in kwargs:
+            warnings.warn(
+                "use_stemmer not specified, defaulting to True",
+                UserWarning,
+                stacklevel=2,
+            )
+            kwargs["use_stemmer"] = True
+        self.rouge_types = rouge_types
+        self.scorer = rouge_scorer.RougeScorer(rouge_types, **kwargs)
+
+    def score(self, outputs: List[Any], labels: List[Any]) -> Tuple[Dict[str, Any], List[Any]]:
+        """Calculate ROUGE scores between predictions and references.
+
+        Args:
+            outputs: A list of generated text outputs.
+            labels: A list of reference text labels.
+
+        Returns:
+            A tuple containing:
+            - aggregate_scores: Dict with average F1 scores for each configured ROUGE type
+            - item_scores: List of dicts with F1 scores for each configured ROUGE type
+        """
+
+        if not outputs:  # Handle empty lists
+            return {rouge_type: 0.0 for rouge_type in self.rouge_types}, []
+
+        # Calculate item scores
+        item_scores = []
+        for output, label in zip(outputs, labels):
+            # Convert to strings if needed
+            output_str = str(output) if output is not None else ""
+            label_str = str(label) if label is not None else ""
+
+            # Calculate ROUGE scores
+            scores = self.scorer.score(output_str, label_str)
+
+            # Extract F1 scores (fmeasure) for all configured rouge types
+            item_score = {
+                rouge_type: scores[rouge_type].fmeasure for rouge_type in self.rouge_types
+            }
+            item_scores.append(item_score)
+
+        # Calculate aggregate scores (average of all items for each rouge type)
+        aggregate_scores = {
+            rouge_type: sum(item[rouge_type] for item in item_scores) / len(item_scores)
+            for rouge_type in self.rouge_types
+        }
+
+        return aggregate_scores, item_scores