scorebook 0.0.1__py3-none-any.whl

scorebook/metrics/metric_base.py

@@ -0,0 +1,28 @@
+"""Base class for evaluation metrics."""
+
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List, Tuple
+
+
+class MetricBase(ABC):
+    """Base class for all evaluation metrics."""
+
+    @property
+    def name(self) -> str:
+        """Return the metric name based on the class name."""
+        return self.__class__.__name__.lower()
+
+    @staticmethod
+    @abstractmethod
+    def score(outputs: List[Any], labels: List[Any]) -> Tuple[Dict[str, Any], List[Any]]:
+        """Calculate the metric score for a list of outputs and labels.
+
+        Args:
+            outputs: A list of inference outputs.
+            labels: A list of ground truth labels.
+
+        Returns:
+            Aggregate metric scores for all items.
+            Individual scores for each item.
+        """
+        raise NotImplementedError("MetricBase is an abstract class")

scorebook/metrics/metric_registry.py

@@ -0,0 +1,105 @@
+"""
+Registry module for evaluation metrics.
+
+This module maintains a centralized registry of available evaluation metrics
+that can be used to assess model performance. It provides a single access point
+to retrieve all implemented metric classes.
+"""
+
+from typing import Any, Callable, Dict, List, Type, Union
+
+from scorebook.metrics.metric_base import MetricBase
+
+
+class MetricRegistry:
+    """A registry for evaluation metrics.
+
+    This class provides a central registry for all evaluation metrics in the system.
+    It allows metrics to be registered with unique names and retrieved either by
+    name or by class. The registry ensures that metrics are properly initialized
+    and accessible throughout the application.
+
+    The registry supports:
+    - Registering new metric classes with optional custom names
+    - Retrieving metric instances by name or class
+    - Listing all available metrics
+
+    Usage:
+        @MetricRegistry.register("custom_name")
+        class MyMetric(MetricBase):
+            ...
+
+        # Get by name
+        metric = MetricRegistry.get("custom_name")
+
+        # Get by class
+        metric = MetricRegistry.get(MyMetric)
+
+        # List available metrics
+        metrics = MetricRegistry.list_metrics()
+    """
+
+    _registry: Dict[str, Type[MetricBase]] = {}
+
+    @classmethod
+    def register(cls) -> Callable[[Type[MetricBase]], Type[MetricBase]]:
+        """
+        Register a metric class in the registry.
+
+        Returns:
+            A decorator that registers the class and returns it.
+
+        Raises:
+            ValueError: If a metric with the given name is already registered.
+        """
+
+        def decorator(metric_cls: Type[MetricBase]) -> Type[MetricBase]:
+
+            key = metric_cls.__name__.lower()
+            if key in cls._registry:
+                raise ValueError(f"Metric '{key}' is already registered")
+            cls._registry[key] = metric_cls
+            return metric_cls
+
+        return decorator
+
+    @classmethod
+    def get(cls, name_or_class: Union[str, Type[MetricBase]], **kwargs: Any) -> MetricBase:
+        """
+        Get an instance of a registered metric by name or class.
+
+        Args:
+            name_or_class: The metric name (string) or class (subclass of BaseMetric).
+            **kwargs: Additional arguments to pass to the metric's constructor.
+
+        Returns:
+            An instance of the requested metric.
+
+        Raises:
+            ValueError: If the metric name is not registered.
+        """
+        # If input is a class that's a subclass of BaseMetric, instantiate it directly
+        if isinstance(name_or_class, type) and issubclass(name_or_class, MetricBase):
+            return name_or_class(**kwargs)
+
+        # If input is a string, look up the class in the registry
+        if isinstance(name_or_class, str):
+            key = name_or_class.lower()
+            if key not in cls._registry:
+                raise ValueError(f"Metric '{name_or_class}' not registered.")
+            return cls._registry[key](**kwargs)
+
+        raise ValueError(
+            f"Invalid metric type: {type(name_or_class)}."
+            f"Must be string name or BaseMetric subclass"
+        )
+
+    @classmethod
+    def list_metrics(cls) -> List[str]:
+        """
+        List all registered metrics.
+
+        Returns:
+            A list of metric names.
+        """
+        return list(cls._registry.keys())

scorebook/metrics/precision.py

@@ -0,0 +1,19 @@
+"""Precision metric implementation for Scorebook."""
+
+from typing import Any, Dict, List, Tuple
+
+from scorebook.metrics.metric_base import MetricBase
+from scorebook.metrics.metric_registry import MetricRegistry
+
+
+@MetricRegistry.register()
+class Precision(MetricBase):
+    """Precision metric for binary classification.
+
+    Precision = TP / (TP + FP)
+    """
+
+    @staticmethod
+    def score(outputs: List[Any], labels: List[Any]) -> Tuple[Dict[str, Any], List[Any]]:
+        """Not implemented."""
+        raise NotImplementedError("Precision not implemented")

scorebook/types/__init__.py

@@ -0,0 +1,11 @@
+"""
+Types package containing data structures and type definitions for the Scorebook framework.
+
+This module provides core data types used throughout the framework for dataset handling
+and evaluation results.
+"""
+
+from scorebook.types.eval_dataset import EvalDataset
+from scorebook.types.eval_result import EvalResult
+
+__all__ = ["EvalDataset", "EvalResult"]

scorebook/types/eval_dataset.py

@@ -0,0 +1,310 @@
+"""Eval Dataset implementation for scorebook."""
+
+import csv
+import json
+from typing import Any, Dict, Iterator, List, Optional, Type, Union
+
+from datasets import Dataset as HuggingFaceDataset
+from datasets import DatasetDict as HuggingFaceDatasetDict
+from datasets import load_dataset
+
+from scorebook.metrics import MetricBase, MetricRegistry
+from scorebook.utils import validate_path
+
+
+class EvalDataset:
+    """Eval Dataset implementation for scorebook."""
+
+    def __init__(
+        self,
+        name: str,
+        label: str,
+        metrics: Union[str, Type[MetricBase], List[Union[str, Type[MetricBase]]]],
+        hf_dataset: HuggingFaceDataset,
+    ):
+        """
+        Create a new scorebook evaluation dataset instance.
+
+        :param name: The name of the evaluation dataset.
+        :param label: The label field of the dataset.
+        :param metrics: The specified metrics associated with the dataset.
+        :param hf_dataset: The dataset as a hugging face dataset object.
+        """
+        self.name: str = name
+        self.label: str = label
+        self.metrics: List[MetricBase] = self._resolve_metrics(metrics)
+        self._hf_dataset: Optional[HuggingFaceDataset] = hf_dataset
+
+    def __len__(self) -> int:
+        """Return the number of items in the dataset."""
+        if self._hf_dataset is None:
+            raise ValueError("Dataset is not initialized")
+        return len(self._hf_dataset)
+
+    def __getitem__(self, key: Union[int, str]) -> Union[Dict[str, Any], List[Any]]:
+        """
+        Allow item access by index (int) or by column name (str).
+
+        - eval_dataset[i] returns the i-th example (dict).
+        - eval_dataset["feature"] returns a list of values for that feature.
+        """
+        if self._hf_dataset is None:
+            raise ValueError("Dataset is not initialized")
+        if isinstance(key, int):
+            return dict(self._hf_dataset[key])  # Ensure we return a Dict[str, Any]
+        elif isinstance(key, str):
+            return list(self._hf_dataset[key])  # Ensure we return a List[Any]
+        else:
+            raise TypeError(f"Invalid key type: {type(key)}. Must be int or str.")
+
+    def __str__(self) -> str:
+        """Return a formatted string summary of the evaluation dataset."""
+        if self._hf_dataset is None:
+            return f"EvalDataset(name='{self.name}', status='uninitialized')"
+
+        num_rows = len(self._hf_dataset)
+        fields = ", ".join(self.column_names)
+        metrics = ", ".join([metric.name for metric in self.metrics])
+
+        return (
+            f"EvalDataset(\n"
+            f"  name='{self.name}',\n"
+            f"  rows={num_rows},\n"
+            f"  label='{self.label}',\n"
+            f"  fields=[{fields}],\n"
+            f"  metrics=[{metrics}]\n"
+            f")"
+        )
+
+    def __iter__(self) -> Iterator[Dict[str, Any]]:
+        """Return an iterator over all examples in the dataset."""
+        if self._hf_dataset is None:
+            raise ValueError("Dataset is not initialized")
+        return iter(self._hf_dataset)
+
+    @property
+    def items(self) -> List[Any]:
+        """Return a list of all examples in the dataset."""
+        if self._hf_dataset is None:
+            raise ValueError("Dataset is not initialized")
+        return list(self._hf_dataset)
+
+    @property
+    def column_names(self) -> List[str]:
+        """Return a list of column/feature names available in the dataset."""
+        if self._hf_dataset is None:
+            raise ValueError("Dataset is not initialized")
+        return list(map(str, self._hf_dataset.column_names))
+
+    @classmethod
+    def from_list(
+        cls,
+        name: str,
+        label: str,
+        metrics: Union[str, Type[MetricBase], List[Union[str, Type[MetricBase]]]],
+        data: List[Dict[str, Any]],
+    ) -> "EvalDataset":
+        """Instantiate an EvalDataset from a list of dictionaries.
+
+        Args:
+            name: The name of the evaluation dataset.
+            label: The field used as the evaluation label (ground truth).
+            metrics: The specified metrics associated with the dataset.
+            data: List of dictionaries containing the dataset examples.
+
+        Returns:
+            A scorebook EvalDataset wrapping a Hugging Face dataset.
+        """
+        return cls(
+            name=name, label=label, metrics=metrics, hf_dataset=HuggingFaceDataset.from_list(data)
+        )
+
+    @classmethod
+    def from_csv(
+        cls,
+        file_path: str,
+        label: str,
+        metrics: Union[str, Type[MetricBase], List[Union[str, Type[MetricBase]]]],
+        name: Optional[str] = None,
+        encoding: str = "utf-8",
+        newline: str = "",
+        **reader_kwargs: Any,
+    ) -> "EvalDataset":
+        """Instantiate a scorebook dataset from a CSV file.
+
+        Args:
+            file_path: Path to the CSV file.
+            label: The field used as the evaluation label (ground truth).
+            metrics: The specified metrics associated with the dataset.
+            name: Optional name for the eval dataset, if not provided, the path is used
+            encoding: Encoding of the CSV file.
+            newline: Newline character of the CSV file.
+            reader_kwargs: Dict of kwargs passed to `csv.DictReader`.
+
+        Returns:
+            A scorebook EvalDataset.
+
+        Raises:
+            FileNotFoundError: If the file does not exist at the given path.
+            ValueError: If the CSV file cannot be parsed or is empty.
+        """
+        reader_kwargs = reader_kwargs or {}
+        path = validate_path(file_path, expected_suffix=".csv")
+
+        try:
+            with open(path, encoding=encoding, newline=newline) as csvfile:
+                reader = csv.DictReader(csvfile, **reader_kwargs)
+                data = [row for row in reader]
+        except csv.Error as e:
+            raise ValueError(f"Failed to parse CSV file {file_path}: {e}") from e
+
+        if not data:
+            raise ValueError(f"CSV file {file_path} is empty or contains only headers.")
+
+        name = name if name else path.stem
+        return cls(
+            name=name,
+            label=label,
+            metrics=metrics,
+            hf_dataset=HuggingFaceDataset.from_list(data),
+        )
+
+    @classmethod
+    def from_json(
+        cls,
+        file_path: str,
+        label: str,
+        metrics: Union[str, Type[MetricBase], List[Union[str, Type[MetricBase]]]],
+        name: Optional[str] = None,
+        split: Optional[str] = None,
+    ) -> "EvalDataset":
+        """Instantiate an EvalDataset from a JSON file.
+
+        The JSON file must follow one of two supported formats:
+
+        1. **Flat format** – a list of dictionaries:
+            [
+                {"input": "What is 2+2?", "label": "4"},
+                {"input": "Capital of France?", "label": "Paris"}
+            ]
+
+        2. **Split format** – a dictionary of named splits:
+            {
+                "train": [{"input": ..., "label": ...}],
+                "test": [{"input": ..., "label": ...}]
+            }
+
+        Args:
+            file_path: Path to the JSON file on disk.
+            label: The field used as the evaluation label (ground truth).
+            metrics: The specified metrics associated with the dataset.
+            name: Optional name for the eval dataset, if not provided, the path is used
+            split: If the JSON uses a split structure, this is the split name to load.
+
+        Returns:
+            A scorebook EvalDataset wrapping a Hugging Face dataset.
+
+        Raises:
+            FileNotFoundError: If the file does not exist.
+            ValueError: If the JSON is invalid or the structure is unsupported.
+        """
+        path = validate_path(file_path, expected_suffix=".json")
+
+        try:
+            with path.open("r", encoding="utf-8") as f:
+                data = json.load(f)
+        except json.JSONDecodeError as e:
+            raise ValueError(f"Invalid JSON in {file_path}: {e}") from e
+
+        if isinstance(data, dict):
+            if split is None:
+                raise ValueError(f"Split name must be provided for split-style JSON: {file_path}")
+            split_data = data.get(split)
+            if split_data is None:
+                raise ValueError(f"Split '{split}' not found in JSON file: {file_path}")
+            if not isinstance(split_data, list):
+                raise ValueError(f"Split '{split}' is not a list of examples in: {file_path}")
+            hf_dataset = HuggingFaceDataset.from_list(split_data)
+        elif isinstance(data, list):
+            hf_dataset = HuggingFaceDataset.from_list(data)
+        else:
+            raise ValueError(f"Unsupported JSON structure in {file_path}. Expected list or dict.")
+
+        name = name if name else path.stem
+        return cls(name=name, label=label, metrics=metrics, hf_dataset=hf_dataset)
+
+    @classmethod
+    def from_huggingface(
+        cls,
+        path: str,
+        label: str,
+        metrics: Union[str, Type[MetricBase], List[Union[str, Type[MetricBase]]]],
+        split: Optional[str] = None,
+        name: Optional[str] = None,
+    ) -> "EvalDataset":
+        """Instantiate an EvalDataset from a dataset available on Hugging Face Hub.
+
+        If a specific split is provided (e.g., "train" or "test"), it will be loaded directly.
+        If no split is specified, the method attempts to load the full dataset. If the dataset
+        is split into multiple subsets (i.e., a DatasetDict), it defaults to loading the "test"
+        split.
+
+        Args:
+            path: The path of the dataset on the Hugging Face Hub.
+            label: The field used as the evaluation label (ground truth).
+            metrics: The specified metrics associated with the dataset.
+            split: Optional name of the split to load.
+            name: Optional dataset configuration name.
+
+        Returns:
+            An EvalDataset wrapping the selected Hugging Face dataset.
+
+        Raises:
+            ValueError: If the dataset cannot be loaded, or the expected split is missing.
+        """
+        try:
+            kwargs = {}
+            if split is not None:
+                kwargs["split"] = split
+            if name is not None:
+                kwargs["name"] = name
+            ds = load_dataset(path, **kwargs)
+        except Exception as e:
+            raise ValueError(f"Failed to load dataset '{path}' from Hugging Face: {e}") from e
+
+        if isinstance(ds, HuggingFaceDataset):
+            hf_dataset = ds
+        elif isinstance(ds, HuggingFaceDatasetDict):
+            if "test" in ds:
+                hf_dataset = ds["test"]
+            else:
+                raise ValueError(
+                    f"Split not specified and no 'test' split found in dataset '{path}'."
+                )
+        else:
+            raise ValueError(f"Unexpected dataset type for '{path}': {type(ds)}")
+
+        return cls(name=path, label=label, metrics=metrics, hf_dataset=hf_dataset)
+
+    @staticmethod
+    def _resolve_metrics(
+        metrics: Union[
+            str, Type[MetricBase], MetricBase, List[Union[str, Type[MetricBase], MetricBase]]
+        ]
+    ) -> List[MetricBase]:
+        """
+        Convert metric names/classes into a list of MetricBase instances using MetricRegistry.
+
+        Used to normalize metrics to a metric type.
+        """
+        if not isinstance(metrics, list):
+            metrics = [metrics]
+
+        resolved: List[MetricBase] = []
+        for m in metrics:
+            if isinstance(m, MetricBase):
+                resolved.append(m)  # Already an instance
+            else:
+                resolved.append(MetricRegistry.get(m))  # Use registry for str or class
+
+        return resolved

scorebook/types/eval_result.py

@@ -0,0 +1,129 @@
+"""
+This module defines the data structures used to represent evaluation results.
+
+including individual prediction outcomes and aggregated dataset metrics.
+"""
+
+import csv
+import json
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Dict, List
+
+from scorebook.types.eval_dataset import EvalDataset
+
+
+@dataclass
+class EvalResult:
+    """
+    Container for evaluation results from an entire dataset.
+
+    Attributes:
+        eval_dataset: The dataset used for evaluation.
+        inference_outputs: A list of model predictions or outputs.
+        metric_scores: A dictionary mapping metric names to their scores.
+    """
+
+    eval_dataset: EvalDataset
+    inference_outputs: List[Any]
+    metric_scores: Dict[str, Dict[str, Any]]
+    hyperparams: Dict[str, Any]
+
+    @property
+    def item_scores(self) -> List[Dict[str, Any]]:
+        """Return a list of dictionaries containing scores for each evaluated item."""
+        results = []
+        metric_names = list(self.metric_scores.keys()) if self.metric_scores else []
+
+        for idx, item in enumerate(self.eval_dataset.items):
+            if idx >= len(self.inference_outputs):
+                break
+
+            result = {
+                "item_id": idx,
+                "dataset_name": self.eval_dataset.name,
+                **{
+                    metric: self.metric_scores[metric]["item_scores"][idx]
+                    for metric in metric_names
+                },
+                **self.hyperparams,
+            }
+            results.append(result)
+
+        return results
+
+    @property
+    def aggregate_scores(self) -> Dict[str, Any]:
+        """Return the aggregated scores across all evaluated items."""
+        result: Dict[str, Any] = {"dataset_name": self.eval_dataset.name}
+        if not self.metric_scores:
+            return result
+
+        for metric, scores in self.metric_scores.items():
+            # Flatten the aggregate scores from each metric into the result
+            result.update(
+                {
+                    key if key == metric else f"{metric}_{key}": value
+                    for key, value in scores["aggregate_scores"].items()
+                }
+            )
+        for hyperparam, value in self.hyperparams.items():
+            result[hyperparam] = value
+        return result
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Return a dictionary representing the evaluation results."""
+        return {
+            "aggregate": [
+                {
+                    **getattr(self.eval_dataset, "hyperparams", {}),
+                    **self.aggregate_scores,
+                }
+            ],
+            "per_sample": [item for item in self.item_scores],
+        }
+
+    def to_csv(self, file_path: str) -> None:
+        """Save evaluation results to a CSV file.
+
+        The CSV will contain item-level results.
+        """
+        Path(file_path).parent.mkdir(parents=True, exist_ok=True)
+
+        with open(file_path, "w", newline="") as f:
+            writer = csv.writer(f)
+
+            # Write a header with all possible metric names
+            item_fields = list(self.eval_dataset.items[0].keys()) if self.eval_dataset.items else []
+            metric_names = list(self.metric_scores.keys()) if self.metric_scores else []
+            header = ["item_id"] + item_fields + ["inference_output"] + metric_names
+            writer.writerow(header)
+
+            # Write item data
+            for idx, item in enumerate(self.eval_dataset.items):
+                if idx >= len(self.inference_outputs):
+                    break
+
+                row = (
+                    [idx]
+                    + list(item.values())
+                    + [self.inference_outputs[idx]]
+                    + [self.metric_scores[metric]["item_scores"][idx] for metric in metric_names]
+                )
+                writer.writerow(row)
+
+    def to_json(self, file_path: str) -> None:
+        """Save evaluation results to a JSON file in structured format (Option 2)."""
+        Path(file_path).parent.mkdir(parents=True, exist_ok=True)
+        with open(file_path, "w") as f:
+            json.dump(self.to_dict(), f, indent=2)
+
+    def __str__(self) -> str:
+        """Return a formatted string representation of the evaluation results."""
+        result = [
+            f"Eval Dataset: {self.eval_dataset.name}",
+            "\nAggregate Scores:",
+        ]
+        for metric_name, score in self.aggregate_scores.items():
+            result.append(f"\n  {metric_name}: {score:.4f}")
+        return "".join(result)

scorebook/types/inference_pipeline.py

@@ -0,0 +1,84 @@
+"""
+Inference pipeline implementation for processing items through model inference.
+
+This module provides a pipeline structure for handling model inference tasks,
+supporting preprocessing, model inference, and postprocessing steps in a
+configurable way.
+"""
+
+import asyncio
+from typing import Any, Callable, Dict, List, Optional, cast
+
+
+class InferencePipeline:
+    """A pipeline for processing items through model inference.
+
+    This class implements a three-stage pipeline that handles:
+    1. Preprocessing of input items
+    2. Model inference
+    3. Postprocessing of model outputs
+
+
+    Attributes:
+        model: Name or identifier of the model being used
+        preprocessor: Function to prepare items for model inference
+        inference_function: Function that performs the actual model inference
+        postprocessor: Function to process the model outputs
+    """
+
+    def __init__(
+        self,
+        model: str,
+        inference_function: Callable,
+        preprocessor: Optional[Callable] = None,
+        postprocessor: Optional[Callable] = None,
+    ) -> None:
+        """Initialize the inference pipeline.
+
+        Args:
+            model: Name or identifier of the model to use
+            inference_function: Function that performs model inference
+            preprocessor: Optional function to prepare items for inference.
+            postprocessor: Optional function to process model outputs.
+        """
+        self.model: str = model
+        self.inference_function = inference_function
+        self.preprocessor: Optional[Callable] = preprocessor
+        self.postprocessor: Optional[Callable] = postprocessor
+
+    async def run(self, items: List[Dict[str, Any]], **hyperparameters: Any) -> List[Any]:
+        """Execute the complete inference pipeline on a list of items.
+
+        Args:
+            items: List of items to process through the pipeline
+            **hyperparameters: Model-specific parameters for inference
+
+        Returns:
+            List of processed outputs after running through the complete pipeline
+        """
+        if self.preprocessor:
+            input_items = [self.preprocessor(item) for item in items]
+        else:
+            input_items = items
+
+        if asyncio.iscoroutinefunction(self.inference_function):
+            inference_outputs = await self.inference_function(input_items, **hyperparameters)
+        else:
+            inference_outputs = self.inference_function(input_items, **hyperparameters)
+
+        if self.postprocessor:
+            return [self.postprocessor(inference_output) for inference_output in inference_outputs]
+        else:
+            return cast(List[Any], inference_outputs)
+
+    async def __call__(self, items: List[Dict[str, Any]], **hyperparameters: Any) -> List[Any]:
+        """Make the pipeline instance callable by wrapping the run method.
+
+        Args:
+            items: List of items to process through the pipeline
+            **hyperparameters: Model-specific parameters for inference
+
+        Returns:
+            List of processed outputs after running through the complete pipeline
+        """
+        return await self.run(items, **hyperparameters)