scorebook 0.0.1__py3-none-any.whl → 0.0.3__py3-none-any.whl

scorebook/__init__.py

@@ -11,5 +11,6 @@ __version__ = importlib.metadata.version(__package__ or __name__)
 
 from scorebook.evaluator import evaluate
 from scorebook.types.eval_dataset import EvalDataset
+from scorebook.utils.build_prompt import build_prompt
 
-__all__ = ["EvalDataset", "evaluate"]
+__all__ = ["EvalDataset", "evaluate", "build_prompt"]

scorebook/evaluator.py

@@ -14,74 +14,33 @@ models on datasets and computing metric scores.
 """
 
 import asyncio
-from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union
-
-from scorebook.types.eval_dataset import EvalDataset
-from scorebook.types.eval_result import EvalResult
+import logging
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
+
+from scorebook.exceptions import (
+    DataMismatchError,
+    MetricComputationError,
+    ParallelExecutionError,
+    ParameterValidationError,
+)
+from scorebook.types import EvalDataset, EvalResult, EvalRunSpec
 from scorebook.utils import evaluation_progress, expand_dict, is_awaitable
 
-
-async def _evaluate_async(
-    inference_callable: Callable,
-    eval_datasets: Union[str, EvalDataset, List[Union[str, EvalDataset]]],
-    hyperparameters: Optional[Dict[str, Any]] = None,
-    experiment_id: Optional[str] = None,
-    item_limit: Optional[int] = None,
-    return_type: str = "dict",
-    score_type: str = "aggregate",
-) -> Union[Dict, List]:
-    """Run inference across datasets/hyperparams, compute metrics, and format results."""
-    _validate_score_type(score_type)
-
-    normalized_datasets = _normalize_datasets(eval_datasets)
-    hyperparam_grid = _expand_hyperparams(hyperparameters)
-
-    eval_results: List[EvalResult] = []
-
-    with evaluation_progress(normalized_datasets, len(hyperparam_grid)) as progress_bars:
-        # Loop through datasets, then hyperparameters for clear progress tracking
-        for dataset_idx, eval_dataset in enumerate(normalized_datasets):
-            with progress_bars.hyperparam_progress_context():
-                # Run inference for each hyperparameter configuration on this dataset
-                for hp_idx, hyperparam_config in enumerate(hyperparam_grid):
-                    items = _clip_items(eval_dataset.items, item_limit)
-                    labels = _labels_for(items, eval_dataset.label)
-
-                    # 1) Run inference
-                    outputs = await _run_inference_callable(
-                        inference_callable, items, hyperparam_config
-                    )
-
-                    # 2) Score metrics
-                    metric_scores = _score_metrics(eval_dataset, outputs, labels)
-
-                    # 3) Wrap into EvalResult
-                    eval_results.append(
-                        EvalResult(eval_dataset, outputs, metric_scores, hyperparam_config)
-                    )
-
-                    # Update inner progress bar
-                    progress_bars.update_hyperparam_progress()
-
-            # Update the outer progress bar
-            progress_bars.update_dataset_progress()
-
-    # TODO: experiment_id handling (left as passthrough to preserve behavior)
-    if experiment_id:
-        pass
-
-    # 4) Format as requested
-    return _format_results(eval_results, return_type, score_type)
+logger = logging.getLogger(__name__)
 
 
 def evaluate(
     inference_callable: Callable,
     eval_datasets: Union[str, EvalDataset, List[Union[str, EvalDataset]]],
-    hyperparameters: Optional[Dict[str, Any]] = None,
+    hyperparameters: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
     experiment_id: Optional[str] = None,
-    item_limit: Optional[int] = None,
-    return_type: str = "dict",
-    score_type: str = "aggregate",
+    project_id: Optional[str] = None,
+    parallel: bool = False,
+    return_dict: bool = True,
+    return_aggregates: bool = True,
+    return_items: bool = False,
+    return_output: bool = False,
+    sample_size: Optional[int] = None,
 ) -> Union[Dict, List]:
     """
     Evaluate model predictions using specified metrics on given datasets.
@@ -101,12 +60,12 @@ def evaluate(
                  - A list of string identifiers
         hyperparameters: Optional dictionary containing hyperparameter sweep configuration.
         experiment_id: Optional string identifier for tracking multiple evaluation runs.
-        item_limit: Optional integer limiting the number of items to evaluate per dataset.
-        return_type: Format of the return value. Currently only "dict" is supported.
-        score_type: Type of score aggregation to return. Options:
-                   - "aggregate": Return aggregated metrics
-                   - "item": Return per-item scores
-                   - "all": Return both aggregate and per-item scores
+        return_dict: If True, returns eval results as a dict
+        return_aggregates: If True, returns aggregate scores for each dataset
+        return_items: If True, returns individual items for each dataset
+        return_output: If True, returns model outputs for each dataset item evaluated
+        sample_size: If set, only return a sample of the dataset items (for debugging)
+        parallel: If True, run inference functions in parallel (requires all functions to be async)
 
     Returns:
         Dictionary mapping dataset names to their evaluation results. For each dataset,
@@ -124,46 +83,194 @@ def evaluate(
 
         results = evaluate(inference_fn, dataset, item_limit=100)
     """
+
+    logger.info(
+        "Starting evaluation: experiment_id=%s, project_id=%s, parallel=%s",
+        experiment_id,
+        project_id,
+        parallel,
+    )
+
     return asyncio.run(
         _evaluate_async(
             inference_callable=inference_callable,
             eval_datasets=eval_datasets,
             hyperparameters=hyperparameters,
             experiment_id=experiment_id,
-            item_limit=item_limit,
-            return_type=return_type,
-            score_type=score_type,
+            project_id=project_id,
+            parallel=parallel,
+            return_dict=return_dict,
+            return_aggregates=return_aggregates,
+            return_items=return_items,
+            return_output=return_output,
+            sample_size=sample_size,
         )
     )
 
 
-# ===== Helper Functions =====
+async def _evaluate_async(
+    inference_callable: Callable,
+    eval_datasets: Union[str, EvalDataset, List[Union[str, EvalDataset]]],
+    hyperparameters: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
+    experiment_id: Optional[str] = None,
+    project_id: Optional[str] = None,
+    return_dict: bool = True,
+    return_aggregates: bool = True,
+    return_items: bool = False,
+    return_output: bool = False,
+    parallel: bool = False,
+    sample_size: Optional[int] = None,
+) -> Union[Dict, List]:
+    _validate_parameters(locals())
+    datasets, adaptive_datasets = _prepare_datasets(eval_datasets, sample_size)
+    hyperparameters = _prepare_hyperparameters(hyperparameters)
+
+    logger.info(
+        "Prepared %d datasets and %d hyperparameter configurations",
+        len(datasets),
+        len(hyperparameters),
+    )
+
+    runs = _build_runs(datasets, hyperparameters)
+    runs.sort(key=lambda run: (run.dataset_idx, run.hp_idx))
+
+    logger.info("Created %d evaluation runs", len(runs))
+
+    with evaluation_progress(datasets, len(hyperparameters), parallel, len(runs)) as progress_bars:
+        if parallel:
+            eval_results = await _run_parallel(inference_callable, runs, progress_bars)
+        else:
+            eval_results = await _run_sequential(inference_callable, runs, progress_bars)
+
+        logger.info("Evaluation completed successfully")
+
+    return _format_results(
+        eval_results, return_dict, return_aggregates, return_items, return_output
+    )
+
+
+# ===== ORCHESTRATION PATHS =====
+
 
+async def _run_parallel(
+    inference_callable: Callable,
+    runs: List[EvalRunSpec],
+    progress_bars: Any,
+) -> List[EvalResult]:
+    logger.debug("Running inference in parallel")
+
+    async def worker(run: EvalRunSpec) -> Tuple[EvalRunSpec, EvalResult]:
+        er = await _execute_run(inference_callable, run)
+        progress_bars.on_eval_run_completed(run.dataset_idx)
+        return run, er
+
+    pairs = await asyncio.gather(*[worker(r) for r in runs])
+    # Return in canonical (dataset_idx, hp_idx) order for stability
+    pairs.sort(key=lambda p: (p[0].dataset_idx, p[0].hp_idx))
+    return [er for _, er in pairs]
+
+
+async def _run_sequential(
+    inference_callable: Callable,
+    runs: List[EvalRunSpec],
+    progress_bars: Any,
+) -> List[EvalResult]:
+    logger.debug("Running inference sequentially")
+    results: List[EvalResult] = []
+    for run in runs:
+        er = await _execute_run(inference_callable, run)
+        results.append(er)
+        progress_bars.on_hyperparam_completed(run.dataset_idx)
+    return results
+
+
+# ===== EVALUATION EXECUTIONS =====
+
+
+async def _execute_run(inference_callable: Callable, run: EvalRunSpec) -> EvalResult:
+    logger.debug("Executing run for %s", run)
+
+    outputs = await _run_inference_callable(inference_callable, run.items, run.hyperparams)
+    logger.debug("Inference completed for run %s", run)
+
+    metric_scores = _score_metrics(run.eval_dataset, outputs, run.labels)
+    logger.debug("Metrics computed for run %s. - scores:  %s", run, list(metric_scores.keys()))
+
+    return EvalResult(run.eval_dataset, outputs, metric_scores, run.hyperparams)
+
+
+# ===== HELPER FUNCTIONS =====
+
+
+def _validate_parameters(params: Dict[str, Any]) -> None:
+    """Validate all parameters for evaluation."""
 
-def _normalize_datasets(
-    datasets: Union[str, EvalDataset, List[Union[str, EvalDataset]]]
-) -> List[EvalDataset]:
+    if params["return_dict"] and not params["return_aggregates"] and not params["return_items"]:
+        raise ParameterValidationError(
+            "When return_dict=True, at least one of return_aggregates or return_items must be True"
+        )
+
+    if params["parallel"] and not is_awaitable(params["inference_callable"]):
+        raise ParallelExecutionError(
+            "parallel=True requires the inference_callable to be async. "
+            "Please make your inference function async or set parallel=False."
+        )
+
+
+def _prepare_datasets(
+    datasets: Union[str, EvalDataset, List[Union[str, EvalDataset]]],
+    sample_size: Optional[int] = None,
+) -> Tuple[List[EvalDataset], List[str]]:
+    """Prepare and separate input datasets into classic and adaptive evaluation datasets."""
+
+    # Ensure datasets is always a list for consistent processing
     if not isinstance(datasets, list):
         datasets = [datasets]
-    # TODO: handle other types (string registry, etc.)
-    return [d for d in datasets if isinstance(d, EvalDataset)]
 
+    # Extract classical datasets TODO: handle other types (string registry)
+    classic_eval_datasets = [dataset for dataset in datasets if isinstance(dataset, EvalDataset)]
 
-def _validate_score_type(score_type: str) -> None:
-    if score_type not in {"aggregate", "item", "all"}:
-        raise ValueError("score_type must be 'aggregate', 'item', or 'all'")
+    # Reduce datasets to a random sample
+    if sample_size:
+        logger.info("Sampling datasets to %d items each", sample_size)
+        for dataset in classic_eval_datasets:
+            dataset.shuffle()
+            if len(dataset) > sample_size:
+                original_size = len(dataset)
+                dataset._hf_dataset = dataset._hf_dataset.select(range(sample_size))
+                logger.debug(
+                    "Sampled dataset '%s' from %d to %d items",
+                    dataset.name,
+                    original_size,
+                    sample_size,
+                )
 
+    # Extract adaptive dataset strings
+    adaptive_eval_datasets = [
+        dataset.replace(":adaptive", "")
+        for dataset in datasets
+        if isinstance(dataset, str) and dataset.endswith(":adaptive")
+    ]
 
-def _expand_hyperparams(hyperparameters: Optional[Dict[str, Any]]) -> Any:
-    return expand_dict(hyperparameters or {})
+    logger.info("Evaluating on classic datasets: %s", [ds.name for ds in classic_eval_datasets])
+    logger.info("Evaluating on adaptive datasets: %s", adaptive_eval_datasets)
 
+    return classic_eval_datasets, adaptive_eval_datasets
 
-def _clip_items(items: List[Dict[str, Any]], item_limit: Optional[int]) -> List[Dict[str, Any]]:
-    return items[:item_limit] if item_limit else items
 
+def _prepare_hyperparameters(
+    hyperparameters: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]]
+) -> List[Dict[str, Any]]:
+    """Prepare hyperparameters for evaluation by returning a list of hyper-param configs."""
+    if hyperparameters is None:
+        return [{}]
+    if not isinstance(hyperparameters, list):
+        expanded: List[Dict[str, Any]] = expand_dict(hyperparameters or {})
+        return expanded
 
-def _labels_for(items: List[Dict[str, Any]], label_key: str) -> List[Any]:
-    return [item.get(label_key) for item in items]
+    logger.info("Evaluating with hyperparameters: %s", hyperparameters)
+
+    return hyperparameters
 
 
 async def _run_inference_callable(
@@ -177,52 +284,96 @@ async def _run_inference_callable(
         return inference_callable(items, **hyperparams)
 
 
-# Yields (eval_dataset, items, labels, hyperparams) for every dataset x hyperparam combo.
-def _iter_dataset_jobs(
+def _build_runs(
     datasets: List[EvalDataset],
-    hyperparam_grid: List[Dict[str, Any]],
-    item_limit: Optional[int],
-) -> Iterable[Tuple[EvalDataset, List[Dict[str, Any]], List[Any], Dict[str, Any]]]:
-    for eval_dataset in datasets:
-        for hp in hyperparam_grid:
-            items = _clip_items(eval_dataset.items, item_limit)
-            labels = _labels_for(items, eval_dataset.label)
-            yield eval_dataset, items, labels, hp
+    hyperparameters: List[Dict[str, Any]],
+) -> List[EvalRunSpec]:
+    """Build RunSpec objects for each dataset/hyperparameter combination."""
+    runs: List[EvalRunSpec] = []
+    for d_idx, ds in enumerate(datasets):
+        items = ds.items
+        labels = [item.get(ds.label) for item in items]
+        for hp_idx, hp in enumerate(hyperparameters):
+            run_spec = EvalRunSpec(d_idx, ds, items, labels, hp, hp_idx)
+            logger.debug("Built RunSpec: %s", run_spec)
+            runs.append(run_spec)
+    return runs
 
 
 def _score_metrics(
     eval_dataset: EvalDataset, outputs: List[Any], labels: List[Any]
 ) -> Dict[str, Dict[str, Any]]:
+    """Compute metric scores for a given dataset and inference outputs."""
     metric_scores: Dict[str, Dict[str, Any]] = {}
+
+    if len(outputs) != len(labels):
+        raise DataMismatchError(len(outputs), len(labels), eval_dataset.name)
+
     for metric in eval_dataset.metrics:
-        aggregate_scores, item_scores = metric.score(outputs, labels)
-        metric_scores[metric.name] = {
-            "aggregate_scores": aggregate_scores,
-            "item_scores": item_scores,
-        }
+        try:
+            aggregate_scores, item_scores = metric.score(outputs, labels)
+            metric_scores[metric.name] = {
+                "aggregate_scores": aggregate_scores,
+                "item_scores": item_scores,
+            }
+        except Exception as e:
+            logger.error(
+                "Failed to compute metric '%s' for dataset '%s': %s",
+                metric.name,
+                eval_dataset.name,
+                str(e),
+            )
+            raise MetricComputationError(metric.name, eval_dataset.name, e)
+
     return metric_scores
 
 
 def _format_results(
-    eval_results: List[EvalResult], return_type: str, score_type: str
+    eval_results: List[EvalResult],
+    return_dict: bool,
+    return_aggregates: bool,
+    return_items: bool,
+    return_output: bool,
 ) -> Union[Dict, List]:
 
-    if return_type != "dict":
-        return {er.eval_dataset.name: er for er in eval_results}
-
-    if score_type == "all":
-        combined: Dict[str, List[Dict[str, Any]]] = {"aggregate": [], "per_sample": []}
+    # Return results as a dict
+    if return_dict:
+
+        # Include both aggregate and item scores in dict returned
+        if return_aggregates and return_items:
+            results: Dict[str, List[Dict[str, Any]]] = {"aggregate_results": [], "item_results": []}
+            for eval_result in eval_results:
+                eval_result_dict = eval_result.to_dict()
+                results["aggregate_results"].extend(eval_result_dict["aggregate_results"])
+                if return_output:
+                    results["item_results"].extend(eval_result_dict["item_results"])
+                else:
+                    results["item_results"].extend(
+                        [
+                            {k: v for k, v in item.items() if k != "inference_output"}
+                            for item in eval_result_dict["item_results"]
+                        ]
+                    )
+            return results
+
+        # Include only aggregate scores in dict returned
+        elif return_aggregates:
+            return [eval_result.aggregate_scores for eval_result in eval_results]
+
+        # Include only item scores in dict returned
+        else:
+            if return_output:
+                return [item for eval_result in eval_results for item in eval_result.item_scores]
+            else:
+                return [
+                    {k: v for k, v in item.items() if k != "inference_output"}
+                    for eval_result in eval_results
+                    for item in eval_result.item_scores
+                ]
+
+    # Return results as an EvalResult object
+    else:
+        out: Dict[str, List[EvalResult]] = {}
         for er in eval_results:
-            d = er.to_dict()
-            combined["aggregate"].extend(d["aggregate"])
-            combined["per_sample"].extend(d["per_sample"])
-        return combined
-
-    if score_type == "aggregate":
-        return [er.aggregate_scores for er in eval_results]
-
-    if score_type == "item":
-        return [item for er in eval_results for item in er.item_scores]
-
-    # Should be unreachable due to validation
-    return {}
+            out.setdefault(er.eval_dataset.name, []).append(er)
+        return out

scorebook/exceptions.py

@@ -0,0 +1,54 @@
+"""
+Custom exceptions for the Scorebook framework.
+
+This module defines specific exception types used throughout the Scorebook
+evaluation framework to provide clear error handling and debugging information.
+"""
+
+
+class ScoreBookError(Exception):
+    """Base exception class for all Scorebook-related errors."""
+
+
+class EvaluationError(ScoreBookError):
+    """Raised when there are errors during model evaluation."""
+
+
+class ParameterValidationError(ScoreBookError):
+    """Raised when invalid parameters are provided to evaluation functions."""
+
+
+class InferenceError(EvaluationError):
+    """Raised when there are errors during model inference."""
+
+
+class MetricComputationError(EvaluationError):
+    """Raised when metric computation fails."""
+
+    def __init__(self, metric_name: str, dataset_name: str, original_error: Exception):
+        """Initialize metric computation error."""
+        self.metric_name = metric_name
+        self.dataset_name = dataset_name
+        self.original_error = original_error
+        super().__init__(
+            f"Failed to compute metric '{metric_name}' for dataset "
+            f"'{dataset_name}': {original_error}"
+        )
+
+
+class DataMismatchError(EvaluationError):
+    """Raised when there's a mismatch between outputs and expected labels."""
+
+    def __init__(self, outputs_count: int, labels_count: int, dataset_name: str):
+        """Initialize data mismatch error."""
+        self.outputs_count = outputs_count
+        self.labels_count = labels_count
+        self.dataset_name = dataset_name
+        super().__init__(
+            f"Output count ({outputs_count}) doesn't match label count ({labels_count}) "
+            f"for dataset '{dataset_name}'"
+        )
+
+
+class ParallelExecutionError(ScoreBookError):
+    """Raised when parallel execution requirements are not met."""

scorebook/inference/__init__.py

@@ -5,7 +5,3 @@ This module provides functionality for running inference with various models
 and processing their responses. It includes utilities for both single and
 batch inference operations.
 """
-
-from scorebook.inference.openai import batch, responses
-
-__all__ = ["responses", "batch"]