scorebook 0.0.2__tar.gz → 0.0.3__tar.gz

{scorebook-0.0.2 → scorebook-0.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: scorebook
-Version: 0.0.2
+Version: 0.0.3
 Summary: A Python project for LLM evaluation.
 Author: Euan Campbell
 Author-email: euan@trismik.com

{scorebook-0.0.2 → scorebook-0.0.3}/pyproject.toml

@@ -14,7 +14,7 @@ dependencies = [
 ]
 
 [tool.poetry]
-version = "0.0.2"  # base version
+version = "0.0.3"  # base version
 packages = [{ include = "scorebook", from = "src" }]
 
 

scorebook-0.0.3/src/scorebook/evaluator.py

@@ -0,0 +1,379 @@
+"""
+Model evaluation functionality for the Scorebook framework.
+
+This module provides the core evaluation logic to assess model predictions
+against ground truth labels using configurable metrics. It supports:
+
+- Batch evaluation of models across multiple datasets
+- Flexible metric computation and aggregation
+- Optional parameter sweeping and experiment tracking
+- Customizable inference functions
+
+The main entry point is the `evaluate()` function which handles running
+models on datasets and computing metric scores.
+"""
+
+import asyncio
+import logging
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
+
+from scorebook.exceptions import (
+    DataMismatchError,
+    MetricComputationError,
+    ParallelExecutionError,
+    ParameterValidationError,
+)
+from scorebook.types import EvalDataset, EvalResult, EvalRunSpec
+from scorebook.utils import evaluation_progress, expand_dict, is_awaitable
+
+logger = logging.getLogger(__name__)
+
+
+def evaluate(
+    inference_callable: Callable,
+    eval_datasets: Union[str, EvalDataset, List[Union[str, EvalDataset]]],
+    hyperparameters: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
+    experiment_id: Optional[str] = None,
+    project_id: Optional[str] = None,
+    parallel: bool = False,
+    return_dict: bool = True,
+    return_aggregates: bool = True,
+    return_items: bool = False,
+    return_output: bool = False,
+    sample_size: Optional[int] = None,
+) -> Union[Dict, List]:
+    """
+    Evaluate model predictions using specified metrics on given datasets.
+
+    This function runs the provided inference callable on one or more evaluation datasets,
+    computes metric scores, and returns the evaluation results. It supports batch processing,
+    parameter sweeping, and different result formatting options.
+
+    Args:
+        inference_callable: A callable function or object that takes (items, hyperparameters)
+                           and returns predictions. Can be a regular function, async function,
+                           or callable instance (like a class with __call__ method).
+        eval_datasets: One or more evaluation datasets to run evaluation on. Can be:
+                 - A single EvalDataset instance
+                 - A list of EvalDataset instances
+                 - A string identifier (for future dataset registry support)
+                 - A list of string identifiers
+        hyperparameters: Optional dictionary containing hyperparameter sweep configuration.
+        experiment_id: Optional string identifier for tracking multiple evaluation runs.
+        return_dict: If True, returns eval results as a dict
+        return_aggregates: If True, returns aggregate scores for each dataset
+        return_items: If True, returns individual items for each dataset
+        return_output: If True, returns model outputs for each dataset item evaluated
+        sample_size: If set, only return a sample of the dataset items (for debugging)
+        parallel: If True, run inference functions in parallel (requires all functions to be async)
+
+    Returns:
+        Dictionary mapping dataset names to their evaluation results. For each dataset,
+        returns a dictionary containing:
+        - items: List of EvalResult objects with predictions and ground truth
+        - metrics: Dictionary mapping metric names to their computed scores
+
+    Example:
+
+    python
+        dataset = EvalDataset.from_huggingface("dataset_name", label="answer", metrics=[Precision])
+        def inference_fn(items):
+            # Model inference logic here - process all items at once
+            return [prediction for item in items]
+
+        results = evaluate(inference_fn, dataset, item_limit=100)
+    """
+
+    logger.info(
+        "Starting evaluation: experiment_id=%s, project_id=%s, parallel=%s",
+        experiment_id,
+        project_id,
+        parallel,
+    )
+
+    return asyncio.run(
+        _evaluate_async(
+            inference_callable=inference_callable,
+            eval_datasets=eval_datasets,
+            hyperparameters=hyperparameters,
+            experiment_id=experiment_id,
+            project_id=project_id,
+            parallel=parallel,
+            return_dict=return_dict,
+            return_aggregates=return_aggregates,
+            return_items=return_items,
+            return_output=return_output,
+            sample_size=sample_size,
+        )
+    )
+
+
+async def _evaluate_async(
+    inference_callable: Callable,
+    eval_datasets: Union[str, EvalDataset, List[Union[str, EvalDataset]]],
+    hyperparameters: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
+    experiment_id: Optional[str] = None,
+    project_id: Optional[str] = None,
+    return_dict: bool = True,
+    return_aggregates: bool = True,
+    return_items: bool = False,
+    return_output: bool = False,
+    parallel: bool = False,
+    sample_size: Optional[int] = None,
+) -> Union[Dict, List]:
+    _validate_parameters(locals())
+    datasets, adaptive_datasets = _prepare_datasets(eval_datasets, sample_size)
+    hyperparameters = _prepare_hyperparameters(hyperparameters)
+
+    logger.info(
+        "Prepared %d datasets and %d hyperparameter configurations",
+        len(datasets),
+        len(hyperparameters),
+    )
+
+    runs = _build_runs(datasets, hyperparameters)
+    runs.sort(key=lambda run: (run.dataset_idx, run.hp_idx))
+
+    logger.info("Created %d evaluation runs", len(runs))
+
+    with evaluation_progress(datasets, len(hyperparameters), parallel, len(runs)) as progress_bars:
+        if parallel:
+            eval_results = await _run_parallel(inference_callable, runs, progress_bars)
+        else:
+            eval_results = await _run_sequential(inference_callable, runs, progress_bars)
+
+        logger.info("Evaluation completed successfully")
+
+    return _format_results(
+        eval_results, return_dict, return_aggregates, return_items, return_output
+    )
+
+
+# ===== ORCHESTRATION PATHS =====
+
+
+async def _run_parallel(
+    inference_callable: Callable,
+    runs: List[EvalRunSpec],
+    progress_bars: Any,
+) -> List[EvalResult]:
+    logger.debug("Running inference in parallel")
+
+    async def worker(run: EvalRunSpec) -> Tuple[EvalRunSpec, EvalResult]:
+        er = await _execute_run(inference_callable, run)
+        progress_bars.on_eval_run_completed(run.dataset_idx)
+        return run, er
+
+    pairs = await asyncio.gather(*[worker(r) for r in runs])
+    # Return in canonical (dataset_idx, hp_idx) order for stability
+    pairs.sort(key=lambda p: (p[0].dataset_idx, p[0].hp_idx))
+    return [er for _, er in pairs]
+
+
+async def _run_sequential(
+    inference_callable: Callable,
+    runs: List[EvalRunSpec],
+    progress_bars: Any,
+) -> List[EvalResult]:
+    logger.debug("Running inference sequentially")
+    results: List[EvalResult] = []
+    for run in runs:
+        er = await _execute_run(inference_callable, run)
+        results.append(er)
+        progress_bars.on_hyperparam_completed(run.dataset_idx)
+    return results
+
+
+# ===== EVALUATION EXECUTIONS =====
+
+
+async def _execute_run(inference_callable: Callable, run: EvalRunSpec) -> EvalResult:
+    logger.debug("Executing run for %s", run)
+
+    outputs = await _run_inference_callable(inference_callable, run.items, run.hyperparams)
+    logger.debug("Inference completed for run %s", run)
+
+    metric_scores = _score_metrics(run.eval_dataset, outputs, run.labels)
+    logger.debug("Metrics computed for run %s. - scores:  %s", run, list(metric_scores.keys()))
+
+    return EvalResult(run.eval_dataset, outputs, metric_scores, run.hyperparams)
+
+
+# ===== HELPER FUNCTIONS =====
+
+
+def _validate_parameters(params: Dict[str, Any]) -> None:
+    """Validate all parameters for evaluation."""
+
+    if params["return_dict"] and not params["return_aggregates"] and not params["return_items"]:
+        raise ParameterValidationError(
+            "When return_dict=True, at least one of return_aggregates or return_items must be True"
+        )
+
+    if params["parallel"] and not is_awaitable(params["inference_callable"]):
+        raise ParallelExecutionError(
+            "parallel=True requires the inference_callable to be async. "
+            "Please make your inference function async or set parallel=False."
+        )
+
+
+def _prepare_datasets(
+    datasets: Union[str, EvalDataset, List[Union[str, EvalDataset]]],
+    sample_size: Optional[int] = None,
+) -> Tuple[List[EvalDataset], List[str]]:
+    """Prepare and separate input datasets into classic and adaptive evaluation datasets."""
+
+    # Ensure datasets is always a list for consistent processing
+    if not isinstance(datasets, list):
+        datasets = [datasets]
+
+    # Extract classical datasets TODO: handle other types (string registry)
+    classic_eval_datasets = [dataset for dataset in datasets if isinstance(dataset, EvalDataset)]
+
+    # Reduce datasets to a random sample
+    if sample_size:
+        logger.info("Sampling datasets to %d items each", sample_size)
+        for dataset in classic_eval_datasets:
+            dataset.shuffle()
+            if len(dataset) > sample_size:
+                original_size = len(dataset)
+                dataset._hf_dataset = dataset._hf_dataset.select(range(sample_size))
+                logger.debug(
+                    "Sampled dataset '%s' from %d to %d items",
+                    dataset.name,
+                    original_size,
+                    sample_size,
+                )
+
+    # Extract adaptive dataset strings
+    adaptive_eval_datasets = [
+        dataset.replace(":adaptive", "")
+        for dataset in datasets
+        if isinstance(dataset, str) and dataset.endswith(":adaptive")
+    ]
+
+    logger.info("Evaluating on classic datasets: %s", [ds.name for ds in classic_eval_datasets])
+    logger.info("Evaluating on adaptive datasets: %s", adaptive_eval_datasets)
+
+    return classic_eval_datasets, adaptive_eval_datasets
+
+
+def _prepare_hyperparameters(
+    hyperparameters: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]]
+) -> List[Dict[str, Any]]:
+    """Prepare hyperparameters for evaluation by returning a list of hyper-param configs."""
+    if hyperparameters is None:
+        return [{}]
+    if not isinstance(hyperparameters, list):
+        expanded: List[Dict[str, Any]] = expand_dict(hyperparameters or {})
+        return expanded
+
+    logger.info("Evaluating with hyperparameters: %s", hyperparameters)
+
+    return hyperparameters
+
+
+async def _run_inference_callable(
+    inference_callable: Callable,
+    items: List[Dict[str, Any]],
+    hyperparams: Dict[str, Any],
+) -> Any:
+    if is_awaitable(inference_callable):
+        return await inference_callable(items, **hyperparams)
+    else:
+        return inference_callable(items, **hyperparams)
+
+
+def _build_runs(
+    datasets: List[EvalDataset],
+    hyperparameters: List[Dict[str, Any]],
+) -> List[EvalRunSpec]:
+    """Build RunSpec objects for each dataset/hyperparameter combination."""
+    runs: List[EvalRunSpec] = []
+    for d_idx, ds in enumerate(datasets):
+        items = ds.items
+        labels = [item.get(ds.label) for item in items]
+        for hp_idx, hp in enumerate(hyperparameters):
+            run_spec = EvalRunSpec(d_idx, ds, items, labels, hp, hp_idx)
+            logger.debug("Built RunSpec: %s", run_spec)
+            runs.append(run_spec)
+    return runs
+
+
+def _score_metrics(
+    eval_dataset: EvalDataset, outputs: List[Any], labels: List[Any]
+) -> Dict[str, Dict[str, Any]]:
+    """Compute metric scores for a given dataset and inference outputs."""
+    metric_scores: Dict[str, Dict[str, Any]] = {}
+
+    if len(outputs) != len(labels):
+        raise DataMismatchError(len(outputs), len(labels), eval_dataset.name)
+
+    for metric in eval_dataset.metrics:
+        try:
+            aggregate_scores, item_scores = metric.score(outputs, labels)
+            metric_scores[metric.name] = {
+                "aggregate_scores": aggregate_scores,
+                "item_scores": item_scores,
+            }
+        except Exception as e:
+            logger.error(
+                "Failed to compute metric '%s' for dataset '%s': %s",
+                metric.name,
+                eval_dataset.name,
+                str(e),
+            )
+            raise MetricComputationError(metric.name, eval_dataset.name, e)
+
+    return metric_scores
+
+
+def _format_results(
+    eval_results: List[EvalResult],
+    return_dict: bool,
+    return_aggregates: bool,
+    return_items: bool,
+    return_output: bool,
+) -> Union[Dict, List]:
+
+    # Return results as a dict
+    if return_dict:
+
+        # Include both aggregate and item scores in dict returned
+        if return_aggregates and return_items:
+            results: Dict[str, List[Dict[str, Any]]] = {"aggregate_results": [], "item_results": []}
+            for eval_result in eval_results:
+                eval_result_dict = eval_result.to_dict()
+                results["aggregate_results"].extend(eval_result_dict["aggregate_results"])
+                if return_output:
+                    results["item_results"].extend(eval_result_dict["item_results"])
+                else:
+                    results["item_results"].extend(
+                        [
+                            {k: v for k, v in item.items() if k != "inference_output"}
+                            for item in eval_result_dict["item_results"]
+                        ]
+                    )
+            return results
+
+        # Include only aggregate scores in dict returned
+        elif return_aggregates:
+            return [eval_result.aggregate_scores for eval_result in eval_results]
+
+        # Include only item scores in dict returned
+        else:
+            if return_output:
+                return [item for eval_result in eval_results for item in eval_result.item_scores]
+            else:
+                return [
+                    {k: v for k, v in item.items() if k != "inference_output"}
+                    for eval_result in eval_results
+                    for item in eval_result.item_scores
+                ]
+
+    # Return results as an EvalResult object
+    else:
+        out: Dict[str, List[EvalResult]] = {}
+        for er in eval_results:
+            out.setdefault(er.eval_dataset.name, []).append(er)
+        return out

scorebook-0.0.3/src/scorebook/exceptions.py

@@ -0,0 +1,54 @@
+"""
+Custom exceptions for the Scorebook framework.
+
+This module defines specific exception types used throughout the Scorebook
+evaluation framework to provide clear error handling and debugging information.
+"""
+
+
+class ScoreBookError(Exception):
+    """Base exception class for all Scorebook-related errors."""
+
+
+class EvaluationError(ScoreBookError):
+    """Raised when there are errors during model evaluation."""
+
+
+class ParameterValidationError(ScoreBookError):
+    """Raised when invalid parameters are provided to evaluation functions."""
+
+
+class InferenceError(EvaluationError):
+    """Raised when there are errors during model inference."""
+
+
+class MetricComputationError(EvaluationError):
+    """Raised when metric computation fails."""
+
+    def __init__(self, metric_name: str, dataset_name: str, original_error: Exception):
+        """Initialize metric computation error."""
+        self.metric_name = metric_name
+        self.dataset_name = dataset_name
+        self.original_error = original_error
+        super().__init__(
+            f"Failed to compute metric '{metric_name}' for dataset "
+            f"'{dataset_name}': {original_error}"
+        )
+
+
+class DataMismatchError(EvaluationError):
+    """Raised when there's a mismatch between outputs and expected labels."""
+
+    def __init__(self, outputs_count: int, labels_count: int, dataset_name: str):
+        """Initialize data mismatch error."""
+        self.outputs_count = outputs_count
+        self.labels_count = labels_count
+        self.dataset_name = dataset_name
+        super().__init__(
+            f"Output count ({outputs_count}) doesn't match label count ({labels_count}) "
+            f"for dataset '{dataset_name}'"
+        )
+
+
+class ParallelExecutionError(ScoreBookError):
+    """Raised when parallel execution requirements are not met."""

{scorebook-0.0.2 → scorebook-0.0.3}/src/scorebook/inference/openai.py

@@ -8,17 +8,19 @@ API communication, request formatting, and response processing.
 
 import asyncio
 import json
+import logging
 import tempfile
 from typing import Any, List
 
-from openai import OpenAI
-from tqdm.asyncio import tqdm
+from openai import AsyncOpenAI
+
+logger = logging.getLogger(__name__)
 
 
 async def responses(
     items: List[Any], model: str = "gpt-4.1-nano", client: Any = None, **hyperparameters: Any
 ) -> List[Any]:
-    """Process multiple inference requests using OpenAI's API.
+    """Process multiple inference requests using OpenAI's Async API.
 
     This asynchronous function handles multiple inference requests,
     manages the API communication, and processes the responses.
@@ -35,13 +37,67 @@ async def responses(
     Raises:
         NotImplementedError: Currently not implemented.
     """
-    if client is None:
-        client = OpenAI()
+    logger.debug("OpenAI responses function called with %d items", len(items))
+    logger.debug("Using model: %s", model)
+    logger.debug("Hyperparameters: %s", hyperparameters)
 
-    results = []
-    for item in items:
-        response = client.responses.create(model=model, input=item)
-        results.append(response)
+    if client is None:
+        logger.debug("Creating new AsyncOpenAI client")
+        client = AsyncOpenAI()
+
+    # Create all tasks concurrently for true parallelism
+    tasks = []
+    for i, item in enumerate(items):
+        logger.debug(
+            "Processing item %d: %s",
+            i,
+            str(item)[:100] + "..." if len(str(item)) > 100 else str(item),
+        )
+
+        # Handle string input from preprocessor - convert to proper messages format
+        if isinstance(item, str):
+            # Convert the string format to proper OpenAI messages array
+            messages = [{"role": "user", "content": item}]
+            logger.debug(
+                "Converted string to messages format: %s",
+                (
+                    messages[0]["content"][:100] + "..."
+                    if len(messages[0]["content"]) > 100
+                    else messages[0]["content"]
+                ),
+            )
+        elif isinstance(item, list):
+            # Already in proper messages format
+            messages = item
+            logger.debug("Item %d already in messages format", i)
+        else:
+            # Fallback: treat as user message
+            messages = [{"role": "user", "content": str(item)}]
+            logger.debug("Item %d converted to fallback format", i)
+
+        logger.debug("Creating OpenAI task %d with messages: %s", i, messages)
+        task = client.chat.completions.create(model=model, messages=messages, **hyperparameters)
+        tasks.append(task)
+
+    logger.debug("Created %d tasks, waiting for OpenAI responses...", len(tasks))
+    # Wait for all requests to complete in parallel
+    results = await asyncio.gather(*tasks)
+    logger.debug("Received %d responses from OpenAI", len(results))
+
+    for i, result in enumerate(results):
+        logger.debug("Response %d type: %s", i, type(result))
+        try:
+            if hasattr(result, "choices") and result.choices:
+                content = result.choices[0].message.content
+                logger.debug(
+                    "Response %d content: %s",
+                    i,
+                    content[:100] + "..." if content and len(content) > 100 else content,
+                )
+            else:
+                logger.debug("Response %d has no choices or unexpected format", i)
+        except Exception as e:
+            logger.error("Error logging response %d: %s", i, e)
 
     return results
 
@@ -70,40 +126,23 @@ async def batch(
         NotImplementedError: Currently not implemented.
     """
     if client is None:
-        client = OpenAI()
+        client = AsyncOpenAI()
 
-    file_id = _upload_batch(items, client)
-    batch_id = _start_batch(file_id, client)
-
-    # Initialize progress bar
-    pbar = tqdm(total=len(items), desc="Batch processing", unit="requests")
+    file_id = await _upload_batch(items, client)
+    batch_id = await _start_batch(file_id, client)
 
     awaiting_batch = True
     while awaiting_batch:
         batch_object = await _get_batch(batch_id, client)
         batch_status = batch_object.status
 
-        if hasattr(batch_object, "request_counts") and batch_object.request_counts:
-            completed = batch_object.request_counts.completed
-            total = batch_object.request_counts.total
-            pbar.n = completed
-            pbar.set_postfix(status=batch_status, completed=f"{completed}/{total}")
-        else:
-            pbar.set_postfix(status=batch_status)
-
-        pbar.refresh()
-
         if batch_status == "completed":
             awaiting_batch = False
-            pbar.n = pbar.total
-            pbar.set_postfix(status="completed")
         elif batch_status == "failed":
             raise Exception("Batch processing failed")
         else:
             await asyncio.sleep(60)
 
-    pbar.close()
-
     # Get the final batch object to access output_file_id
     final_batch_object = await _get_batch(batch_id, client)
     output_file_id = final_batch_object.output_file_id
@@ -112,7 +151,7 @@ async def batch(
     return batch_result
 
 
-def _upload_batch(items: List[Any], client: Any) -> str:
+async def _upload_batch(items: List[Any], client: Any) -> str:
     """Create a .jsonl file from preprocessed items and upload to OpenAI for batch processing.
 
     Args:
@@ -121,10 +160,9 @@ def _upload_batch(items: List[Any], client: Any) -> str:
     Returns:
         The file ID returned by OpenAI after uploading.
     """
-    print("Uploading batch...")
     # Instantiate OpenAI client
     if client is None:
-        client = OpenAI()
+        client = AsyncOpenAI()
 
     # Create temp .jsonl file
     with tempfile.NamedTemporaryFile(mode="w+", suffix=".jsonl", delete=False) as f:
@@ -141,13 +179,13 @@ def _upload_batch(items: List[Any], client: Any) -> str:
 
     # Upload file to OpenAI
     with open(file_path, "rb") as upload_file:
-        response = client.files.create(file=upload_file, purpose="batch")
+        response = await client.files.create(file=upload_file, purpose="batch")
 
     return str(response.id)
 
 
-def _start_batch(file_id: str, client: Any) -> str:
-    batch_response = client.batches.create(
+async def _start_batch(file_id: str, client: Any) -> str:
+    batch_response = await client.batches.create(
         input_file_id=file_id,
         endpoint="/v1/chat/completions",
         completion_window="24h",
@@ -156,13 +194,13 @@ def _start_batch(file_id: str, client: Any) -> str:
 
 
 async def _get_batch(batch_id: str, client: Any) -> Any:
-    batch_object = client.batches.retrieve(batch_id)
+    batch_object = await client.batches.retrieve(batch_id)
     return batch_object
 
 
 async def _get_results_file(output_file_id: str, client: Any) -> List[str]:
     """Download and parse the batch results file from OpenAI."""
-    response = client.files.content(output_file_id)
+    response = await client.files.content(output_file_id)
 
     # Parse the JSONL content
     content = response.content.decode("utf-8")

{scorebook-0.0.2 → scorebook-0.0.3}/src/scorebook/types/__init__.py

@@ -7,5 +7,6 @@ and evaluation results.
 
 from scorebook.types.eval_dataset import EvalDataset
 from scorebook.types.eval_result import EvalResult
+from scorebook.types.eval_run_spec import EvalRunSpec
 
-__all__ = ["EvalDataset", "EvalResult"]
+__all__ = ["EvalDataset", "EvalResult", "EvalRunSpec"]

{scorebook-0.0.2 → scorebook-0.0.3}/src/scorebook/types/eval_dataset.py

@@ -86,6 +86,12 @@ class EvalDataset:
             raise ValueError("Dataset is not initialized")
         return iter(self._hf_dataset)
 
+    def shuffle(self) -> None:
+        """Randomly shuffle the dataset items."""
+        if self._hf_dataset is None:
+            raise ValueError("Dataset is not initialized")
+        self._hf_dataset.shuffle()
+
     @property
     def items(self) -> List[Any]:
         """Return a list of all examples in the dataset."""

scorebook-0.0.3/src/scorebook/types/eval_run_spec.py

@@ -0,0 +1,28 @@
+"""Evaluation run specification types for Scorebook."""
+
+from typing import Any, Dict, List, NamedTuple
+
+from scorebook.types import EvalDataset
+
+
+class EvalRunSpec(NamedTuple):
+    """Represents a single evaluation run configuration."""
+
+    dataset_idx: int
+    eval_dataset: EvalDataset
+    items: List[Dict[str, Any]]
+    labels: List[Any]
+    hyperparams: Dict[str, Any]
+    hp_idx: int
+
+    def __str__(self) -> str:
+        """Return a formatted string summary of the evaluation run specification."""
+        hyperparams_str = ", ".join([f"{k}={v}" for k, v in self.hyperparams.items()])
+
+        return (
+            f"EvalRunSpec(dataset_idx={self.dataset_idx},"
+            f"  hp_idx={self.hp_idx},"
+            f"  dataset_name='{self.eval_dataset.name}',"
+            f"  hyperparams=[{hyperparams_str}]"
+            f")"
+        )

{scorebook-0.0.2 → scorebook-0.0.3}/src/scorebook/types/inference_pipeline.py

@@ -57,7 +57,7 @@ class InferencePipeline:
             List of processed outputs after running through the complete pipeline
         """
         if self.preprocessor:
-            input_items = [self.preprocessor(item, hyperparameters) for item in items]
+            input_items = [self.preprocessor(item, **hyperparameters) for item in items]
         else:
             input_items = items
 
@@ -68,7 +68,7 @@ class InferencePipeline:
 
         if self.postprocessor:
             return [
-                self.postprocessor(inference_output, hyperparameters)
+                self.postprocessor(inference_output, **hyperparameters)
                 for inference_output in inference_outputs
             ]
         else:

scorebook-0.0.3/src/scorebook/utils/logging_utils.py

@@ -0,0 +1 @@
+"""Logging utilities for Scorebook evaluation framework."""

scorebook-0.0.3/src/scorebook/utils/progress_bars.py

@@ -0,0 +1,146 @@
+"""Progress bar utilities for evaluation tracking."""
+
+from contextlib import contextmanager
+from typing import Any, Generator, List, Optional
+
+from tqdm import tqdm
+
+
+class EvaluationProgressBars:
+    """Manages nested progress bars for evaluation tracking."""
+
+    def __init__(
+        self, datasets: List[Any], hyperparam_count: int, parallel: bool, total_eval_runs: int
+    ) -> None:
+        """Initialize progress bar manager.
+
+        Args:
+            datasets: List of datasets being evaluated
+            hyperparam_count: Number of hyperparameter configurations per dataset
+            parallel: Whether running in parallel mode
+            total_eval_runs: Total number of EvalRunSpecs (dataset_count * hyperparam_count)
+        """
+        self.datasets = datasets
+        self.hyperparam_count = hyperparam_count
+        self.parallel = parallel
+        self.total_eval_runs = total_eval_runs
+
+        self.dataset_pbar: Optional[tqdm] = None
+        self.hyperparam_pbar: Optional[tqdm] = None
+
+        # Track progress per dataset
+        self.current_dataset_idx = 0
+        self.completed_hyperparams_per_dataset: dict[int, int] = {}
+        self.completed_eval_runs = 0
+
+    def start_progress_bars(self) -> None:
+        """Start both progress bars."""
+        # Top level: Datasets
+        self.dataset_pbar = tqdm(
+            total=len(self.datasets),
+            desc="Datasets   ",
+            unit="dataset",
+            position=0,
+            leave=True,
+            ncols=80,
+            bar_format="{desc} {percentage:3.0f}%|{bar:40}| {n_fmt}/{total_fmt}",
+        )
+
+        # Bottom level: Hyperparameters/Eval runs
+        if self.parallel:
+            # In parallel mode: show eval runs completed out of total
+            self.hyperparam_pbar = tqdm(
+                total=self.total_eval_runs,
+                desc="Eval Runs  ",
+                unit="run",
+                position=1,
+                leave=False,
+                ncols=80,
+                bar_format="{desc} {percentage:3.0f}%|{bar:40}| {n_fmt}/{total_fmt}",
+            )
+        else:
+            # In sequential mode: show hyperparams per dataset
+            self.hyperparam_pbar = tqdm(
+                total=self.hyperparam_count,
+                desc="Hyperparams",
+                unit="config",
+                position=1,
+                leave=False,
+                ncols=80,
+                bar_format="{desc} {percentage:3.0f}%|{bar:40}| {n_fmt}/{total_fmt}",
+            )
+
+    def on_eval_run_completed(self, dataset_idx: int) -> None:
+        """Update progress when an eval run (EvalRunSpec) completes in parallel mode."""
+        if not self.parallel:
+            return
+
+        self.completed_eval_runs += 1
+        if self.hyperparam_pbar:
+            self.hyperparam_pbar.update(1)
+
+        # Track how many runs completed for this dataset
+        self.completed_hyperparams_per_dataset[dataset_idx] = (
+            self.completed_hyperparams_per_dataset.get(dataset_idx, 0) + 1
+        )
+
+        # Check if this dataset is complete
+        if self.completed_hyperparams_per_dataset[dataset_idx] == self.hyperparam_count:
+            if self.dataset_pbar:
+                self.dataset_pbar.update(1)
+
+    def on_hyperparam_completed(self, dataset_idx: int) -> None:
+        """Update progress when a hyperparameter config completes in sequential mode."""
+        if self.parallel:
+            return
+
+        if self.hyperparam_pbar:
+            self.hyperparam_pbar.update(1)
+
+        # Track completed hyperparams for this dataset
+        self.completed_hyperparams_per_dataset[dataset_idx] = (
+            self.completed_hyperparams_per_dataset.get(dataset_idx, 0) + 1
+        )
+
+        # Check if this dataset is complete
+        if self.completed_hyperparams_per_dataset[dataset_idx] == self.hyperparam_count:
+            # Update dataset progress
+            if self.dataset_pbar:
+                self.dataset_pbar.update(1)
+
+            # Reset hyperparameter progress for next dataset (if any)
+            if dataset_idx < len(self.datasets) - 1:
+                if self.hyperparam_pbar:
+                    self.hyperparam_pbar.reset()
+
+    def close_progress_bars(self) -> None:
+        """Close both progress bars."""
+        if self.hyperparam_pbar:
+            self.hyperparam_pbar.close()
+            self.hyperparam_pbar = None
+        if self.dataset_pbar:
+            self.dataset_pbar.close()
+            self.dataset_pbar = None
+
+
+@contextmanager
+def evaluation_progress(
+    datasets: List[Any], hyperparam_count: int, parallel: bool, total_eval_runs: int
+) -> Generator[EvaluationProgressBars, None, None]:
+    """Context manager for evaluation progress bars.
+
+    Args:
+        datasets: List of datasets being evaluated
+        hyperparam_count: Number of hyperparameter configurations per dataset
+        parallel: Whether running in parallel mode
+        total_eval_runs: Total number of EvalRunSpecs
+
+    Yields:
+        EvaluationProgressBars: Progress bar manager instance
+    """
+    progress_bars = EvaluationProgressBars(datasets, hyperparam_count, parallel, total_eval_runs)
+    progress_bars.start_progress_bars()
+    try:
+        yield progress_bars
+    finally:
+        progress_bars.close_progress_bars()

scorebook-0.0.2/src/scorebook/evaluator.py

@@ -1,271 +0,0 @@
-"""
-Model evaluation functionality for the Scorebook framework.
-
-This module provides the core evaluation logic to assess model predictions
-against ground truth labels using configurable metrics. It supports:
-
-- Batch evaluation of models across multiple datasets
-- Flexible metric computation and aggregation
-- Optional parameter sweeping and experiment tracking
-- Customizable inference functions
-
-The main entry point is the `evaluate()` function which handles running
-models on datasets and computing metric scores.
-"""
-
-import asyncio
-from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union
-
-from scorebook.types.eval_dataset import EvalDataset
-from scorebook.types.eval_result import EvalResult
-from scorebook.utils import evaluation_progress, expand_dict, is_awaitable
-
-
-async def _evaluate_async(
-    inference_callable: Callable,
-    eval_datasets: Union[str, EvalDataset, List[Union[str, EvalDataset]]],
-    hyperparameters: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
-    experiment_id: Optional[str] = None,
-    return_dict: bool = True,
-    return_aggregates: bool = True,
-    return_items: bool = False,
-    return_output: bool = False,
-    sample_size: Optional[int] = None,
-) -> Union[Dict, List]:
-    """Run inference across datasets/hyperparams, compute metrics, and format results."""
-
-    # Validate parameters
-    if return_dict and not return_aggregates and not return_items:
-        raise ValueError(
-            "When return_dict=True, at least one of return_aggregates or return_items must be True"
-        )
-
-    normalized_datasets = _normalize_datasets(eval_datasets)
-
-    if hyperparameters is None:
-        hyperparam_grid: List[Dict[str, Any]] = [{}]
-    elif not isinstance(hyperparameters, list):
-        hyperparam_grid = _expand_hyperparams(hyperparameters)
-    else:
-        hyperparam_grid = hyperparameters
-
-    eval_results: List[EvalResult] = []
-
-    with evaluation_progress(normalized_datasets, len(hyperparam_grid)) as progress_bars:
-        # Loop through datasets, then hyperparameters for clear progress tracking
-        for dataset_idx, eval_dataset in enumerate(normalized_datasets):
-            with progress_bars.hyperparam_progress_context():
-                # Run inference for each hyperparameter configuration on this dataset
-                for hp_idx, hyperparam_config in enumerate(hyperparam_grid):
-
-                    if sample_size:
-                        items = _get_items_sample(eval_dataset.items, sample_size)
-                    else:
-                        items = eval_dataset.items
-
-                    labels = _get_labels_for_items(items, eval_dataset.label)
-
-                    # 1) Run inference
-                    outputs = await _run_inference_callable(
-                        inference_callable, items, hyperparam_config
-                    )
-
-                    # 2) Score metrics
-                    metric_scores = _score_metrics(eval_dataset, outputs, labels)
-
-                    # 3) Wrap into EvalResult
-                    eval_results.append(
-                        EvalResult(eval_dataset, outputs, metric_scores, hyperparam_config)
-                    )
-
-                    # Update inner progress bar
-                    progress_bars.update_hyperparam_progress()
-
-            # Update the outer progress bar
-            progress_bars.update_dataset_progress()
-
-    # TODO: experiment_id handling (left as passthrough to preserve behavior)
-    if experiment_id:
-        pass
-
-    # 4) Format as requested
-    return _format_results(
-        eval_results, return_dict, return_aggregates, return_items, return_output
-    )
-
-
-def evaluate(
-    inference_callable: Callable,
-    eval_datasets: Union[str, EvalDataset, List[Union[str, EvalDataset]]],
-    hyperparameters: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
-    experiment_id: Optional[str] = None,
-    return_dict: bool = True,
-    return_aggregates: bool = True,
-    return_items: bool = False,
-    return_output: bool = False,
-    sample_size: Optional[int] = None,
-) -> Union[Dict, List]:
-    """
-    Evaluate model predictions using specified metrics on given datasets.
-
-    This function runs the provided inference callable on one or more evaluation datasets,
-    computes metric scores, and returns the evaluation results. It supports batch processing,
-    parameter sweeping, and different result formatting options.
-
-    Args:
-        inference_callable: A callable function or object that takes (items, hyperparameters)
-                           and returns predictions. Can be a regular function, async function,
-                           or callable instance (like a class with __call__ method).
-        eval_datasets: One or more evaluation datasets to run evaluation on. Can be:
-                 - A single EvalDataset instance
-                 - A list of EvalDataset instances
-                 - A string identifier (for future dataset registry support)
-                 - A list of string identifiers
-        hyperparameters: Optional dictionary containing hyperparameter sweep configuration.
-        experiment_id: Optional string identifier for tracking multiple evaluation runs.
-        return_dict: If True, returns eval results as a dict
-        return_aggregates: If True, returns aggregate scores for each dataset
-        return_items: If True, returns individual items for each dataset
-        return_output: If True, returns model outputs for each dataset item evaluated
-        sample_size: If set, only return a sample of the dataset items (for debugging)
-
-    Returns:
-        Dictionary mapping dataset names to their evaluation results. For each dataset,
-        returns a dictionary containing:
-        - items: List of EvalResult objects with predictions and ground truth
-        - metrics: Dictionary mapping metric names to their computed scores
-
-    Example:
-
-    python
-        dataset = EvalDataset.from_huggingface("dataset_name", label="answer", metrics=[Precision])
-        def inference_fn(items):
-            # Model inference logic here - process all items at once
-            return [prediction for item in items]
-
-        results = evaluate(inference_fn, dataset, item_limit=100)
-    """
-    return asyncio.run(
-        _evaluate_async(
-            inference_callable=inference_callable,
-            eval_datasets=eval_datasets,
-            hyperparameters=hyperparameters,
-            experiment_id=experiment_id,
-            return_dict=return_dict,
-            return_aggregates=return_aggregates,
-            return_items=return_items,
-            return_output=return_output,
-            sample_size=sample_size,
-        )
-    )
-
-
-# ===== Helper Functions =====
-
-
-def _normalize_datasets(
-    datasets: Union[str, EvalDataset, List[Union[str, EvalDataset]]]
-) -> List[EvalDataset]:
-    if not isinstance(datasets, list):
-        datasets = [datasets]
-    # TODO: handle other types (string registry, etc.)
-    return [d for d in datasets if isinstance(d, EvalDataset)]
-
-
-def _expand_hyperparams(hyperparameters: Optional[Dict[str, Any]]) -> Any:
-    return expand_dict(hyperparameters or {})
-
-
-def _get_items_sample(
-    items: List[Dict[str, Any]], item_limit: Optional[int]
-) -> List[Dict[str, Any]]:
-    return items[:item_limit] if item_limit else items
-
-
-def _get_labels_for_items(items: List[Dict[str, Any]], label_key: str) -> List[Any]:
-    return [item.get(label_key) for item in items]
-
-
-async def _run_inference_callable(
-    inference_callable: Callable,
-    items: List[Dict[str, Any]],
-    hyperparams: Dict[str, Any],
-) -> Any:
-    if is_awaitable(inference_callable):
-        return await inference_callable(items, **hyperparams)
-    else:
-        return inference_callable(items, **hyperparams)
-
-
-# Yields (eval_dataset, items, labels, hyperparams) for every dataset x hyperparam combo.
-def _iter_dataset_jobs(
-    datasets: List[EvalDataset],
-    hyperparam_grid: List[Dict[str, Any]],
-    sample_size: Optional[int],
-) -> Iterable[Tuple[EvalDataset, List[Dict[str, Any]], List[Any], Dict[str, Any]]]:
-    for eval_dataset in datasets:
-        for hp in hyperparam_grid:
-            items = _get_items_sample(eval_dataset.items, sample_size)
-            labels = _get_labels_for_items(items, eval_dataset.label)
-            yield eval_dataset, items, labels, hp
-
-
-def _score_metrics(
-    eval_dataset: EvalDataset, outputs: List[Any], labels: List[Any]
-) -> Dict[str, Dict[str, Any]]:
-    metric_scores: Dict[str, Dict[str, Any]] = {}
-    for metric in eval_dataset.metrics:
-        aggregate_scores, item_scores = metric.score(outputs, labels)
-        metric_scores[metric.name] = {
-            "aggregate_scores": aggregate_scores,
-            "item_scores": item_scores,
-        }
-    return metric_scores
-
-
-def _format_results(
-    eval_results: List[EvalResult],
-    return_dict: bool,
-    return_aggregates: bool,
-    return_items: bool,
-    return_output: bool,
-) -> Union[Dict, List]:
-
-    # Return results as a dict
-    if return_dict:
-
-        # Include both aggregate and item scores in dict returned
-        if return_aggregates and return_items:
-            results: Dict[str, List[Dict[str, Any]]] = {"aggregate_results": [], "item_results": []}
-            for eval_result in eval_results:
-                eval_result_dict = eval_result.to_dict()
-                results["aggregate_results"].extend(eval_result_dict["aggregate_results"])
-                if return_output:
-                    results["item_results"].extend(eval_result_dict["item_results"])
-                else:
-                    results["item_results"].extend(
-                        [
-                            {k: v for k, v in item.items() if k != "inference_output"}
-                            for item in eval_result_dict["item_results"]
-                        ]
-                    )
-            return results
-
-        # Include only aggregate scores in dict returned
-        elif return_aggregates:
-            return [eval_result.aggregate_scores for eval_result in eval_results]
-
-        # Include only item scores in dict returned
-        else:
-            if return_output:
-                return [item for eval_result in eval_results for item in eval_result.item_scores]
-            else:
-                return [
-                    {k: v for k, v in item.items() if k != "inference_output"}
-                    for eval_result in eval_results
-                    for item in eval_result.item_scores
-                ]
-
-    # Return results as an EvalResult object
-    else:
-        return {er.eval_dataset.name: er for er in eval_results}

scorebook-0.0.2/src/scorebook/utils/progress_bars.py

@@ -1,89 +0,0 @@
-"""Progress bar utilities for evaluation tracking."""
-
-from contextlib import contextmanager
-from typing import Any, Generator, List, Optional
-
-from tqdm import tqdm
-
-
-class EvaluationProgressBars:
-    """Manages nested progress bars for evaluation tracking."""
-
-    def __init__(self, datasets: List[Any], hyperparam_count: int) -> None:
-        """Initialize progress bar manager.
-
-        Args:
-            datasets: List of datasets being evaluated
-            hyperparam_count: Number of hyperparameter configurations per dataset
-        """
-        self.datasets = datasets
-        self.hyperparam_count = hyperparam_count
-        self.dataset_pbar: Optional[tqdm] = None
-        self.hyperparam_pbar: Optional[tqdm] = None
-
-    def start_dataset_progress(self) -> None:
-        """Start the outer progress bar for datasets."""
-        self.dataset_pbar = tqdm(
-            total=len(self.datasets),
-            desc="Datasets   ",
-            unit="dataset",
-            position=0,
-            leave=True,
-            ncols=80,
-            bar_format="{desc} {percentage:3.0f}%|{bar:40}| {n_fmt}/{total_fmt}",
-        )
-
-    def update_dataset_progress(self) -> None:
-        """Update the dataset progress bar."""
-        if self.dataset_pbar:
-            self.dataset_pbar.update(1)
-
-    def close_dataset_progress(self) -> None:
-        """Close the dataset progress bar."""
-        if self.dataset_pbar:
-            self.dataset_pbar.close()
-            self.dataset_pbar = None
-
-    @contextmanager
-    def hyperparam_progress_context(self) -> Generator[tqdm, None, None]:
-        """Context manager for hyperparameter progress bar."""
-        self.hyperparam_pbar = tqdm(
-            total=self.hyperparam_count,
-            desc="Hyperparams",
-            unit="config",
-            position=1,
-            leave=False,
-            ncols=80,
-            bar_format="{desc} {percentage:3.0f}%|{bar:40}| {n_fmt}/{total_fmt}",
-        )
-        try:
-            yield self.hyperparam_pbar
-        finally:
-            self.hyperparam_pbar.close()
-            self.hyperparam_pbar = None
-
-    def update_hyperparam_progress(self) -> None:
-        """Update the hyperparameter progress bar."""
-        if self.hyperparam_pbar:
-            self.hyperparam_pbar.update(1)
-
-
-@contextmanager
-def evaluation_progress(
-    datasets: List[Any], hyperparam_count: int
-) -> Generator[EvaluationProgressBars, None, None]:
-    """Context manager for evaluation progress bars.
-
-    Args:
-        datasets: List of datasets being evaluated
-        hyperparam_count: Number of hyperparameter configurations per dataset
-
-    Yields:
-        EvaluationProgressBars: Progress bar manager instance
-    """
-    progress_bars = EvaluationProgressBars(datasets, hyperparam_count)
-    progress_bars.start_dataset_progress()
-    try:
-        yield progress_bars
-    finally:
-        progress_bars.close_dataset_progress()