scorebook 0.0.2__py3-none-any.whl → 0.0.4__py3-none-any.whl

scorebook/evaluator.py

@@ -14,84 +14,19 @@ models on datasets and computing metric scores.
 """
 
 import asyncio
-from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Union
-
-from scorebook.types.eval_dataset import EvalDataset
-from scorebook.types.eval_result import EvalResult
+import logging
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
+
+from scorebook.exceptions import (
+    DataMismatchError,
+    MetricComputationError,
+    ParallelExecutionError,
+    ParameterValidationError,
+)
+from scorebook.types import EvalDataset, EvalResult, EvalRunSpec
 from scorebook.utils import evaluation_progress, expand_dict, is_awaitable
 
-
-async def _evaluate_async(
-    inference_callable: Callable,
-    eval_datasets: Union[str, EvalDataset, List[Union[str, EvalDataset]]],
-    hyperparameters: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
-    experiment_id: Optional[str] = None,
-    return_dict: bool = True,
-    return_aggregates: bool = True,
-    return_items: bool = False,
-    return_output: bool = False,
-    sample_size: Optional[int] = None,
-) -> Union[Dict, List]:
-    """Run inference across datasets/hyperparams, compute metrics, and format results."""
-
-    # Validate parameters
-    if return_dict and not return_aggregates and not return_items:
-        raise ValueError(
-            "When return_dict=True, at least one of return_aggregates or return_items must be True"
-        )
-
-    normalized_datasets = _normalize_datasets(eval_datasets)
-
-    if hyperparameters is None:
-        hyperparam_grid: List[Dict[str, Any]] = [{}]
-    elif not isinstance(hyperparameters, list):
-        hyperparam_grid = _expand_hyperparams(hyperparameters)
-    else:
-        hyperparam_grid = hyperparameters
-
-    eval_results: List[EvalResult] = []
-
-    with evaluation_progress(normalized_datasets, len(hyperparam_grid)) as progress_bars:
-        # Loop through datasets, then hyperparameters for clear progress tracking
-        for dataset_idx, eval_dataset in enumerate(normalized_datasets):
-            with progress_bars.hyperparam_progress_context():
-                # Run inference for each hyperparameter configuration on this dataset
-                for hp_idx, hyperparam_config in enumerate(hyperparam_grid):
-
-                    if sample_size:
-                        items = _get_items_sample(eval_dataset.items, sample_size)
-                    else:
-                        items = eval_dataset.items
-
-                    labels = _get_labels_for_items(items, eval_dataset.label)
-
-                    # 1) Run inference
-                    outputs = await _run_inference_callable(
-                        inference_callable, items, hyperparam_config
-                    )
-
-                    # 2) Score metrics
-                    metric_scores = _score_metrics(eval_dataset, outputs, labels)
-
-                    # 3) Wrap into EvalResult
-                    eval_results.append(
-                        EvalResult(eval_dataset, outputs, metric_scores, hyperparam_config)
-                    )
-
-                    # Update inner progress bar
-                    progress_bars.update_hyperparam_progress()
-
-            # Update the outer progress bar
-            progress_bars.update_dataset_progress()
-
-    # TODO: experiment_id handling (left as passthrough to preserve behavior)
-    if experiment_id:
-        pass
-
-    # 4) Format as requested
-    return _format_results(
-        eval_results, return_dict, return_aggregates, return_items, return_output
-    )
+logger = logging.getLogger(__name__)
 
 
 def evaluate(
@@ -99,6 +34,8 @@ def evaluate(
     eval_datasets: Union[str, EvalDataset, List[Union[str, EvalDataset]]],
     hyperparameters: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
     experiment_id: Optional[str] = None,
+    project_id: Optional[str] = None,
+    parallel: bool = False,
     return_dict: bool = True,
     return_aggregates: bool = True,
     return_items: bool = False,
@@ -128,6 +65,7 @@ def evaluate(
         return_items: If True, returns individual items for each dataset
         return_output: If True, returns model outputs for each dataset item evaluated
         sample_size: If set, only return a sample of the dataset items (for debugging)
+        parallel: If True, run inference functions in parallel (requires all functions to be async)
 
     Returns:
         Dictionary mapping dataset names to their evaluation results. For each dataset,
@@ -145,12 +83,22 @@ def evaluate(
 
         results = evaluate(inference_fn, dataset, item_limit=100)
     """
+
+    logger.info(
+        "Starting evaluation: experiment_id=%s, project_id=%s, parallel=%s",
+        experiment_id,
+        project_id,
+        parallel,
+    )
+
     return asyncio.run(
         _evaluate_async(
             inference_callable=inference_callable,
             eval_datasets=eval_datasets,
             hyperparameters=hyperparameters,
             experiment_id=experiment_id,
+            project_id=project_id,
+            parallel=parallel,
             return_dict=return_dict,
             return_aggregates=return_aggregates,
             return_items=return_items,
@@ -160,30 +108,169 @@ def evaluate(
     )
 
 
-# ===== Helper Functions =====
+async def _evaluate_async(
+    inference_callable: Callable,
+    eval_datasets: Union[str, EvalDataset, List[Union[str, EvalDataset]]],
+    hyperparameters: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
+    experiment_id: Optional[str] = None,
+    project_id: Optional[str] = None,
+    return_dict: bool = True,
+    return_aggregates: bool = True,
+    return_items: bool = False,
+    return_output: bool = False,
+    parallel: bool = False,
+    sample_size: Optional[int] = None,
+) -> Union[Dict, List]:
+    _validate_parameters(locals())
+    datasets, adaptive_datasets = _prepare_datasets(eval_datasets, sample_size)
+    hyperparameters = _prepare_hyperparameters(hyperparameters)
+
+    logger.info(
+        "Prepared %d datasets and %d hyperparameter configurations",
+        len(datasets),
+        len(hyperparameters),
+    )
 
+    runs = _build_runs(datasets, hyperparameters)
+    runs.sort(key=lambda run: (run.dataset_idx, run.hp_idx))
 
-def _normalize_datasets(
-    datasets: Union[str, EvalDataset, List[Union[str, EvalDataset]]]
-) -> List[EvalDataset]:
-    if not isinstance(datasets, list):
-        datasets = [datasets]
-    # TODO: handle other types (string registry, etc.)
-    return [d for d in datasets if isinstance(d, EvalDataset)]
+    logger.info("Created %d evaluation runs", len(runs))
+
+    with evaluation_progress(datasets, len(hyperparameters), parallel, len(runs)) as progress_bars:
+        if parallel:
+            eval_results = await _run_parallel(inference_callable, runs, progress_bars)
+        else:
+            eval_results = await _run_sequential(inference_callable, runs, progress_bars)
+
+        logger.info("Evaluation completed successfully")
+
+    return _format_results(
+        eval_results, return_dict, return_aggregates, return_items, return_output
+    )
+
+
+# ===== ORCHESTRATION PATHS =====
+
+
+async def _run_parallel(
+    inference_callable: Callable,
+    runs: List[EvalRunSpec],
+    progress_bars: Any,
+) -> List[EvalResult]:
+    logger.debug("Running inference in parallel")
+
+    async def worker(run: EvalRunSpec) -> Tuple[EvalRunSpec, EvalResult]:
+        er = await _execute_run(inference_callable, run)
+        progress_bars.on_eval_run_completed(run.dataset_idx)
+        return run, er
+
+    pairs = await asyncio.gather(*[worker(r) for r in runs])
+    # Return in canonical (dataset_idx, hp_idx) order for stability
+    pairs.sort(key=lambda p: (p[0].dataset_idx, p[0].hp_idx))
+    return [er for _, er in pairs]
+
+
+async def _run_sequential(
+    inference_callable: Callable,
+    runs: List[EvalRunSpec],
+    progress_bars: Any,
+) -> List[EvalResult]:
+    logger.debug("Running inference sequentially")
+    results: List[EvalResult] = []
+    for run in runs:
+        er = await _execute_run(inference_callable, run)
+        results.append(er)
+        progress_bars.on_hyperparam_completed(run.dataset_idx)
+    return results
 
 
-def _expand_hyperparams(hyperparameters: Optional[Dict[str, Any]]) -> Any:
-    return expand_dict(hyperparameters or {})
+# ===== EVALUATION EXECUTIONS =====
 
 
-def _get_items_sample(
-    items: List[Dict[str, Any]], item_limit: Optional[int]
+async def _execute_run(inference_callable: Callable, run: EvalRunSpec) -> EvalResult:
+    logger.debug("Executing run for %s", run)
+
+    outputs = await _run_inference_callable(inference_callable, run.items, run.hyperparams)
+    logger.debug("Inference completed for run %s", run)
+
+    metric_scores = _score_metrics(run.eval_dataset, outputs, run.labels)
+    logger.debug("Metrics computed for run %s. - scores:  %s", run, list(metric_scores.keys()))
+
+    return EvalResult(run.eval_dataset, outputs, metric_scores, run.hyperparams)
+
+
+# ===== HELPER FUNCTIONS =====
+
+
+def _validate_parameters(params: Dict[str, Any]) -> None:
+    """Validate all parameters for evaluation."""
+
+    if params["return_dict"] and not params["return_aggregates"] and not params["return_items"]:
+        raise ParameterValidationError(
+            "When return_dict=True, at least one of return_aggregates or return_items must be True"
+        )
+
+    if params["parallel"] and not is_awaitable(params["inference_callable"]):
+        raise ParallelExecutionError(
+            "parallel=True requires the inference_callable to be async. "
+            "Please make your inference function async or set parallel=False."
+        )
+
+
+def _prepare_datasets(
+    datasets: Union[str, EvalDataset, List[Union[str, EvalDataset]]],
+    sample_size: Optional[int] = None,
+) -> Tuple[List[EvalDataset], List[str]]:
+    """Prepare and separate input datasets into classic and adaptive evaluation datasets."""
+
+    # Ensure datasets is always a list for consistent processing
+    if not isinstance(datasets, list):
+        datasets = [datasets]
+
+    # Extract classical datasets TODO: handle other types (string registry)
+    classic_eval_datasets = [dataset for dataset in datasets if isinstance(dataset, EvalDataset)]
+
+    # Reduce datasets to a random sample
+    if sample_size:
+        logger.info("Sampling datasets to %d items each", sample_size)
+        for dataset in classic_eval_datasets:
+            dataset.shuffle()
+            if len(dataset) > sample_size:
+                original_size = len(dataset)
+                dataset._hf_dataset = dataset._hf_dataset.select(range(sample_size))
+                logger.debug(
+                    "Sampled dataset '%s' from %d to %d items",
+                    dataset.name,
+                    original_size,
+                    sample_size,
+                )
+
+    # Extract adaptive dataset strings
+    adaptive_eval_datasets = [
+        dataset.replace(":adaptive", "")
+        for dataset in datasets
+        if isinstance(dataset, str) and dataset.endswith(":adaptive")
+    ]
+
+    logger.info("Evaluating on classic datasets: %s", [ds.name for ds in classic_eval_datasets])
+    logger.info("Evaluating on adaptive datasets: %s", adaptive_eval_datasets)
+
+    return classic_eval_datasets, adaptive_eval_datasets
+
+
+def _prepare_hyperparameters(
+    hyperparameters: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]]
 ) -> List[Dict[str, Any]]:
-    return items[:item_limit] if item_limit else items
+    """Prepare hyperparameters for evaluation by returning a list of hyper-param configs."""
+    if hyperparameters is None:
+        return [{}]
+    if not isinstance(hyperparameters, list):
+        expanded: List[Dict[str, Any]] = expand_dict(hyperparameters or {})
+        return expanded
 
+    logger.info("Evaluating with hyperparameters: %s", hyperparameters)
 
-def _get_labels_for_items(items: List[Dict[str, Any]], label_key: str) -> List[Any]:
-    return [item.get(label_key) for item in items]
+    return hyperparameters
 
 
 async def _run_inference_callable(
@@ -197,29 +284,47 @@ async def _run_inference_callable(
         return inference_callable(items, **hyperparams)
 
 
-# Yields (eval_dataset, items, labels, hyperparams) for every dataset x hyperparam combo.
-def _iter_dataset_jobs(
+def _build_runs(
     datasets: List[EvalDataset],
-    hyperparam_grid: List[Dict[str, Any]],
-    sample_size: Optional[int],
-) -> Iterable[Tuple[EvalDataset, List[Dict[str, Any]], List[Any], Dict[str, Any]]]:
-    for eval_dataset in datasets:
-        for hp in hyperparam_grid:
-            items = _get_items_sample(eval_dataset.items, sample_size)
-            labels = _get_labels_for_items(items, eval_dataset.label)
-            yield eval_dataset, items, labels, hp
+    hyperparameters: List[Dict[str, Any]],
+) -> List[EvalRunSpec]:
+    """Build RunSpec objects for each dataset/hyperparameter combination."""
+    runs: List[EvalRunSpec] = []
+    for d_idx, ds in enumerate(datasets):
+        items = ds.items
+        labels = [item.get(ds.label) for item in items]
+        for hp_idx, hp in enumerate(hyperparameters):
+            run_spec = EvalRunSpec(d_idx, ds, items, labels, hp, hp_idx)
+            logger.debug("Built RunSpec: %s", run_spec)
+            runs.append(run_spec)
+    return runs
 
 
 def _score_metrics(
     eval_dataset: EvalDataset, outputs: List[Any], labels: List[Any]
 ) -> Dict[str, Dict[str, Any]]:
+    """Compute metric scores for a given dataset and inference outputs."""
     metric_scores: Dict[str, Dict[str, Any]] = {}
+
+    if len(outputs) != len(labels):
+        raise DataMismatchError(len(outputs), len(labels), eval_dataset.name)
+
     for metric in eval_dataset.metrics:
-        aggregate_scores, item_scores = metric.score(outputs, labels)
-        metric_scores[metric.name] = {
-            "aggregate_scores": aggregate_scores,
-            "item_scores": item_scores,
-        }
+        try:
+            aggregate_scores, item_scores = metric.score(outputs, labels)
+            metric_scores[metric.name] = {
+                "aggregate_scores": aggregate_scores,
+                "item_scores": item_scores,
+            }
+        except Exception as e:
+            logger.error(
+                "Failed to compute metric '%s' for dataset '%s': %s",
+                metric.name,
+                eval_dataset.name,
+                str(e),
+            )
+            raise MetricComputationError(metric.name, eval_dataset.name, e)
+
     return metric_scores
 
 
@@ -268,4 +373,7 @@ def _format_results(
 
     # Return results as an EvalResult object
     else:
-        return {er.eval_dataset.name: er for er in eval_results}
+        out: Dict[str, List[EvalResult]] = {}
+        for er in eval_results:
+            out.setdefault(er.eval_dataset.name, []).append(er)
+        return out

scorebook/exceptions.py

@@ -0,0 +1,54 @@
+"""
+Custom exceptions for the Scorebook framework.
+
+This module defines specific exception types used throughout the Scorebook
+evaluation framework to provide clear error handling and debugging information.
+"""
+
+
+class ScoreBookError(Exception):
+    """Base exception class for all Scorebook-related errors."""
+
+
+class EvaluationError(ScoreBookError):
+    """Raised when there are errors during model evaluation."""
+
+
+class ParameterValidationError(ScoreBookError):
+    """Raised when invalid parameters are provided to evaluation functions."""
+
+
+class InferenceError(EvaluationError):
+    """Raised when there are errors during model inference."""
+
+
+class MetricComputationError(EvaluationError):
+    """Raised when metric computation fails."""
+
+    def __init__(self, metric_name: str, dataset_name: str, original_error: Exception):
+        """Initialize metric computation error."""
+        self.metric_name = metric_name
+        self.dataset_name = dataset_name
+        self.original_error = original_error
+        super().__init__(
+            f"Failed to compute metric '{metric_name}' for dataset "
+            f"'{dataset_name}': {original_error}"
+        )
+
+
+class DataMismatchError(EvaluationError):
+    """Raised when there's a mismatch between outputs and expected labels."""
+
+    def __init__(self, outputs_count: int, labels_count: int, dataset_name: str):
+        """Initialize data mismatch error."""
+        self.outputs_count = outputs_count
+        self.labels_count = labels_count
+        self.dataset_name = dataset_name
+        super().__init__(
+            f"Output count ({outputs_count}) doesn't match label count ({labels_count}) "
+            f"for dataset '{dataset_name}'"
+        )
+
+
+class ParallelExecutionError(ScoreBookError):
+    """Raised when parallel execution requirements are not met."""

scorebook/inference/openai.py

@@ -8,17 +8,19 @@ API communication, request formatting, and response processing.
 
 import asyncio
 import json
+import logging
 import tempfile
 from typing import Any, List
 
-from openai import OpenAI
-from tqdm.asyncio import tqdm
+from openai import AsyncOpenAI
+
+logger = logging.getLogger(__name__)
 
 
 async def responses(
     items: List[Any], model: str = "gpt-4.1-nano", client: Any = None, **hyperparameters: Any
 ) -> List[Any]:
-    """Process multiple inference requests using OpenAI's API.
+    """Process multiple inference requests using OpenAI's Async API.
 
     This asynchronous function handles multiple inference requests,
     manages the API communication, and processes the responses.
@@ -35,13 +37,67 @@ async def responses(
     Raises:
         NotImplementedError: Currently not implemented.
     """
-    if client is None:
-        client = OpenAI()
+    logger.debug("OpenAI responses function called with %d items", len(items))
+    logger.debug("Using model: %s", model)
+    logger.debug("Hyperparameters: %s", hyperparameters)
 
-    results = []
-    for item in items:
-        response = client.responses.create(model=model, input=item)
-        results.append(response)
+    if client is None:
+        logger.debug("Creating new AsyncOpenAI client")
+        client = AsyncOpenAI()
+
+    # Create all tasks concurrently for true parallelism
+    tasks = []
+    for i, item in enumerate(items):
+        logger.debug(
+            "Processing item %d: %s",
+            i,
+            str(item)[:100] + "..." if len(str(item)) > 100 else str(item),
+        )
+
+        # Handle string input from preprocessor - convert to proper messages format
+        if isinstance(item, str):
+            # Convert the string format to proper OpenAI messages array
+            messages = [{"role": "user", "content": item}]
+            logger.debug(
+                "Converted string to messages format: %s",
+                (
+                    messages[0]["content"][:100] + "..."
+                    if len(messages[0]["content"]) > 100
+                    else messages[0]["content"]
+                ),
+            )
+        elif isinstance(item, list):
+            # Already in proper messages format
+            messages = item
+            logger.debug("Item %d already in messages format", i)
+        else:
+            # Fallback: treat as user message
+            messages = [{"role": "user", "content": str(item)}]
+            logger.debug("Item %d converted to fallback format", i)
+
+        logger.debug("Creating OpenAI task %d with messages: %s", i, messages)
+        task = client.chat.completions.create(model=model, messages=messages, **hyperparameters)
+        tasks.append(task)
+
+    logger.debug("Created %d tasks, waiting for OpenAI responses...", len(tasks))
+    # Wait for all requests to complete in parallel
+    results = await asyncio.gather(*tasks)
+    logger.debug("Received %d responses from OpenAI", len(results))
+
+    for i, result in enumerate(results):
+        logger.debug("Response %d type: %s", i, type(result))
+        try:
+            if hasattr(result, "choices") and result.choices:
+                content = result.choices[0].message.content
+                logger.debug(
+                    "Response %d content: %s",
+                    i,
+                    content[:100] + "..." if content and len(content) > 100 else content,
+                )
+            else:
+                logger.debug("Response %d has no choices or unexpected format", i)
+        except Exception as e:
+            logger.error("Error logging response %d: %s", i, e)
 
     return results
 
@@ -70,40 +126,23 @@ async def batch(
         NotImplementedError: Currently not implemented.
     """
     if client is None:
-        client = OpenAI()
+        client = AsyncOpenAI()
 
-    file_id = _upload_batch(items, client)
-    batch_id = _start_batch(file_id, client)
-
-    # Initialize progress bar
-    pbar = tqdm(total=len(items), desc="Batch processing", unit="requests")
+    file_id = await _upload_batch(items, client)
+    batch_id = await _start_batch(file_id, client)
 
     awaiting_batch = True
     while awaiting_batch:
         batch_object = await _get_batch(batch_id, client)
         batch_status = batch_object.status
 
-        if hasattr(batch_object, "request_counts") and batch_object.request_counts:
-            completed = batch_object.request_counts.completed
-            total = batch_object.request_counts.total
-            pbar.n = completed
-            pbar.set_postfix(status=batch_status, completed=f"{completed}/{total}")
-        else:
-            pbar.set_postfix(status=batch_status)
-
-        pbar.refresh()
-
         if batch_status == "completed":
             awaiting_batch = False
-            pbar.n = pbar.total
-            pbar.set_postfix(status="completed")
         elif batch_status == "failed":
             raise Exception("Batch processing failed")
         else:
             await asyncio.sleep(60)
 
-    pbar.close()
-
     # Get the final batch object to access output_file_id
     final_batch_object = await _get_batch(batch_id, client)
     output_file_id = final_batch_object.output_file_id
@@ -112,7 +151,7 @@ async def batch(
     return batch_result
 
 
-def _upload_batch(items: List[Any], client: Any) -> str:
+async def _upload_batch(items: List[Any], client: Any) -> str:
     """Create a .jsonl file from preprocessed items and upload to OpenAI for batch processing.
 
     Args:
@@ -121,10 +160,9 @@ def _upload_batch(items: List[Any], client: Any) -> str:
     Returns:
         The file ID returned by OpenAI after uploading.
     """
-    print("Uploading batch...")
     # Instantiate OpenAI client
     if client is None:
-        client = OpenAI()
+        client = AsyncOpenAI()
 
     # Create temp .jsonl file
     with tempfile.NamedTemporaryFile(mode="w+", suffix=".jsonl", delete=False) as f:
@@ -141,13 +179,13 @@ def _upload_batch(items: List[Any], client: Any) -> str:
 
     # Upload file to OpenAI
     with open(file_path, "rb") as upload_file:
-        response = client.files.create(file=upload_file, purpose="batch")
+        response = await client.files.create(file=upload_file, purpose="batch")
 
     return str(response.id)
 
 
-def _start_batch(file_id: str, client: Any) -> str:
-    batch_response = client.batches.create(
+async def _start_batch(file_id: str, client: Any) -> str:
+    batch_response = await client.batches.create(
         input_file_id=file_id,
         endpoint="/v1/chat/completions",
         completion_window="24h",
@@ -156,13 +194,13 @@ def _start_batch(file_id: str, client: Any) -> str:
 
 
 async def _get_batch(batch_id: str, client: Any) -> Any:
-    batch_object = client.batches.retrieve(batch_id)
+    batch_object = await client.batches.retrieve(batch_id)
     return batch_object
 
 
 async def _get_results_file(output_file_id: str, client: Any) -> List[str]:
     """Download and parse the batch results file from OpenAI."""
-    response = client.files.content(output_file_id)
+    response = await client.files.content(output_file_id)
 
     # Parse the JSONL content
     content = response.content.decode("utf-8")

scorebook/types/__init__.py

@@ -7,5 +7,6 @@ and evaluation results.
 
 from scorebook.types.eval_dataset import EvalDataset
 from scorebook.types.eval_result import EvalResult
+from scorebook.types.eval_run_spec import EvalRunSpec
 
-__all__ = ["EvalDataset", "EvalResult"]
+__all__ = ["EvalDataset", "EvalResult", "EvalRunSpec"]

scorebook/types/eval_dataset.py

@@ -86,6 +86,12 @@ class EvalDataset:
             raise ValueError("Dataset is not initialized")
         return iter(self._hf_dataset)
 
+    def shuffle(self) -> None:
+        """Randomly shuffle the dataset items."""
+        if self._hf_dataset is None:
+            raise ValueError("Dataset is not initialized")
+        self._hf_dataset.shuffle()
+
     @property
     def items(self) -> List[Any]:
         """Return a list of all examples in the dataset."""

scorebook/types/eval_run_spec.py

@@ -0,0 +1,28 @@
+"""Evaluation run specification types for Scorebook."""
+
+from typing import Any, Dict, List, NamedTuple
+
+from scorebook.types import EvalDataset
+
+
+class EvalRunSpec(NamedTuple):
+    """Represents a single evaluation run configuration."""
+
+    dataset_idx: int
+    eval_dataset: EvalDataset
+    items: List[Dict[str, Any]]
+    labels: List[Any]
+    hyperparams: Dict[str, Any]
+    hp_idx: int
+
+    def __str__(self) -> str:
+        """Return a formatted string summary of the evaluation run specification."""
+        hyperparams_str = ", ".join([f"{k}={v}" for k, v in self.hyperparams.items()])
+
+        return (
+            f"EvalRunSpec(dataset_idx={self.dataset_idx},"
+            f"  hp_idx={self.hp_idx},"
+            f"  dataset_name='{self.eval_dataset.name}',"
+            f"  hyperparams=[{hyperparams_str}]"
+            f")"
+        )

scorebook/types/inference_pipeline.py

@@ -57,7 +57,7 @@ class InferencePipeline:
             List of processed outputs after running through the complete pipeline
         """
         if self.preprocessor:
-            input_items = [self.preprocessor(item, hyperparameters) for item in items]
+            input_items = [self.preprocessor(item, **hyperparameters) for item in items]
         else:
             input_items = items
 
@@ -68,7 +68,7 @@ class InferencePipeline:
 
         if self.postprocessor:
             return [
-                self.postprocessor(inference_output, hyperparameters)
+                self.postprocessor(inference_output, **hyperparameters)
                 for inference_output in inference_outputs
             ]
         else:

scorebook/utils/logging_utils.py

@@ -0,0 +1 @@
+"""Logging utilities for Scorebook evaluation framework."""

scorebook/utils/progress_bars.py

@@ -9,20 +9,33 @@ from tqdm import tqdm
 class EvaluationProgressBars:
     """Manages nested progress bars for evaluation tracking."""
 
-    def __init__(self, datasets: List[Any], hyperparam_count: int) -> None:
+    def __init__(
+        self, datasets: List[Any], hyperparam_count: int, parallel: bool, total_eval_runs: int
+    ) -> None:
         """Initialize progress bar manager.
 
         Args:
             datasets: List of datasets being evaluated
             hyperparam_count: Number of hyperparameter configurations per dataset
+            parallel: Whether running in parallel mode
+            total_eval_runs: Total number of EvalRunSpecs (dataset_count * hyperparam_count)
         """
         self.datasets = datasets
         self.hyperparam_count = hyperparam_count
+        self.parallel = parallel
+        self.total_eval_runs = total_eval_runs
+
         self.dataset_pbar: Optional[tqdm] = None
         self.hyperparam_pbar: Optional[tqdm] = None
 
-    def start_dataset_progress(self) -> None:
-        """Start the outer progress bar for datasets."""
+        # Track progress per dataset
+        self.current_dataset_idx = 0
+        self.completed_hyperparams_per_dataset: dict[int, int] = {}
+        self.completed_eval_runs = 0
+
+    def start_progress_bars(self) -> None:
+        """Start both progress bars."""
+        # Top level: Datasets
         self.dataset_pbar = tqdm(
             total=len(self.datasets),
             desc="Datasets   ",
@@ -33,57 +46,101 @@ class EvaluationProgressBars:
             bar_format="{desc} {percentage:3.0f}%|{bar:40}| {n_fmt}/{total_fmt}",
         )
 
-    def update_dataset_progress(self) -> None:
-        """Update the dataset progress bar."""
-        if self.dataset_pbar:
-            self.dataset_pbar.update(1)
+        # Bottom level: Hyperparameters/Eval runs
+        if self.parallel:
+            # In parallel mode: show eval runs completed out of total
+            self.hyperparam_pbar = tqdm(
+                total=self.total_eval_runs,
+                desc="Eval Runs  ",
+                unit="run",
+                position=1,
+                leave=False,
+                ncols=80,
+                bar_format="{desc} {percentage:3.0f}%|{bar:40}| {n_fmt}/{total_fmt}",
+            )
+        else:
+            # In sequential mode: show hyperparams per dataset
+            self.hyperparam_pbar = tqdm(
+                total=self.hyperparam_count,
+                desc="Hyperparams",
+                unit="config",
+                position=1,
+                leave=False,
+                ncols=80,
+                bar_format="{desc} {percentage:3.0f}%|{bar:40}| {n_fmt}/{total_fmt}",
+            )
 
-    def close_dataset_progress(self) -> None:
-        """Close the dataset progress bar."""
-        if self.dataset_pbar:
-            self.dataset_pbar.close()
-            self.dataset_pbar = None
+    def on_eval_run_completed(self, dataset_idx: int) -> None:
+        """Update progress when an eval run (EvalRunSpec) completes in parallel mode."""
+        if not self.parallel:
+            return
 
-    @contextmanager
-    def hyperparam_progress_context(self) -> Generator[tqdm, None, None]:
-        """Context manager for hyperparameter progress bar."""
-        self.hyperparam_pbar = tqdm(
-            total=self.hyperparam_count,
-            desc="Hyperparams",
-            unit="config",
-            position=1,
-            leave=False,
-            ncols=80,
-            bar_format="{desc} {percentage:3.0f}%|{bar:40}| {n_fmt}/{total_fmt}",
+        self.completed_eval_runs += 1
+        if self.hyperparam_pbar:
+            self.hyperparam_pbar.update(1)
+
+        # Track how many runs completed for this dataset
+        self.completed_hyperparams_per_dataset[dataset_idx] = (
+            self.completed_hyperparams_per_dataset.get(dataset_idx, 0) + 1
         )
-        try:
-            yield self.hyperparam_pbar
-        finally:
-            self.hyperparam_pbar.close()
-            self.hyperparam_pbar = None
 
-    def update_hyperparam_progress(self) -> None:
-        """Update the hyperparameter progress bar."""
+        # Check if this dataset is complete
+        if self.completed_hyperparams_per_dataset[dataset_idx] == self.hyperparam_count:
+            if self.dataset_pbar:
+                self.dataset_pbar.update(1)
+
+    def on_hyperparam_completed(self, dataset_idx: int) -> None:
+        """Update progress when a hyperparameter config completes in sequential mode."""
+        if self.parallel:
+            return
+
         if self.hyperparam_pbar:
             self.hyperparam_pbar.update(1)
 
+        # Track completed hyperparams for this dataset
+        self.completed_hyperparams_per_dataset[dataset_idx] = (
+            self.completed_hyperparams_per_dataset.get(dataset_idx, 0) + 1
+        )
+
+        # Check if this dataset is complete
+        if self.completed_hyperparams_per_dataset[dataset_idx] == self.hyperparam_count:
+            # Update dataset progress
+            if self.dataset_pbar:
+                self.dataset_pbar.update(1)
+
+            # Reset hyperparameter progress for next dataset (if any)
+            if dataset_idx < len(self.datasets) - 1:
+                if self.hyperparam_pbar:
+                    self.hyperparam_pbar.reset()
+
+    def close_progress_bars(self) -> None:
+        """Close both progress bars."""
+        if self.hyperparam_pbar:
+            self.hyperparam_pbar.close()
+            self.hyperparam_pbar = None
+        if self.dataset_pbar:
+            self.dataset_pbar.close()
+            self.dataset_pbar = None
+
 
 @contextmanager
 def evaluation_progress(
-    datasets: List[Any], hyperparam_count: int
+    datasets: List[Any], hyperparam_count: int, parallel: bool, total_eval_runs: int
 ) -> Generator[EvaluationProgressBars, None, None]:
     """Context manager for evaluation progress bars.
 
     Args:
         datasets: List of datasets being evaluated
         hyperparam_count: Number of hyperparameter configurations per dataset
+        parallel: Whether running in parallel mode
+        total_eval_runs: Total number of EvalRunSpecs
 
     Yields:
         EvaluationProgressBars: Progress bar manager instance
     """
-    progress_bars = EvaluationProgressBars(datasets, hyperparam_count)
-    progress_bars.start_dataset_progress()
+    progress_bars = EvaluationProgressBars(datasets, hyperparam_count, parallel, total_eval_runs)
+    progress_bars.start_progress_bars()
     try:
         yield progress_bars
     finally:
-        progress_bars.close_dataset_progress()
+        progress_bars.close_progress_bars()

{scorebook-0.0.2.dist-info → scorebook-0.0.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: scorebook
-Version: 0.0.2
+Version: 0.0.4
 Summary: A Python project for LLM evaluation.
 Author: Euan Campbell
 Author-email: euan@trismik.com

{scorebook-0.0.2.dist-info → scorebook-0.0.4.dist-info}/RECORD

@@ -1,8 +1,9 @@
 scorebook/__init__.py,sha256=7ac3KpXU3kKFekq8mZ3cVbF7oQ6Q9E-uqX7ijyte1Q0,406
-scorebook/evaluator.py,sha256=oC7y3NOzC8CTQZ4FjGrN2ESmxPtWuYBBv4iThu0fqzQ,10624
+scorebook/evaluator.py,sha256=mS3G3PI26nHzqkYX4tqusQZJL5Q1xTxzqshAdwscl0s,14170
+scorebook/exceptions.py,sha256=emq2QY-4mW6VXlq1dxunPjt-xZpLQIxo8Ck_gYxz1VE,1827
 scorebook/inference/__init__.py,sha256=tqSXSyVurc_YRfPypYed8iTH7Fwt7iFCXMxBXnqY-9I,242
 scorebook/inference/bedrock.py,sha256=wllq0ysNFQKWJDEqoN-k96Jx43BHCAvfxm14zMRCf90,10074
-scorebook/inference/openai.py,sha256=XD1dbPrEHQJVXOMtqCt9a0yQ-qR381N5XXhCrgz8jio,5826
+scorebook/inference/openai.py,sha256=FqXua4v4PTYSHrdTm_9fM0Us8Mo2n2LSN94CwRipRw4,7658
 scorebook/inference/portkey.py,sha256=OHSS-sa2aLxuO6fEfG8MsPlhXc_95_-6j7ImbCkY8KE,5952
 scorebook/inference/vertex.py,sha256=jv_Nbt1NJQ6mMUyEuW_idxhj_3fugBojshtpGP9fMeY,9874
 scorebook/metrics/__init__.py,sha256=be_riJNojebXw2xfkMsHHjl3HFKgk9jQWlLkXJHhheI,782
@@ -10,19 +11,21 @@ scorebook/metrics/accuracy.py,sha256=5KQ4hfOn9M94sB7WsXUelJWJiuKfoCGQEl5q5q9vNfo
 scorebook/metrics/metric_base.py,sha256=I3L0DGcRojFp93UGFnXG1tZ2UK9ilTcXXJG6lj5ddXA,857
 scorebook/metrics/metric_registry.py,sha256=jWwt9P3zvtFLlEYrd60v7LS7X251nZczouE02zcCxWg,3402
 scorebook/metrics/precision.py,sha256=AaYPYYKnY74Nwqp_p3jd2Ewf3VHNOJjoRWf5fhb-tXk,563
-scorebook/types/__init__.py,sha256=xQMOae_fIBbeyeuqoa7SbNwjxAiVinPBbckOcUzo57U,358
-scorebook/types/eval_dataset.py,sha256=OvavGx6b0h-HiSJ67iUgr5IMmYVdYTPPodsh29Y6-ds,13657
+scorebook/types/__init__.py,sha256=dXY3Y-GiMipVExzVu7H5pbdFfg4HBMEKxqSTfENywSs,427
+scorebook/types/eval_dataset.py,sha256=dCqOHjGaEb7pGG1VF4aGFn6hngFvlxpxddqsDtM4nTs,13870
 scorebook/types/eval_result.py,sha256=R2zuWrx8p9_4A2W3Gmlp-xGgmelPdg8QB5PoV1hiqRc,4728
-scorebook/types/inference_pipeline.py,sha256=97hlaGkojfFcM8xmZb9jZIVHYwfBZOiZvqXw0dph1KA,3247
+scorebook/types/eval_run_spec.py,sha256=nf7LGa_dG60Qb385W6O6qiu7VlJ03-dpo2X1PgKGcRQ,845
+scorebook/types/inference_pipeline.py,sha256=-HcGGbwM34fGJ_FlXcyqj_pV6DjWIXRGgICiN_63UsU,3251
 scorebook/utils/__init__.py,sha256=l_bfi9lAMz1oyGnuyKuzYasQKt2DJwffqsbfSl4-GIQ,452
 scorebook/utils/async_utils.py,sha256=OeNvMrOT9P4rIyaCf5IbR3ZIFMtEzXgoAArNbINRtMU,728
 scorebook/utils/build_prompt.py,sha256=L_Y84a1ewm3GvwnSSuUXfPO_M0QL1Dl8UgOS_l_zvh4,1617
 scorebook/utils/io_helpers.py,sha256=ksOJ9ILcZqqt-HwRUYy1NMQbS6RuMh8i2ZzUADLMlQ8,913
 scorebook/utils/jinja_helpers.py,sha256=ksIKHiKdj8N0o7ZJZGasfbSNoAY6K5d9X_KM6mcKYD4,4208
+scorebook/utils/logging_utils.py,sha256=M4BXt369mJo037WYpvuWDoe3oGWVdHWaGo4Vbl0WDL0,60
 scorebook/utils/mappers.py,sha256=OcUnPBrnSUxZNhAzJhVmVWUWmqIKFXLTrK-xLi6_SUg,1259
-scorebook/utils/progress_bars.py,sha256=BlKqYlXDbik5eUn5nf5f7QnMvnTj8CU_CfXKxCWp3Ww,2909
+scorebook/utils/progress_bars.py,sha256=TBz41w3yFujsO9n8vUjeubgOrmdiAMI2P2SSVqTJzAA,5269
 scorebook/utils/transform_helpers.py,sha256=UnVLtFvcJrtmBEmLsuA4rrX4iJlNUKxm2DkIOGLl-2o,1030
-scorebook-0.0.2.dist-info/LICENSE,sha256=JLH1g9FhxHZf6CBCeQ_xAisPtICVObuNGW1bLPiTYEs,1068
-scorebook-0.0.2.dist-info/METADATA,sha256=tm4mid_Y-83CgejujK7NvzpFslOQkKjhZXDxBS5YNTY,11409
-scorebook-0.0.2.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-scorebook-0.0.2.dist-info/RECORD,,
+scorebook-0.0.4.dist-info/LICENSE,sha256=JLH1g9FhxHZf6CBCeQ_xAisPtICVObuNGW1bLPiTYEs,1068
+scorebook-0.0.4.dist-info/METADATA,sha256=7odU7Q8SHfuHru2oBBk1XlZ2tXLi2WaSShbUhfmX60A,11409
+scorebook-0.0.4.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+scorebook-0.0.4.dist-info/RECORD,,