scorebook 0.0.8__py3-none-any.whl → 0.0.10__py3-none-any.whl

scorebook/evaluate/evaluate_helpers.py

@@ -0,0 +1,365 @@
+"""Helper utilities shared by synchronous and asynchronous evaluation flows."""
+
+import asyncio
+import dataclasses
+import inspect
+import logging
+from typing import Any, Callable, Dict, Iterable, List, Literal, Mapping, Optional, Union
+
+from trismik._async.client import TrismikAsyncClient
+from trismik._sync.client import TrismikClient
+from trismik.types import TrismikMultipleChoiceTextItem
+
+from scorebook import EvalDataset
+from scorebook.exceptions import (
+    DataMismatchError,
+    MetricComputationError,
+    ParameterValidationError,
+    ScoreBookError,
+)
+from scorebook.settings import TRISMIK_SERVICE_URL
+from scorebook.trismik.credentials import get_token
+from scorebook.types import AdaptiveEvalDataset, AdaptiveEvalRunSpec, EvalResult, EvalRunSpec
+from scorebook.utils import expand_dict, is_awaitable
+
+logger = logging.getLogger(__name__)
+
+
+def resolve_upload_results(upload_results: Union[Literal["auto"], bool]) -> bool:
+    """Resolve the upload_results parameter based on trismik login status."""
+
+    if upload_results == "auto":
+        upload_results = get_token() is not None
+        logger.debug("Auto upload results resolved to: %s", upload_results)
+
+    return upload_results
+
+
+def validate_parameters(params: Dict[str, Any], caller: Callable[..., Any]) -> None:
+    """Validate all parameters for evaluation."""
+
+    caller_is_async = is_awaitable(caller)
+
+    # Sync evaluate() should only accept sync inference functions
+    if not caller_is_async and is_awaitable(params.get("inference")):
+        raise ParameterValidationError(
+            "evaluate() only accepts synchronous inference functions. "
+            "Use evaluate_async() for async inference functions."
+        )
+
+    # Async evaluate_async() should only accept async inference functions
+    if caller_is_async and not is_awaitable(params.get("inference")):
+        raise ParameterValidationError(
+            "evaluate_async() only accepts asynchronous inference functions. "
+            "Use evaluate() for sync inference functions."
+        )
+
+    # If returning a dict, it must contain items and/or aggregates
+    if params["return_dict"] and not params["return_aggregates"] and not params["return_items"]:
+        raise ParameterValidationError(
+            "When return_dict=True, at least one of return_aggregates or return_items must be True"
+        )
+
+    # If uploading results, experiment_id and project_id must be specified
+    if params["upload_results"]:
+        if params["experiment_id"] is None or params["project_id"] is None:
+            raise ParameterValidationError(
+                "experiment_id and project_id are required for upload_results=True"
+            )
+
+    logger.debug("Parameter validation successful")
+
+
+def prepare_datasets(
+    datasets: Union[str, EvalDataset, List[Union[str, EvalDataset]]],
+    sample_size: Optional[int] = None,
+) -> List[Union[EvalDataset, AdaptiveEvalDataset]]:
+    """Prepare and separate input datasets into classic and adaptive evaluation datasets."""
+
+    # Ensure datasets is always a list for consistent processing
+    if not isinstance(datasets, list):
+        datasets = [datasets]
+
+    datasets_out: List[Union[EvalDataset, AdaptiveEvalDataset]] = []
+    for dataset in datasets:
+
+        # Prepare classic datasets
+        if isinstance(dataset, EvalDataset):
+
+            if sample_size is not None:
+                dataset = dataset.sample(sample_size)
+
+            datasets_out.append(dataset)
+
+        # Prepare adaptive datasets
+        elif isinstance(dataset, str) and dataset.endswith(":adaptive"):
+            datasets_out.append(AdaptiveEvalDataset(dataset.replace(":adaptive", "")))
+
+        # TODO: dataset name string registry
+        elif isinstance(dataset, str):
+            pass
+
+        else:
+            raise ParameterValidationError(f"Unrecognized dataset type: {type(dataset)}")
+
+    return datasets_out
+
+
+def prepare_hyperparameter_configs(
+    hyperparameters: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]]
+) -> List[Dict[str, Any]]:
+    """Prepare hyperparameters for evaluation by returning a list of hyper-param configs."""
+    if hyperparameters is None:
+        return [{}]
+    if not isinstance(hyperparameters, list):
+        expanded: List[Dict[str, Any]] = expand_dict(hyperparameters or {})
+        return expanded
+
+    logger.info("Evaluating with hyperparameters: %s", hyperparameters)
+
+    return hyperparameters
+
+
+def build_eval_run_specs(
+    datasets: List[Union[EvalDataset, str]],
+    hyperparameters: Any,
+    experiment_id: Optional[str],
+    project_id: Optional[str],
+    metadata: Optional[Dict[str, Any]] = None,
+) -> List[Union[EvalRunSpec, AdaptiveEvalRunSpec]]:
+    """Build All RunSpec objects for each dataset/hyperparameter combination."""
+
+    eval_run_specs: List[Union[EvalRunSpec, AdaptiveEvalRunSpec]] = []
+    for dataset_index, dataset in enumerate(datasets):
+        for hyperparameters_index, hyperparameter_config in enumerate(hyperparameters):
+
+            # Create classic eval run spec
+            if isinstance(dataset, EvalDataset):
+                eval_run_specs.append(
+                    build_classic_eval_run_spec(
+                        dataset, dataset_index, hyperparameter_config, hyperparameters_index
+                    )
+                )
+
+            # Create adaptive eval run spec from string
+            elif isinstance(dataset, AdaptiveEvalDataset):
+                if not experiment_id or not project_id:
+                    raise ScoreBookError(
+                        "experiment_id and project_id are required for adaptive evaluations"
+                    )
+                eval_run_specs.append(
+                    build_adaptive_eval_run_spec(
+                        dataset.name,
+                        dataset_index,
+                        hyperparameter_config,
+                        hyperparameters_index,
+                        experiment_id,
+                        project_id,
+                        metadata,
+                    )
+                )
+
+            # Log warning - should never happen
+            else:
+                logger.warning("Unrecognized dataset type: %s", dataset)
+
+    return eval_run_specs
+
+
+def build_classic_eval_run_spec(
+    dataset: EvalDataset,
+    dataset_index: int,
+    hyperparameters: Dict[str, Any],
+    hyperparameters_index: int,
+) -> EvalRunSpec:
+    """Build EvalRunSpec objects for a classic dataset and hyperparameter combination."""
+    items = dataset.items
+    labels = [item.get(dataset.label) for item in items]
+    eval_run_spec = EvalRunSpec(
+        dataset,
+        dataset_index,
+        hyperparameters,
+        hyperparameters_index,
+        items,
+        labels,
+    )
+    logger.debug("Built EvalRunSpec: %s", eval_run_spec)
+    return eval_run_spec
+
+
+def build_adaptive_eval_run_spec(
+    adaptive_dataset: str,
+    dataset_index: int,
+    hyperparameter_config: Dict[str, Any],
+    hyperparameter_config_index: int,
+    experiment_id: str,
+    project_id: str,
+    metadata: Optional[Dict[str, Any]] = None,
+) -> AdaptiveEvalRunSpec:
+    """Build AdaptiveEvalRunSpec objects for a dataset/hyperparameter combination."""
+    dataset = adaptive_dataset.replace(":adaptive", "")
+    adaptive_eval_run_spec = AdaptiveEvalRunSpec(
+        dataset,
+        dataset_index,
+        hyperparameter_config,
+        hyperparameter_config_index,
+        experiment_id,
+        project_id,
+        metadata,
+    )
+    logger.debug("Built AdaptiveEvalRunSpec: %s", adaptive_eval_run_spec)
+    return adaptive_eval_run_spec
+
+
+def score_metrics(
+    eval_dataset: EvalDataset, outputs: List[Any], labels: List[Any]
+) -> Dict[str, Dict[str, Any]]:
+    """Compute metric scores for a given dataset and inference outputs."""
+    metric_scores: Dict[str, Dict[str, Any]] = {}
+
+    if len(outputs) != len(labels):
+        raise DataMismatchError(len(outputs), len(labels), eval_dataset.name)
+
+    for metric in eval_dataset.metrics:
+        try:
+            aggregate_scores, item_scores = metric.score(outputs, labels)
+            metric_scores[metric.name] = {
+                "aggregate_scores": aggregate_scores,
+                "item_scores": item_scores,
+            }
+        except Exception as e:
+            logger.error(
+                "Failed to compute metric '%s' for dataset '%s': %s",
+                metric.name,
+                eval_dataset.name,
+                str(e),
+            )
+            raise MetricComputationError(metric.name, eval_dataset.name, e)
+
+    return metric_scores
+
+
+def create_trismik_async_client() -> TrismikAsyncClient:
+    """Create a new async Trismik client instance."""
+    api_key = get_token()
+    logger.debug("Creating new async Trismik client")
+    return TrismikAsyncClient(service_url=TRISMIK_SERVICE_URL, api_key=api_key)
+
+
+def create_trismik_sync_client() -> TrismikClient:
+    """Create a new sync Trismik client instance."""
+    api_key = get_token()
+    logger.debug("Creating new sync Trismik client")
+    return TrismikClient(service_url=TRISMIK_SERVICE_URL, api_key=api_key)
+
+
+def get_model_name(
+    inference_callable: Optional[Callable] = None, metadata: Optional[Dict[str, Any]] = None
+) -> str:
+    """Determine a model's name with the fallback "unspecified"."""
+
+    # First priority: metadata.model
+    if metadata and "model" in metadata:
+        return str(metadata["model"])
+
+    # Second priority: inference_pipeline.model (if callable is an InferencePipeline)
+    if inference_callable and hasattr(inference_callable, "model"):
+        return str(inference_callable.model)
+
+    # Fallback: "unspecified"
+    return "unspecified"
+
+
+def format_results(
+    eval_result: EvalResult,
+    return_dict: bool,
+    return_aggregates: bool,
+    return_items: bool,
+    return_output: bool,
+) -> Union[EvalResult, Dict, List]:
+    """Format an `EvalResult` into the requested output structure."""
+
+    # Return results as a dict
+    if return_dict:
+        results = {}
+
+        if return_aggregates:
+            results["aggregate_results"] = eval_result.aggregate_scores
+
+        if return_items:
+            item_scores = eval_result.item_scores
+
+            # Remove inference output if not requested
+            if not return_output:
+                for item in item_scores:
+                    item.pop("inference_output", None)
+
+            results["item_results"] = item_scores
+
+        # If both are requested, return the combined structure
+        if return_aggregates and return_items:
+            return results
+        # If only aggregates requested, return just the list
+        elif return_aggregates:
+            return results["aggregate_results"]
+        # If only items requested, return just the list
+        else:
+            return results["item_results"]
+
+    # Return results as an EvalResult object
+    else:
+        return eval_result
+
+
+def make_trismik_inference(
+    inference_function: Callable[..., Any],
+    return_list: bool = False,
+) -> Callable[[Any], Any]:
+    """Wrap an inference function for flexible input handling.
+
+    Takes a function expecting list[dict] and makes it accept single dict
+    or TrismikMultipleChoiceTextItem.
+    """
+
+    # Check if the inference function is async
+    is_async = inspect.iscoroutinefunction(inference_function) or (
+        hasattr(inference_function, "__call__")
+        and inspect.iscoroutinefunction(inference_function.__call__)
+    )
+
+    def sync_trismik_inference_function(eval_items: Any, **kwargs: Any) -> Any:
+        # Single TrismikMultipleChoiceTextItem dataclass
+        if isinstance(eval_items, TrismikMultipleChoiceTextItem):
+            eval_item_dict = dataclasses.asdict(eval_items)
+            results = inference_function([eval_item_dict], **kwargs)
+            if is_async:
+                results = asyncio.run(results)
+            return results if return_list else results[0]
+
+        # Single item (a mapping)
+        if isinstance(eval_items, Mapping):
+            results = inference_function([eval_items], **kwargs)
+            if is_async:
+                results = asyncio.run(results)
+            return results if return_list else results[0]
+
+        # Iterable of items (but not a string/bytes)
+        if isinstance(eval_items, Iterable) and not isinstance(eval_items, (str, bytes)):
+            # Convert any TrismikMultipleChoiceTextItem instances to dicts
+            converted_items = []
+            for item in eval_items:
+                if isinstance(item, TrismikMultipleChoiceTextItem):
+                    converted_items.append(dataclasses.asdict(item))
+                else:
+                    converted_items.append(item)
+            results = inference_function(converted_items, **kwargs)
+            if is_async:
+                results = asyncio.run(results)
+            return results
+
+        raise TypeError(
+            "Expected a single item (Mapping[str, Any] or TrismikMultipleChoiceTextItem) "
+            "or an iterable of such items."
+        )
+
+    return sync_trismik_inference_function

scorebook/inference/__init__.py

@@ -5,3 +5,7 @@ This module provides functionality for running inference with various models
 and processing their responses. It includes utilities for both single and
 batch inference operations.
 """
+
+from scorebook.inference.inference_pipeline import InferencePipeline
+
+__all__ = ["InferencePipeline"]

scorebook/inference/clients/__init__.py

@@ -0,0 +1,8 @@
+"""
+Inference clients for various LLM providers.
+
+This module provides client implementations for different LLM providers including
+OpenAI, AWS Bedrock, Google Vertex AI, and Portkey.
+"""
+
+__all__ = ["bedrock", "openai", "portkey", "vertex"]

scorebook/inference/{openai.py → clients/openai.py}

@@ -10,7 +10,7 @@ import asyncio
 import json
 import logging
 import tempfile
-from typing import Any, List
+from typing import Any, List, Optional
 
 from openai import AsyncOpenAI
 
@@ -18,7 +18,10 @@ logger = logging.getLogger(__name__)
 
 
 async def responses(
-    items: List[Any], model: str = "gpt-4.1-nano", client: Any = None, **hyperparameters: Any
+    items: List[Any],
+    model: str = "gpt-4.1-nano",
+    client: Optional[AsyncOpenAI] = None,
+    **hyperparameters: Any,
 ) -> List[Any]:
     """Process multiple inference requests using OpenAI's Async API.
 
@@ -28,23 +31,28 @@ async def responses(
     Args:
         items: List of preprocessed items to process.
         model: OpenAI model to use.
-        client: Optional OpenAI client instance.
+        client: Optional OpenAI client instance. If not provided, creates a new client
+            with automatic cleanup using a context manager.
         hyperparameters: Dictionary of hyperparameters for inference.
 
     Returns:
         List of raw model responses.
-
-    Raises:
-        NotImplementedError: Currently not implemented.
     """
+    if client is None:
+        async with AsyncOpenAI() as client:
+            return await _do_responses(items, model, client, **hyperparameters)
+    else:
+        return await _do_responses(items, model, client, **hyperparameters)
+
+
+async def _do_responses(
+    items: List[Any], model: str, client: AsyncOpenAI, **hyperparameters: Any
+) -> List[Any]:
+    """Process responses internally with provided client."""
     logger.debug("OpenAI responses function called with %d items", len(items))
     logger.debug("Using model: %s", model)
     logger.debug("Hyperparameters: %s", hyperparameters)
 
-    if client is None:
-        logger.debug("Creating new AsyncOpenAI client")
-        client = AsyncOpenAI()
-
     # Create all tasks concurrently for true parallelism
     tasks = []
     for i, item in enumerate(items):
@@ -127,7 +135,7 @@ async def responses(
 async def batch(
     items: List[Any],
     model: str = "gpt-4.1-nano",
-    client: Any = None,
+    client: Optional[AsyncOpenAI] = None,
     **hyperparameters: Any,
 ) -> List[Any]:
     """Process multiple inference requests in batch using OpenAI's API.
@@ -138,18 +146,24 @@ async def batch(
     Args:
         items: List of preprocessed items to process.
         model: OpenAI model to use.
-        client: Optional OpenAI client instance.
+        client: Optional OpenAI client instance. If not provided, creates a new client
+            with automatic cleanup using a context manager.
         hyperparameters: Dictionary of hyperparameters for inference.
 
     Returns:
         A list of raw model responses.
-
-    Raises:
-        NotImplementedError: Currently not implemented.
     """
     if client is None:
-        client = AsyncOpenAI()
+        async with AsyncOpenAI() as client:
+            return await _do_batch(items, model, client, **hyperparameters)
+    else:
+        return await _do_batch(items, model, client, **hyperparameters)
 
+
+async def _do_batch(
+    items: List[Any], model: str, client: AsyncOpenAI, **hyperparameters: Any
+) -> List[Any]:
+    """Process batch internally with provided client."""
     file_id = await _upload_batch(items, client)
     batch_id = await _start_batch(file_id, client)
 
@@ -173,18 +187,16 @@ async def batch(
     return batch_result
 
 
-async def _upload_batch(items: List[Any], client: Any) -> str:
+async def _upload_batch(items: List[Any], client: AsyncOpenAI) -> str:
     """Create a .jsonl file from preprocessed items and upload to OpenAI for batch processing.
 
     Args:
         items: A list of preprocessed items, each representing a single dataset eval item.
+        client: OpenAI client instance.
 
     Returns:
         The file ID returned by OpenAI after uploading.
     """
-    # Instantiate OpenAI client
-    if client is None:
-        client = AsyncOpenAI()
 
     # Create temp .jsonl file
     with tempfile.NamedTemporaryFile(mode="w+", suffix=".jsonl", delete=False) as f:
@@ -206,7 +218,7 @@ async def _upload_batch(items: List[Any], client: Any) -> str:
     return str(response.id)
 
 
-async def _start_batch(file_id: str, client: Any) -> str:
+async def _start_batch(file_id: str, client: AsyncOpenAI) -> str:
     batch_response = await client.batches.create(
         input_file_id=file_id,
         endpoint="/v1/chat/completions",
@@ -215,12 +227,12 @@ async def _start_batch(file_id: str, client: Any) -> str:
     return str(batch_response.id)
 
 
-async def _get_batch(batch_id: str, client: Any) -> Any:
+async def _get_batch(batch_id: str, client: AsyncOpenAI) -> Any:
     batch_object = await client.batches.retrieve(batch_id)
     return batch_object
 
 
-async def _get_results_file(output_file_id: str, client: Any) -> List[str]:
+async def _get_results_file(output_file_id: str, client: AsyncOpenAI) -> List[str]:
     """Download and parse the batch results file from OpenAI."""
     response = await client.files.content(output_file_id)
 

scorebook/{inference_pipeline.py → inference/inference_pipeline.py}

@@ -9,6 +9,8 @@ configurable way.
 import asyncio
 from typing import Any, Callable, Dict, List, Optional, cast
 
+from scorebook.utils import is_awaitable
+
 
 class InferencePipeline:
     """A pipeline for processing items through model inference.
@@ -18,6 +20,8 @@ class InferencePipeline:
     2. Model inference
     3. Postprocessing of model outputs
 
+    The pipeline automatically adapts to sync or async execution based on the
+    inference function provided during initialization.
 
     Attributes:
         model: Name or identifier of the model being used
@@ -35,6 +39,9 @@ class InferencePipeline:
     ) -> None:
         """Initialize the inference pipeline.
 
+        The pipeline will automatically become sync or async based on the
+        inference_function provided.
+
         Args:
             model: Name or identifier of the model to use
             inference_function: Function that performs model inference
@@ -46,8 +53,59 @@ class InferencePipeline:
         self.preprocessor: Optional[Callable] = preprocessor
         self.postprocessor: Optional[Callable] = postprocessor
 
+        # Dynamically change the class to provide appropriate sync/async interface
+        self.__class__ = (
+            _AsyncInferencePipeline if is_awaitable(inference_function) else _SyncInferencePipeline
+        )
+
+
+class _SyncInferencePipeline(InferencePipeline):
+    """Synchronous version of InferencePipeline."""
+
+    def run(self, items: List[Dict[str, Any]], **hyperparameters: Any) -> List[Any]:
+        """Execute the complete inference pipeline synchronously.
+
+        Args:
+            items: List of items to process through the pipeline
+            **hyperparameters: Model-specific parameters for inference
+
+        Returns:
+            List of processed outputs after running through the complete pipeline
+        """
+        if self.preprocessor:
+            input_items = [self.preprocessor(item, **hyperparameters) for item in items]
+        else:
+            input_items = items
+
+        # Sync inference function - call directly
+        inference_outputs = self.inference_function(input_items, **hyperparameters)
+
+        if self.postprocessor:
+            return [
+                self.postprocessor(inference_output, **hyperparameters)
+                for inference_output in inference_outputs
+            ]
+        else:
+            return cast(List[Any], inference_outputs)
+
+    def __call__(self, items: List[Dict[str, Any]], **hyperparameters: Any) -> List[Any]:
+        """Make the pipeline instance callable synchronously.
+
+        Args:
+            items: List of items to process through the pipeline
+            **hyperparameters: Model-specific parameters for inference
+
+        Returns:
+            List of processed outputs after running through the complete pipeline
+        """
+        return self.run(items, **hyperparameters)
+
+
+class _AsyncInferencePipeline(InferencePipeline):
+    """Asynchronous version of InferencePipeline."""
+
     async def run(self, items: List[Dict[str, Any]], **hyperparameters: Any) -> List[Any]:
-        """Execute the complete inference pipeline on a list of items.
+        """Execute the complete inference pipeline asynchronously.
 
         Args:
             items: List of items to process through the pipeline
@@ -61,10 +119,14 @@ class InferencePipeline:
         else:
             input_items = items
 
-        if asyncio.iscoroutinefunction(self.inference_function):
+        # Handle both sync and async inference functions
+        if is_awaitable(self.inference_function):
             inference_outputs = await self.inference_function(input_items, **hyperparameters)
         else:
-            inference_outputs = self.inference_function(input_items, **hyperparameters)
+            # Run sync function in thread pool to avoid blocking
+            inference_outputs = await asyncio.to_thread(
+                self.inference_function, input_items, **hyperparameters
+            )
 
         if self.postprocessor:
             return [
@@ -75,7 +137,7 @@ class InferencePipeline:
             return cast(List[Any], inference_outputs)
 
     async def __call__(self, items: List[Dict[str, Any]], **hyperparameters: Any) -> List[Any]:
-        """Make the pipeline instance callable by wrapping the run method.
+        """Make the pipeline instance callable asynchronously.
 
         Args:
             items: List of items to process through the pipeline

scorebook/settings.py

@@ -0,0 +1,18 @@
+"""Configuration settings for Scorebook."""
+
+import os
+
+# Optional: Load environment variables from .env file if python-dotenv is available
+try:
+    from dotenv import load_dotenv
+
+    load_dotenv(verbose=False)
+except ImportError:  # pragma: no cover
+    pass  # python-dotenv not installed, skip .env file loading
+
+# Trismik API settings
+TRISMIK_API_BASE_URL = "https://api.trismik.com"
+TRISMIK_ADAPTIVE_TESTING_URL = f"{TRISMIK_API_BASE_URL}/adaptive-testing"
+
+# Allow override via environment variable
+TRISMIK_SERVICE_URL = os.environ.get("TRISMIK_SERVICE_URL", TRISMIK_ADAPTIVE_TESTING_URL)

scorebook/trismik/__init__.py

@@ -0,0 +1,10 @@
+"""Trismik authentication and API integration.
+
+Note: Trismik evaluation functionality has been moved to scorebook.evaluate module.
+This module now only provides authentication functions.
+"""
+
+# Import shared credential functions
+from .credentials import get_stored_token, get_token, login, logout, whoami
+
+__all__ = ["login", "logout", "whoami", "get_stored_token", "get_token"]

scorebook/utils/__init__.py

@@ -1,9 +1,16 @@
 """Utility functions and common helpers for the Scorebook framework."""
 
-from scorebook.utils.async_utils import is_awaitable
+from scorebook.utils.async_utils import async_nullcontext, is_awaitable
 from scorebook.utils.build_prompt import build_prompt
 from scorebook.utils.io_helpers import validate_path
 from scorebook.utils.progress_bars import evaluation_progress
 from scorebook.utils.transform_helpers import expand_dict
 
-__all__ = ["is_awaitable", "validate_path", "expand_dict", "evaluation_progress", "build_prompt"]
+__all__ = [
+    "async_nullcontext",
+    "is_awaitable",
+    "validate_path",
+    "expand_dict",
+    "evaluation_progress",
+    "build_prompt",
+]