scorebook 0.0.1__tar.gz → 0.0.2__tar.gz

{scorebook-0.0.1 → scorebook-0.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: scorebook
-Version: 0.0.1
+Version: 0.0.2
 Summary: A Python project for LLM evaluation.
 Author: Euan Campbell
 Author-email: euan@trismik.com
@@ -11,16 +11,26 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Provides-Extra: bedrock
 Provides-Extra: examples
 Provides-Extra: openai
 Provides-Extra: portkey
+Provides-Extra: vertex
 Requires-Dist: accelerate ; extra == "examples"
+Requires-Dist: boto3 (==1.40.0) ; extra == "bedrock"
 Requires-Dist: datasets (>=3.6.0)
+Requires-Dist: fsspec[gcs] ; extra == "vertex"
+Requires-Dist: google-cloud-storage ; extra == "vertex"
+Requires-Dist: google-genai ; extra == "vertex"
+Requires-Dist: notebook (>=7.4.5,<8.0.0)
 Requires-Dist: notebook ; extra == "examples"
 Requires-Dist: openai ; extra == "openai"
+Requires-Dist: pandas ; extra == "vertex"
 Requires-Dist: portkey-ai ; extra == "portkey"
+Requires-Dist: python-dotenv ; extra == "bedrock"
 Requires-Dist: python-dotenv ; extra == "openai"
 Requires-Dist: python-dotenv ; extra == "portkey"
+Requires-Dist: python-dotenv ; extra == "vertex"
 Requires-Dist: torch ; extra == "examples"
 Requires-Dist: torchaudio ; extra == "examples"
 Requires-Dist: torchvision ; extra == "examples"

{scorebook-0.0.1 → scorebook-0.0.2}/pyproject.toml

@@ -10,10 +10,11 @@ readme = "README.md"
 requires-python = ">=3.9"
 dependencies = [
     "datasets>=3.6.0",
+    "notebook (>=7.4.5,<8.0.0)",
 ]
 
 [tool.poetry]
-version = "0.0.1"  # base version
+version = "0.0.2"  # base version
 packages = [{ include = "scorebook", from = "src" }]
 
 
@@ -28,10 +29,13 @@ flake8 = "^7.0.0"
 mypy = "^1.15.0"
 autoflake = "^2.3.1"
 toml = "^0.10.2"
+types-pyyaml = "^6.0.12.20250822"
 
 [project.optional-dependencies]
 openai = ["openai", "python-dotenv"]
 portkey = ["portkey-ai", "python-dotenv"]
+bedrock = ["boto3==1.40.0", "python-dotenv"]
+vertex = ["google-genai", "pandas", "google-cloud-storage", "fsspec[gcs]", "python-dotenv"]
 examples = ["transformers", "torch", "torchvision", "torchaudio", "accelerate", "notebook"]
 
 

{scorebook-0.0.1 → scorebook-0.0.2}/src/scorebook/__init__.py

@@ -11,5 +11,6 @@ __version__ = importlib.metadata.version(__package__ or __name__)
 
 from scorebook.evaluator import evaluate
 from scorebook.types.eval_dataset import EvalDataset
+from scorebook.utils.build_prompt import build_prompt
 
-__all__ = ["EvalDataset", "evaluate"]
+__all__ = ["EvalDataset", "evaluate", "build_prompt"]

{scorebook-0.0.1 → scorebook-0.0.2}/src/scorebook/evaluator.py

@@ -24,17 +24,30 @@ from scorebook.utils import evaluation_progress, expand_dict, is_awaitable
 async def _evaluate_async(
     inference_callable: Callable,
     eval_datasets: Union[str, EvalDataset, List[Union[str, EvalDataset]]],
-    hyperparameters: Optional[Dict[str, Any]] = None,
+    hyperparameters: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
     experiment_id: Optional[str] = None,
-    item_limit: Optional[int] = None,
-    return_type: str = "dict",
-    score_type: str = "aggregate",
+    return_dict: bool = True,
+    return_aggregates: bool = True,
+    return_items: bool = False,
+    return_output: bool = False,
+    sample_size: Optional[int] = None,
 ) -> Union[Dict, List]:
     """Run inference across datasets/hyperparams, compute metrics, and format results."""
-    _validate_score_type(score_type)
+
+    # Validate parameters
+    if return_dict and not return_aggregates and not return_items:
+        raise ValueError(
+            "When return_dict=True, at least one of return_aggregates or return_items must be True"
+        )
 
     normalized_datasets = _normalize_datasets(eval_datasets)
-    hyperparam_grid = _expand_hyperparams(hyperparameters)
+
+    if hyperparameters is None:
+        hyperparam_grid: List[Dict[str, Any]] = [{}]
+    elif not isinstance(hyperparameters, list):
+        hyperparam_grid = _expand_hyperparams(hyperparameters)
+    else:
+        hyperparam_grid = hyperparameters
 
     eval_results: List[EvalResult] = []
 
@@ -44,8 +57,13 @@ async def _evaluate_async(
             with progress_bars.hyperparam_progress_context():
                 # Run inference for each hyperparameter configuration on this dataset
                 for hp_idx, hyperparam_config in enumerate(hyperparam_grid):
-                    items = _clip_items(eval_dataset.items, item_limit)
-                    labels = _labels_for(items, eval_dataset.label)
+
+                    if sample_size:
+                        items = _get_items_sample(eval_dataset.items, sample_size)
+                    else:
+                        items = eval_dataset.items
+
+                    labels = _get_labels_for_items(items, eval_dataset.label)
 
                     # 1) Run inference
                     outputs = await _run_inference_callable(
@@ -71,17 +89,21 @@ async def _evaluate_async(
         pass
 
     # 4) Format as requested
-    return _format_results(eval_results, return_type, score_type)
+    return _format_results(
+        eval_results, return_dict, return_aggregates, return_items, return_output
+    )
 
 
 def evaluate(
     inference_callable: Callable,
     eval_datasets: Union[str, EvalDataset, List[Union[str, EvalDataset]]],
-    hyperparameters: Optional[Dict[str, Any]] = None,
+    hyperparameters: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
     experiment_id: Optional[str] = None,
-    item_limit: Optional[int] = None,
-    return_type: str = "dict",
-    score_type: str = "aggregate",
+    return_dict: bool = True,
+    return_aggregates: bool = True,
+    return_items: bool = False,
+    return_output: bool = False,
+    sample_size: Optional[int] = None,
 ) -> Union[Dict, List]:
     """
     Evaluate model predictions using specified metrics on given datasets.
@@ -101,12 +123,11 @@ def evaluate(
                  - A list of string identifiers
         hyperparameters: Optional dictionary containing hyperparameter sweep configuration.
         experiment_id: Optional string identifier for tracking multiple evaluation runs.
-        item_limit: Optional integer limiting the number of items to evaluate per dataset.
-        return_type: Format of the return value. Currently only "dict" is supported.
-        score_type: Type of score aggregation to return. Options:
-                   - "aggregate": Return aggregated metrics
-                   - "item": Return per-item scores
-                   - "all": Return both aggregate and per-item scores
+        return_dict: If True, returns eval results as a dict
+        return_aggregates: If True, returns aggregate scores for each dataset
+        return_items: If True, returns individual items for each dataset
+        return_output: If True, returns model outputs for each dataset item evaluated
+        sample_size: If set, only return a sample of the dataset items (for debugging)
 
     Returns:
         Dictionary mapping dataset names to their evaluation results. For each dataset,
@@ -130,9 +151,11 @@ def evaluate(
             eval_datasets=eval_datasets,
             hyperparameters=hyperparameters,
             experiment_id=experiment_id,
-            item_limit=item_limit,
-            return_type=return_type,
-            score_type=score_type,
+            return_dict=return_dict,
+            return_aggregates=return_aggregates,
+            return_items=return_items,
+            return_output=return_output,
+            sample_size=sample_size,
         )
     )
 
@@ -149,20 +172,17 @@ def _normalize_datasets(
     return [d for d in datasets if isinstance(d, EvalDataset)]
 
 
-def _validate_score_type(score_type: str) -> None:
-    if score_type not in {"aggregate", "item", "all"}:
-        raise ValueError("score_type must be 'aggregate', 'item', or 'all'")
-
-
 def _expand_hyperparams(hyperparameters: Optional[Dict[str, Any]]) -> Any:
     return expand_dict(hyperparameters or {})
 
 
-def _clip_items(items: List[Dict[str, Any]], item_limit: Optional[int]) -> List[Dict[str, Any]]:
+def _get_items_sample(
+    items: List[Dict[str, Any]], item_limit: Optional[int]
+) -> List[Dict[str, Any]]:
     return items[:item_limit] if item_limit else items
 
 
-def _labels_for(items: List[Dict[str, Any]], label_key: str) -> List[Any]:
+def _get_labels_for_items(items: List[Dict[str, Any]], label_key: str) -> List[Any]:
     return [item.get(label_key) for item in items]
 
 
@@ -181,12 +201,12 @@ async def _run_inference_callable(
 def _iter_dataset_jobs(
     datasets: List[EvalDataset],
     hyperparam_grid: List[Dict[str, Any]],
-    item_limit: Optional[int],
+    sample_size: Optional[int],
 ) -> Iterable[Tuple[EvalDataset, List[Dict[str, Any]], List[Any], Dict[str, Any]]]:
     for eval_dataset in datasets:
         for hp in hyperparam_grid:
-            items = _clip_items(eval_dataset.items, item_limit)
-            labels = _labels_for(items, eval_dataset.label)
+            items = _get_items_sample(eval_dataset.items, sample_size)
+            labels = _get_labels_for_items(items, eval_dataset.label)
             yield eval_dataset, items, labels, hp
 
 
@@ -204,25 +224,48 @@ def _score_metrics(
 
 
 def _format_results(
-    eval_results: List[EvalResult], return_type: str, score_type: str
+    eval_results: List[EvalResult],
+    return_dict: bool,
+    return_aggregates: bool,
+    return_items: bool,
+    return_output: bool,
 ) -> Union[Dict, List]:
 
-    if return_type != "dict":
+    # Return results as a dict
+    if return_dict:
+
+        # Include both aggregate and item scores in dict returned
+        if return_aggregates and return_items:
+            results: Dict[str, List[Dict[str, Any]]] = {"aggregate_results": [], "item_results": []}
+            for eval_result in eval_results:
+                eval_result_dict = eval_result.to_dict()
+                results["aggregate_results"].extend(eval_result_dict["aggregate_results"])
+                if return_output:
+                    results["item_results"].extend(eval_result_dict["item_results"])
+                else:
+                    results["item_results"].extend(
+                        [
+                            {k: v for k, v in item.items() if k != "inference_output"}
+                            for item in eval_result_dict["item_results"]
+                        ]
+                    )
+            return results
+
+        # Include only aggregate scores in dict returned
+        elif return_aggregates:
+            return [eval_result.aggregate_scores for eval_result in eval_results]
+
+        # Include only item scores in dict returned
+        else:
+            if return_output:
+                return [item for eval_result in eval_results for item in eval_result.item_scores]
+            else:
+                return [
+                    {k: v for k, v in item.items() if k != "inference_output"}
+                    for eval_result in eval_results
+                    for item in eval_result.item_scores
+                ]
+
+    # Return results as an EvalResult object
+    else:
         return {er.eval_dataset.name: er for er in eval_results}
-
-    if score_type == "all":
-        combined: Dict[str, List[Dict[str, Any]]] = {"aggregate": [], "per_sample": []}
-        for er in eval_results:
-            d = er.to_dict()
-            combined["aggregate"].extend(d["aggregate"])
-            combined["per_sample"].extend(d["per_sample"])
-        return combined
-
-    if score_type == "aggregate":
-        return [er.aggregate_scores for er in eval_results]
-
-    if score_type == "item":
-        return [item for er in eval_results for item in er.item_scores]
-
-    # Should be unreachable due to validation
-    return {}

{scorebook-0.0.1 → scorebook-0.0.2}/src/scorebook/inference/__init__.py

@@ -5,7 +5,3 @@ This module provides functionality for running inference with various models
 and processing their responses. It includes utilities for both single and
 batch inference operations.
 """
-
-from scorebook.inference.openai import batch, responses
-
-__all__ = ["responses", "batch"]

scorebook-0.0.2/src/scorebook/inference/bedrock.py

@@ -0,0 +1,305 @@
+"""
+AWS Bedrock batch inference implementation for Scorebook.
+
+This module provides utilities for running batch inference using AWS Bedrock's
+Model Invocation Jobs, supporting large-scale asynchronous processing. It handles
+API communication, request formatting, response processing, and S3 operations.
+"""
+
+import asyncio
+import json
+import os
+import tempfile
+import uuid
+from datetime import datetime
+from typing import Any, Dict, List, Optional, Tuple
+
+import boto3
+from botocore.config import Config
+from botocore.exceptions import ClientError
+from tqdm.asyncio import tqdm
+
+
+async def batch(
+    items: List[Any],
+    model: Optional[str] = None,
+    aws_region: Optional[str] = None,
+    aws_profile: Optional[str] = None,
+    bucket: Optional[str] = None,
+    input_prefix: Optional[str] = None,
+    output_prefix: Optional[str] = None,
+    role_arn: Optional[str] = None,
+    **hyperparameters: Any,
+) -> List[Any]:
+    """Process multiple inference requests in batch using AWS Bedrock.
+
+    This asynchronous function handles batch processing of inference requests,
+    optimizing for cost and throughput using AWS Bedrock's Model Invocation Jobs.
+
+    Args:
+        items: List of preprocessed items to process.
+        model: Bedrock model ID (e.g., 'us.anthropic.claude-3-5-sonnet-20241022-v2:0').
+        aws_region: AWS region for Bedrock and S3.
+        aws_profile: AWS profile name for authentication.
+        bucket: S3 bucket name for input/output data.
+        input_prefix: S3 prefix for input data.
+        output_prefix: S3 prefix for output data.
+        role_arn: IAM role ARN for Bedrock execution.
+        hyperparameters: Additional parameters for the batch requests.
+
+    Returns:
+        A list of raw model responses.
+    """
+    # Set up AWS session and clients
+    session_kwargs = {}
+    if aws_profile:
+        session_kwargs["profile_name"] = aws_profile
+    if aws_region:
+        session_kwargs["region_name"] = aws_region
+
+    session = boto3.Session(**session_kwargs)
+
+    boto_config = Config(region_name=aws_region, retries={"max_attempts": 10, "mode": "adaptive"})
+
+    s3_client = session.client("s3", config=boto_config)
+    bedrock_client = session.client("bedrock", config=boto_config)
+
+    # Upload batch data
+    input_uri = await _upload_batch(
+        items, s3_client, bucket, input_prefix, model, **hyperparameters
+    )
+
+    # Start batch job
+    job_arn = await _start_batch_job(
+        bedrock_client, model, input_uri, bucket, output_prefix, role_arn
+    )
+
+    # Wait for completion with progress tracking
+    await _wait_for_completion(bedrock_client, job_arn, len(items))
+
+    # Retrieve results
+    results = await _get_batch_results(s3_client, bedrock_client, job_arn)
+
+    return results
+
+
+async def _upload_batch(
+    items: List[Any],
+    s3_client: Any,
+    bucket: Optional[str],
+    input_prefix: Optional[str],
+    model: Optional[str],
+    **hyperparameters: Any,
+) -> str:
+    """Create a JSONL file from preprocessed items and upload to S3 for batch processing."""
+
+    # Generate unique run ID and key
+    run_id = datetime.utcnow().strftime("%Y%m%dT%H%M%S") + "-" + uuid.uuid4().hex[:8]
+
+    if input_prefix:
+        input_key = f"{input_prefix.rstrip('/')}/inputs-{run_id}.jsonl"
+    else:
+        input_key = f"inputs-{run_id}.jsonl"
+
+    # Create temp JSONL file
+    with tempfile.NamedTemporaryFile(mode="w", suffix=".jsonl", delete=False) as f:
+        for i, item in enumerate(items):
+            # Construct batch request in Bedrock format
+            record = {
+                "recordId": f"rec-{i:04d}",
+                "modelInput": _build_claude_messages_payload(item, **hyperparameters),
+            }
+            f.write(json.dumps(record, separators=(",", ":")) + "\n")
+        file_path = f.name
+
+    # Upload to S3
+    try:
+        body = open(file_path, "rb").read()
+        s3_client.put_object(
+            Bucket=bucket,
+            Key=input_key,
+            Body=body,
+            StorageClass="INTELLIGENT_TIERING",
+            ContentType="application/json",
+        )
+        input_uri = f"s3://{bucket}/{input_key}"
+    except Exception as e:
+        raise Exception(f"Failed to upload file to S3: {e}")
+    finally:
+        # Clean up temp file
+        os.unlink(file_path)
+
+    return input_uri
+
+
+def _build_claude_messages_payload(item: Any, **hyperparameters: Any) -> Dict[str, Any]:
+    """Build Claude messages payload for Bedrock batch processing."""
+
+    # item is a list of messages from our preprocessor
+    messages = item
+
+    # Convert to Bedrock format and extract system message
+    bedrock_messages = []
+    system_content = None
+
+    for msg in messages:
+        if msg["role"] == "system":
+            system_content = msg["content"]
+        else:
+            bedrock_messages.append(
+                {"role": msg["role"], "content": [{"type": "text", "text": msg["content"]}]}
+            )
+
+    payload = {
+        "anthropic_version": "bedrock-2023-05-31",
+        "max_tokens": 256,
+        "messages": bedrock_messages,
+    }
+
+    if system_content:
+        payload["system"] = system_content
+
+    payload.update(hyperparameters)
+    return payload
+
+
+async def _start_batch_job(
+    bedrock_client: Any,
+    model: Optional[str],
+    input_uri: str,
+    bucket: Optional[str],
+    output_prefix: Optional[str],
+    role_arn: Optional[str],
+) -> str:
+    """Start a Bedrock Model Invocation Job."""
+
+    # Generate unique job name and output URI
+    run_id = datetime.utcnow().strftime("%Y%m%dT%H%M%S") + "-" + uuid.uuid4().hex[:8]
+    job_name = f"bedrock-batch-{run_id}"
+
+    if output_prefix:
+        output_uri = f"s3://{bucket}/{output_prefix.rstrip('/')}/job-{run_id}/"
+    else:
+        output_uri = f"s3://{bucket}/job-{run_id}/"
+
+    try:
+        response = bedrock_client.create_model_invocation_job(
+            jobName=job_name,
+            modelId=model,
+            roleArn=role_arn,
+            inputDataConfig={"s3InputDataConfig": {"s3Uri": input_uri}},
+            outputDataConfig={"s3OutputDataConfig": {"s3Uri": output_uri}},
+            tags=[{"key": "project", "value": "scorebook-batch"}],
+        )
+        job_arn: str = response["jobArn"]
+        return job_arn
+    except ClientError as e:
+        error_info = e.response.get("Error", {})
+        raise Exception(f"Failed to create batch job: {error_info}")
+
+
+async def _wait_for_completion(bedrock_client: Any, job_arn: str, total_items: int) -> None:
+    """Wait for batch job completion with progress tracking."""
+
+    # Initialize progress bar
+    pbar = tqdm(total=total_items, desc="Batch processing", unit="requests")
+
+    terminal_states = {"Completed", "Failed", "Stopped"}
+    sleep_time = 15
+
+    while True:
+        try:
+            desc = bedrock_client.get_model_invocation_job(jobIdentifier=job_arn)
+            status = desc["status"]
+
+            # Get progress if available
+            job_state = desc.get("jobState", {})
+            progress = job_state.get("percentComplete")
+
+            # Update progress bar
+            if progress is not None:
+                completed = int((progress / 100) * total_items)
+                pbar.n = completed
+                pbar.set_postfix(status=status, progress=f"{progress}%")
+            else:
+                pbar.set_postfix(status=status)
+
+            pbar.refresh()
+
+            if status in terminal_states:
+                if status == "Completed":
+                    pbar.n = pbar.total
+                    pbar.set_postfix(status="COMPLETED")
+                else:
+                    pbar.close()
+                    error_msg = desc.get("failureMessage", f"Job ended with status {status}")
+                    raise Exception(f"Batch job failed: {error_msg}")
+                break
+
+            # Wait before checking again
+            await asyncio.sleep(sleep_time)
+
+        except Exception as e:
+            pbar.close()
+            raise e
+
+    pbar.close()
+
+
+async def _get_batch_results(s3_client: Any, bedrock_client: Any, job_arn: str) -> List[str]:
+    """Download and parse batch results from S3."""
+
+    # Get job details to find output location
+    desc = bedrock_client.get_model_invocation_job(jobIdentifier=job_arn)
+    output_uri = desc["outputDataConfig"]["s3OutputDataConfig"]["s3Uri"]
+
+    bucket_name, prefix = s3_uri_to_bucket_and_prefix(output_uri)
+
+    # Find the output JSONL file
+    try:
+        response = s3_client.list_objects_v2(Bucket=bucket_name, Prefix=prefix)
+        contents = response.get("Contents", [])
+
+        # Look for the output JSONL file
+        output_key = None
+        for obj in contents:
+            if obj["Key"].endswith(".jsonl.out"):
+                output_key = obj["Key"]
+                break
+
+        if not output_key:
+            raise Exception("No output JSONL file found in S3")
+
+        # Download and parse results
+        obj_response = s3_client.get_object(Bucket=bucket_name, Key=output_key)
+        content = obj_response["Body"].read().decode("utf-8")
+
+        results = []
+        for line in content.strip().split("\n"):
+            if line.strip():
+                result_obj = json.loads(line)
+                # Extract text from Claude response format
+                model_output = result_obj.get("modelOutput", {})
+                content_list = model_output.get("content", [])
+                if content_list and len(content_list) > 0:
+                    text = content_list[0].get("text", "")
+                    results.append(text)
+                else:
+                    results.append("")
+
+        return results
+
+    except Exception as e:
+        raise Exception(f"Failed to retrieve batch results: {e}")
+
+
+def s3_uri_to_bucket_and_prefix(s3_uri: str) -> Tuple[str, str]:
+    """Parse S3 URI to bucket and prefix."""
+    # Parse S3 URI
+    if s3_uri.startswith("s3://"):
+        uri_parts = s3_uri[5:].split("/", 1)
+        bucket_name = uri_parts[0]
+        prefix = uri_parts[1] if len(uri_parts) > 1 else ""
+    else:
+        raise ValueError(f"Invalid S3 URI: {s3_uri}")
+    return bucket_name, prefix

scorebook-0.0.2/src/scorebook/inference/vertex.py

@@ -0,0 +1,295 @@
+"""
+Google Cloud Vertex AI batch inference implementation for Scorebook.
+
+This module provides utilities for running batch inference using Google Cloud
+Vertex AI Gemini models, supporting large-scale asynchronous processing. It handles
+API communication, request formatting, response processing, and Cloud Storage operations.
+"""
+
+import asyncio
+import json
+import os
+import tempfile
+from datetime import datetime
+from typing import Any, Dict, List, Optional, Union
+
+import fsspec
+import pandas as pd
+from google import genai
+from google.cloud import storage
+from google.genai import types
+from tqdm.asyncio import tqdm
+
+
+async def responses(
+    items: List[
+        Union[
+            str,
+            List[str],
+            types.Content,
+            List[types.Content],
+            types.FunctionCall,
+            List[types.FunctionCall],
+            types.Part,
+            List[types.Part],
+        ]
+    ],
+    model: str,
+    client: Optional[genai.Client] = None,
+    project_id: Optional[str] = None,
+    location: str = "us-central1",
+    system_instruction: Optional[str] = None,
+    **hyperparameters: Any,
+) -> List[types.GenerateContentResponse]:
+    """Process multiple inference requests using Google Cloud Vertex AI.
+
+    This asynchronous function handles multiple inference requests,
+    manages the API communication, and processes the responses.
+
+    Args:
+        items: List of preprocessed items to process.
+        model: Gemini model ID to use (e.g., 'gemini-2.0-flash-001').
+        client: Optional Vertex AI client instance.
+        project_id: Google Cloud Project ID. If None, uses GOOGLE_CLOUD_PROJECT env var.
+        location: Google Cloud region (default: 'us-central1').
+        system_instruction: Optional system instruction to guide model behavior.
+        hyperparameters: Additional parameters for the requests.
+
+    Returns:
+        List of raw model responses.
+    """
+    if client is None:
+        if project_id is None:
+            project_id = os.getenv("GOOGLE_CLOUD_PROJECT")
+            if not project_id:
+                raise ValueError(
+                    "Project ID must be provided or set in GOOGLE_CLOUD_PROJECT "
+                    "environment variable"
+                )
+
+        client = genai.Client(
+            vertexai=True,
+            project=project_id,
+            location=location,
+            http_options=types.HttpOptions(api_version="v1"),
+        )
+
+    # Create config if system_instruction or hyperparameters are provided
+    config = None
+    if system_instruction or hyperparameters:
+        config_dict = {}
+        if system_instruction:
+            config_dict["system_instruction"] = system_instruction
+        if hyperparameters:
+            config_dict.update(hyperparameters)
+        config = types.GenerateContentConfig(**config_dict)
+
+    results = []
+    for item in items:
+        response = client.models.generate_content(
+            model=model,
+            contents=item,
+            config=config,
+        )
+        results.append(response)
+
+    return results
+
+
+async def batch(
+    items: List[Any],
+    model: str,
+    project_id: Optional[str] = None,
+    location: str = "us-central1",
+    input_bucket: Optional[str] = None,
+    output_bucket: Optional[str] = None,
+    **hyperparameters: Any,
+) -> List[Any]:
+    """Process multiple inference requests in batch using Google Cloud Vertex AI.
+
+    This asynchronous function handles batch processing of inference requests,
+    optimizing for cost and throughput using Google Cloud's batch prediction API.
+
+    Args:
+        items: List of preprocessed items to process.
+        model: Gemini model ID to use (e.g., 'gemini-2.0-flash-001').
+        project_id: Google Cloud Project ID. If None, uses GOOGLE_CLOUD_PROJECT env var.
+        location: Google Cloud region (default: 'us-central1').
+        input_bucket: GCS bucket for input data (required).
+        output_bucket: GCS bucket for output data (required).
+        hyperparameters: Additional parameters for the batch requests.
+
+    Returns:
+        A list of raw model responses.
+    """
+    # Set up project ID
+    if project_id is None:
+        project_id = os.getenv("GOOGLE_CLOUD_PROJECT")
+        if not project_id:
+            raise ValueError(
+                "Project ID must be provided or set in GOOGLE_CLOUD_PROJECT " "environment variable"
+            )
+
+    if not input_bucket or not output_bucket:
+        raise ValueError("Both input_bucket and output_bucket must be provided")
+
+    # Initialize client
+    client = genai.Client(vertexai=True, project=project_id, location=location)
+
+    # Upload batch data
+    input_uri = await _upload_batch(items, input_bucket, model, project_id, **hyperparameters)
+
+    # Start batch job
+    batch_job = await _start_batch_job(client, model, input_uri, output_bucket)
+
+    # Wait for completion with progress tracking
+    await _wait_for_completion(client, batch_job, len(items))
+
+    # Retrieve results
+    results = await _get_batch_results(batch_job)
+
+    return results
+
+
+async def _upload_batch(
+    items: List[Any], input_bucket: str, model: str, project_id: str, **hyperparameters: Any
+) -> str:
+    """Create a JSONL file from preprocessed items and upload to GCS for batch processing."""
+
+    # Create temp JSONL file
+    with tempfile.NamedTemporaryFile(mode="w", suffix=".jsonl", delete=False) as f:
+        for item in items:
+            # Construct batch request in Vertex AI format
+            request_data: Dict[str, Any] = {
+                "request": {
+                    "contents": [
+                        {
+                            "role": "user",
+                            "parts": [
+                                {
+                                    "text": (
+                                        str(item)
+                                        if not isinstance(item, list)
+                                        else item[0]["content"]
+                                    )
+                                }
+                            ],
+                        }
+                    ]
+                }
+            }
+
+            # Only add generationConfig if hyperparameters are provided
+            if hyperparameters:
+                request_data["request"]["generationConfig"] = hyperparameters
+            f.write(json.dumps(request_data) + "\n")
+        file_path = f.name
+
+    # Upload to GCS using Python client
+    timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
+
+    # Parse bucket name and path from input_bucket
+    if input_bucket.startswith("gs://"):
+        bucket_path = input_bucket[5:]  # Remove 'gs://' prefix
+    else:
+        bucket_path = input_bucket
+
+    # Split bucket name and path
+    bucket_name = bucket_path.split("/")[0]
+    bucket_prefix = "/".join(bucket_path.split("/")[1:]) if "/" in bucket_path else ""
+
+    # Create blob path
+    blob_name = (
+        f"{bucket_prefix}/batch_input_{timestamp}.jsonl"
+        if bucket_prefix
+        else f"batch_input_{timestamp}.jsonl"
+    )
+
+    # Upload using Google Cloud Storage client
+    try:
+        gcs_client = storage.Client(project=project_id)
+        bucket = gcs_client.bucket(bucket_name)
+        blob = bucket.blob(blob_name)
+
+        with open(file_path, "rb") as f:
+            blob.upload_from_file(f)
+
+        input_uri = f"gs://{bucket_name}/{blob_name}"
+
+    except Exception as e:
+        raise Exception(f"Failed to upload file to GCS: {e}")
+    finally:
+        # Clean up temp file
+        os.unlink(file_path)
+
+    return input_uri
+
+
+async def _start_batch_job(
+    client: genai.Client, model: str, input_uri: str, output_bucket: str
+) -> Any:
+    """Start a batch prediction job."""
+    batch_job = client.batches.create(
+        model=model,
+        src=input_uri,
+        config=types.CreateBatchJobConfig(dest=output_bucket),
+    )
+    return batch_job
+
+
+async def _wait_for_completion(client: genai.Client, batch_job: Any, total_items: int) -> None:
+    """Wait for batch job completion with progress tracking."""
+    # Initialize progress bar
+    pbar = tqdm(total=total_items, desc="Batch processing", unit="requests")
+
+    while True:
+        # Refresh job status
+        batch_job = client.batches.get(name=batch_job.name)
+        state = batch_job.state
+
+        # Update progress bar
+        pbar.set_postfix(status=str(state).replace("JobState.JOB_STATE_", ""))
+        pbar.refresh()
+
+        if state.name == "JOB_STATE_SUCCEEDED":
+            pbar.n = pbar.total
+            pbar.set_postfix(status="COMPLETED")
+            break
+        elif state.name == "JOB_STATE_FAILED":
+            pbar.close()
+            error_msg = getattr(batch_job, "error", "Unknown error")
+            raise Exception(f"Batch job failed: {error_msg}")
+        elif state.name in ["JOB_STATE_CANCELLED", "JOB_STATE_PAUSED"]:
+            pbar.close()
+            raise Exception(f"Batch job was {state.name}")
+
+        # Wait before checking again
+        await asyncio.sleep(30)
+
+    pbar.close()
+
+
+async def _get_batch_results(batch_job: Any) -> List[str]:
+    """Download and parse batch results from GCS."""
+
+    # Set up GCS filesystem
+    fs = fsspec.filesystem("gcs")
+
+    # Find predictions file - the pattern is: dest_uri/prediction-model-*/predictions.jsonl
+    output_uri = batch_job.dest.gcs_uri.rstrip("/")
+    file_paths = fs.glob(f"{output_uri}/prediction-model-*/predictions.jsonl")
+
+    if not file_paths:
+        raise Exception("No predictions file found in output bucket")
+
+    # Load and parse results
+    df = pd.read_json(f"gs://{file_paths[0]}", lines=True)
+
+    results = []
+    for _, row in df.iterrows():
+        # Extract text content from successful responses
+        response = row["response"]
+        text_content = response["candidates"][0]["content"]["parts"][0]["text"]
+        results.append(text_content)
+
+    return results

{scorebook-0.0.1 → scorebook-0.0.2}/src/scorebook/types/eval_dataset.py

@@ -4,6 +4,7 @@ import csv
 import json
 from typing import Any, Dict, Iterator, List, Optional, Type, Union
 
+import yaml
 from datasets import Dataset as HuggingFaceDataset
 from datasets import DatasetDict as HuggingFaceDatasetDict
 from datasets import load_dataset
@@ -21,6 +22,7 @@ class EvalDataset:
         label: str,
         metrics: Union[str, Type[MetricBase], List[Union[str, Type[MetricBase]]]],
         hf_dataset: HuggingFaceDataset,
+        prompt_template: Optional[str] = None,
     ):
         """
         Create a new scorebook evaluation dataset instance.
@@ -29,11 +31,13 @@ class EvalDataset:
         :param label: The label field of the dataset.
         :param metrics: The specified metrics associated with the dataset.
         :param hf_dataset: The dataset as a hugging face dataset object.
+        :param prompt_template: Optional prompt template for building prompts from dataset items.
         """
         self.name: str = name
         self.label: str = label
         self.metrics: List[MetricBase] = self._resolve_metrics(metrics)
         self._hf_dataset: Optional[HuggingFaceDataset] = hf_dataset
+        self.prompt_template: Optional[str] = prompt_template
 
     def __len__(self) -> int:
         """Return the number of items in the dataset."""
@@ -286,6 +290,52 @@ class EvalDataset:
 
         return cls(name=path, label=label, metrics=metrics, hf_dataset=hf_dataset)
 
+    @classmethod
+    def from_yaml(cls, file_path: str) -> "EvalDataset":
+        """Instantiate an EvalDataset from a YAML file.
+
+        The YAML file should contain configuration for loading a dataset, including:
+        - name: Name of the dataset or Hugging Face dataset path
+        - label: The field used as the evaluation label
+        - metrics: List of metrics to evaluate
+        - split: Optional split name to load
+        - template: Optional prompt template
+
+        Returns:
+            An EvalDataset instance configured according to the YAML file.
+
+        Raises:
+            ValueError: If the YAML file is invalid or missing required fields.
+        """
+        path = validate_path(file_path, expected_suffix=".yaml")
+
+        try:
+            with path.open("r", encoding="utf-8") as f:
+                config = yaml.safe_load(f)
+        except yaml.YAMLError as e:
+            raise ValueError(f"Invalid YAML in {file_path}: {e}") from e
+
+        # Validate required fields
+        required_fields = ["name", "label", "metrics"]
+        missing_fields = [field for field in required_fields if field not in config]
+        if missing_fields:
+            raise ValueError(f"Missing required fields in YAML config: {', '.join(missing_fields)}")
+
+        # Load the dataset from Hugging Face
+        dataset = cls.from_huggingface(
+            path=config["name"],
+            label=config["label"],
+            metrics=config["metrics"],
+            split=config.get("split"),  # Optional field
+            name=config.get("config"),  # Optional field
+        )
+
+        # Add template if provided
+        if "template" in config:
+            dataset.prompt_template = config["template"]
+
+        return dataset
+
     @staticmethod
     def _resolve_metrics(
         metrics: Union[

{scorebook-0.0.1 → scorebook-0.0.2}/src/scorebook/types/eval_result.py

@@ -42,6 +42,7 @@ class EvalResult:
             result = {
                 "item_id": idx,
                 "dataset_name": self.eval_dataset.name,
+                "inference_output": self.inference_outputs[idx],
                 **{
                     metric: self.metric_scores[metric]["item_scores"][idx]
                     for metric in metric_names
@@ -74,13 +75,13 @@ class EvalResult:
     def to_dict(self) -> Dict[str, Any]:
         """Return a dictionary representing the evaluation results."""
         return {
-            "aggregate": [
+            "aggregate_results": [
                 {
                     **getattr(self.eval_dataset, "hyperparams", {}),
                     **self.aggregate_scores,
                 }
             ],
-            "per_sample": [item for item in self.item_scores],
+            "item_results": [item for item in self.item_scores],
         }
 
     def to_csv(self, file_path: str) -> None:
@@ -113,7 +114,10 @@ class EvalResult:
                 writer.writerow(row)
 
     def to_json(self, file_path: str) -> None:
-        """Save evaluation results to a JSON file in structured format (Option 2)."""
+        """Save evaluation results to a JSON file in a structured format.
+
+        The JSON file will contain both aggregate & item results, produced by the to_dict() method.
+        """
         Path(file_path).parent.mkdir(parents=True, exist_ok=True)
         with open(file_path, "w") as f:
             json.dump(self.to_dict(), f, indent=2)

{scorebook-0.0.1 → scorebook-0.0.2}/src/scorebook/types/inference_pipeline.py

@@ -57,7 +57,7 @@ class InferencePipeline:
             List of processed outputs after running through the complete pipeline
         """
         if self.preprocessor:
-            input_items = [self.preprocessor(item) for item in items]
+            input_items = [self.preprocessor(item, hyperparameters) for item in items]
         else:
             input_items = items
 
@@ -67,7 +67,10 @@ class InferencePipeline:
             inference_outputs = self.inference_function(input_items, **hyperparameters)
 
         if self.postprocessor:
-            return [self.postprocessor(inference_output) for inference_output in inference_outputs]
+            return [
+                self.postprocessor(inference_output, hyperparameters)
+                for inference_output in inference_outputs
+            ]
         else:
             return cast(List[Any], inference_outputs)
 

{scorebook-0.0.1 → scorebook-0.0.2}/src/scorebook/utils/__init__.py

@@ -1,8 +1,9 @@
 """Utility functions and common helpers for the Scorebook framework."""
 
 from scorebook.utils.async_utils import is_awaitable
+from scorebook.utils.build_prompt import build_prompt
 from scorebook.utils.io_helpers import validate_path
 from scorebook.utils.progress_bars import evaluation_progress
 from scorebook.utils.transform_helpers import expand_dict
 
-__all__ = ["is_awaitable", "validate_path", "expand_dict", "evaluation_progress"]
+__all__ = ["is_awaitable", "validate_path", "expand_dict", "evaluation_progress", "build_prompt"]

scorebook-0.0.2/src/scorebook/utils/build_prompt.py

@@ -0,0 +1,52 @@
+"""
+Module for building prompt strings using Jinja2 templating.
+
+Provides functionality to render prompts from templates with custom filters
+and global variables, using strict undefined handling for better error detection.
+"""
+
+from typing import Any, Dict, Optional
+
+from jinja2 import BaseLoader, Environment, StrictUndefined
+
+from scorebook.utils.jinja_helpers import default_jinja_filters, default_jinja_globals
+
+
+def build_prompt(
+    prompt_template: str,
+    prompt_args: Dict[str, Any],
+    filters: Optional[Dict[str, Any]] = None,
+    globals_dict: Optional[Dict[str, Any]] = None,
+) -> str:
+    """
+    Build a prompt string from a template and arguments.
+
+    Args:
+        prompt_template: Jinja2 template string
+        prompt_args: Dictionary of arguments to pass to the template
+        filters: Dictionary of Jinja2 filters. Defaults to default_jinja_filters().
+        globals_dict: Dictionary of global functions/variables. Defaults to default_jinja_globals().
+
+    Returns:
+        str: Rendered prompt string
+    """
+
+    # Use defaults if not provided
+    filters = filters or default_jinja_filters()
+    globals_dict = globals_dict or default_jinja_globals()
+
+    # Create a Jinja2 environment with strict undefined handling
+    env = Environment(
+        loader=BaseLoader(),
+        undefined=StrictUndefined,
+        trim_blocks=True,
+        lstrip_blocks=True,
+    )
+
+    # Add filters and globals
+    env.filters.update(filters)
+    env.globals.update(globals_dict)
+
+    # Render the template
+    template = env.from_string(prompt_template)
+    return str(template.render(**prompt_args))

scorebook-0.0.2/src/scorebook/utils/jinja_helpers.py

@@ -0,0 +1,146 @@
+"""Jinja2 template helper functions for Scorebook."""
+
+import json
+import re
+from typing import Any, Dict, List
+
+# Helper functions for use in Jinja templates
+
+
+def number_to_letter(index: int, uppercase: bool = True) -> str:
+    """Convert a number to a letter (0->A, 1->B, etc.).
+
+    Args:
+        index: The number to convert to a letter (0-based index, must be 0-25)
+        uppercase: If True, returns uppercase letter; if False, returns lowercase
+
+    Returns:
+        str: A letter from A-Z (or a-z if uppercase is False)
+
+    Raises:
+        ValueError: If index is less than 0 or greater than 25
+    """
+    if not 0 <= index <= 25:
+        raise ValueError("Index must be between 0 and 25 inclusive")
+    letter = chr(65 + index)
+    return letter if uppercase else letter.lower()
+
+
+def letter_to_number(letter: str) -> int:
+    """Convert a letter to a number (A->0, B->1, etc.).
+
+    Args:
+        letter: A single letter character (A-Z or a-z)
+
+    Returns:
+        int: The zero-based position of the letter in the alphabet
+
+    Raises:
+        ValueError: If the input is not a single letter character
+    """
+    if not letter.isalpha() or len(letter) != 1:
+        raise ValueError("Input must be a single letter character")
+    return ord(letter.upper()) - 65
+
+
+def format_list(items: List[Any], separator: str = ", ", last_separator: str = " and ") -> str:
+    """Format a list with proper separators and conjunction.
+
+    Examples:
+        format_list(["a", "b", "c"]) -> "a, b and c"
+        format_list(["a", "b"]) -> "a and b"
+        format_list(["a"]) -> "a"
+    """
+    if not items:
+        return ""
+    if len(items) == 1:
+        return str(items[0])
+    if len(items) == 2:
+        return f"{items[0]}{last_separator}{items[1]}"
+    return f"{separator.join(str(item) for item in items[:-1])}{last_separator}{items[-1]}"
+
+
+def truncate_text(text: str, max_length: int, suffix: str = "...") -> str:
+    """Truncate text to a maximum length with optional suffix."""
+    if len(text) <= max_length:
+        return text
+    return text[: max_length - len(suffix)] + suffix
+
+
+def format_number(number: float, precision: int = 2) -> str:
+    """Format a number with specified decimal places."""
+    return f"{number:.{precision}f}"
+
+
+def extract_initials(text: str) -> str:
+    """Extract initials from a text string.
+
+    Examples:
+        extract_initials("John Doe") -> "JD"
+        extract_initials("Machine Learning Model") -> "MLM"
+    """
+    words = re.findall(r"\b[A-Za-z]", text)
+    return "".join(words).upper()
+
+
+def json_pretty(obj: Any, indent: int = 2) -> str:
+    """Pretty-print an object as JSON."""
+    return json.dumps(obj, indent=indent, ensure_ascii=False)
+
+
+def percentage(value: float, total: float, precision: int = 1) -> str:
+    """Calculate and format a percentage.
+
+    Examples:
+        percentage(25, 100) -> "25.0%"
+        percentage(1, 3, 2) -> "33.33%"
+    """
+    if total == 0:
+        return "0.0%"
+    pct = (value / total) * 100
+    return f"{pct:.{precision}f}%"
+
+
+def ordinal(n: int) -> str:
+    """Convert number to ordinal format like 1st, 2nd, 3rd, etc."""
+    if 10 <= n % 100 <= 20:
+        suffix = "th"
+    else:
+        suffix = {1: "st", 2: "nd", 3: "rd"}.get(n % 10, "th")
+    return f"{n}{suffix}"
+
+
+def default_jinja_globals() -> Dict[str, Any]:
+    """Get default global functions for Jinja templates."""
+    return {
+        "number_to_letter": number_to_letter,
+        "letter_to_number": letter_to_number,
+        "format_list": format_list,
+        "truncate_text": truncate_text,
+        "format_number": format_number,
+        "extract_initials": extract_initials,
+        "json_pretty": json_pretty,
+        "percentage": percentage,
+        "ordinal": ordinal,
+        "max": max,
+        "min": min,
+        "len": len,
+        "abs": abs,
+        "round": round,
+        "sum": sum,
+        "sorted": sorted,
+        "enumerate": enumerate,
+        "zip": zip,
+        "range": range,
+    }
+
+
+def default_jinja_filters() -> Dict[str, Any]:
+    """Get default filters for Jinja templates."""
+    return {
+        "chr": chr,
+        "ord": ord,
+        "abs": abs,
+        "round": round,
+        "len": len,
+    }