scorebook 0.0.13__py3-none-any.whl → 0.0.15__py3-none-any.whl

tutorials/examples/6-providers/vertex/batch_example.py

@@ -0,0 +1,166 @@
+"""
+Google Cloud Vertex AI Batch Inference Example.
+
+This example demonstrates how to leverage Google Cloud Vertex AI's Batch API for
+cost-effective, large-scale model evaluation using Scorebook. It uses Gemini models
+for batch processing with automatic GCS upload/download and job management.
+
+This example requires Google Cloud SDK (gsutil) to be installed and authenticated,
+and a Google Cloud project with Vertex AI enabled. Set the project ID in the
+GOOGLE_CLOUD_PROJECT environment variable or pass it as a command line argument.
+
+Compare with the Portkey batch example to understand the differences
+between different cloud providers' batch processing approaches.
+"""
+
+import json
+import os
+from pathlib import Path
+from typing import Any
+
+from dotenv import load_dotenv
+
+from scorebook import EvalDataset, InferencePipeline, evaluate
+from scorebook.inference.clients.vertex import batch
+from scorebook.metrics.accuracy import Accuracy
+
+
+def main() -> None:
+    """Run the Vertex AI batch inference example."""
+    # Load environment variables from .env file for configuration
+    load_dotenv()
+
+    output_dir, model_name, input_bucket, output_bucket, project_id = setup_arguments()
+
+    # Step 1: Load the evaluation dataset
+    dataset = EvalDataset.from_json(
+        "examples/example_datasets/dataset.json", label="answer", metrics=[Accuracy]
+    )
+
+    # Step 2: Define the preprocessing function for Vertex AI Batch API
+    def preprocessor(eval_item: dict) -> list:
+        """Pre-process dataset items into Vertex AI Batch API format."""
+        prompt = eval_item["question"]
+
+        # Create the batch API request messages format for Vertex AI
+        messages = [
+            {
+                "role": "system",
+                "content": "Answer the question directly and concisely as a single word",
+            },
+            {"role": "user", "content": prompt},
+        ]
+
+        return messages
+
+    # Step 3: Define the postprocessing function
+    def postprocessor(response: str) -> str:
+        """Post-process Vertex AI batch response to extract the answer."""
+        # The batch function returns the message content directly
+        return response.strip()
+
+    # Step 4: Create the inference pipeline for batch processing
+
+    async def inference_function(items: list, **hyperparams: Any) -> Any:  # noqa
+        return await batch(
+            items,
+            model=model_name,
+            project_id=project_id,
+            input_bucket=input_bucket,
+            output_bucket=output_bucket,
+            **hyperparams,
+        )
+
+    inference_pipeline = InferencePipeline(
+        model=model_name,
+        preprocessor=preprocessor,
+        inference_function=inference_function,
+        postprocessor=postprocessor,
+    )
+
+    # Step 5: Run the batch evaluation
+    print(f"Running Vertex AI Batch API evaluation with model: {model_name}")
+    print(f"Project ID: {project_id}")
+    print(f"Input bucket: {input_bucket}")
+    print(f"Output bucket: {output_bucket}")
+    print(f"Processing {len(dataset)} items using batch inference...")
+    print("Note: Batch processing may take several minutes to complete.")
+
+    # For demonstration, limit to 25 items
+    results = evaluate(inference_pipeline, dataset, item_limit=25, score_type="all")
+    print("\nBatch evaluation completed!")
+    print(results)
+
+    # Step 6: Save results to file
+    output_file = output_dir / "vertex_batch_output.json"
+    with open(output_file, "w") as f:
+        json.dump(results, f, indent=4)
+        print(f"Results saved in {output_file}")
+
+
+# ============================================================================
+# Utility Functions
+# ============================================================================
+
+
+def setup_arguments() -> tuple[Path, str, str, str, str]:
+    """Parse command line arguments."""
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Run Vertex AI Batch API evaluation and save results."
+    )
+    parser.add_argument(
+        "--output-dir",
+        type=str,
+        default=str(Path.cwd() / "results"),
+        help="Directory to save evaluation outputs (JSON).",
+    )
+    parser.add_argument(
+        "--model",
+        type=str,
+        required=True,
+        help="Gemini model to use for batch inference (e.g., gemini-2.0-flash-001)",
+    )
+    parser.add_argument(
+        "--input-bucket",
+        type=str,
+        required=True,
+        help="GCS bucket name for input data (without gs:// prefix)",
+    )
+    parser.add_argument(
+        "--output-bucket",
+        type=str,
+        required=True,
+        help="GCS bucket name for output data (without gs:// prefix)",
+    )
+    parser.add_argument(
+        "--project-id",
+        type=str,
+        help="Google Cloud Project ID (defaults to GOOGLE_CLOUD_PROJECT env var)",
+    )
+
+    args = parser.parse_args()
+
+    output_dir = Path(args.output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    # Handle project ID fallback
+    project_id = args.project_id or os.getenv("GOOGLE_CLOUD_PROJECT")
+    if not project_id:
+        raise ValueError(
+            "Project ID must be provided via --project-id or "
+            "GOOGLE_CLOUD_PROJECT environment variable"
+        )
+
+    return (
+        output_dir,
+        str(args.model),
+        str(args.input_bucket),
+        str(args.output_bucket),
+        str(project_id),
+    )
+
+
+if __name__ == "__main__":
+    main()

tutorials/examples/6-providers/vertex/messages_example.py

@@ -0,0 +1,142 @@
+"""
+Google Cloud Vertex AI Model Inference Example.
+
+This example demonstrates how to evaluate language models using Google Cloud
+Vertex AI's Gemini models with Scorebook for real-time API calls.
+
+Prerequisites: Google Cloud SDK (gcloud) authenticated and GOOGLE_CLOUD_PROJECT
+environment variable set, or pass project ID as command line argument.
+"""
+
+import json
+from pathlib import Path
+from typing import Any, Dict
+
+from dotenv import load_dotenv
+
+from scorebook import EvalDataset, InferencePipeline, evaluate
+from scorebook.inference.clients.vertex import responses
+from scorebook.metrics.accuracy import Accuracy
+
+
+def main() -> None:
+    """Run the Vertex AI inference example."""
+    # Load environment variables from .env file for configuration
+    load_dotenv()
+
+    output_dir, model_name, project_id = setup_arguments()
+
+    # Step 1: Load the evaluation dataset
+    # Create an EvalDataset from local JSON file
+    # - Uses 'answer' field as ground truth labels
+    # - Configures Accuracy metric for evaluation
+    # - Loads from examples/example_datasets/dataset.json
+    dataset = EvalDataset.from_json(
+        "examples/example_datasets/dataset.json", label="answer", metrics=[Accuracy]
+    )
+
+    # Step 2: Define the preprocessing function
+    # Convert raw dataset items into Vertex AI API-compatible format
+    # This function formats the question for the Gemini model
+    def preprocessor(eval_item: Dict[str, str]) -> str:
+        """Pre-process dataset items into Vertex AI string format."""
+        return eval_item["question"]
+
+    # Step 3: Define the postprocessing function
+    # Extract the final answer from Vertex AI API response
+    # Handles response parsing and returns the response text
+    def postprocessor(response: Any) -> str:
+        """Post-process Vertex AI response to extract the answer."""
+        return str(response.text.strip())
+
+    # Step 4: Create the inference pipeline for cloud-based evaluation
+    # Combine preprocessing, Vertex AI inference, and postprocessing
+    # Uses scorebook's built-in Vertex AI responses function for API calls
+
+    # Create a system message with instructions for direct answers
+    system_prompt = """
+Answer the question directly and concisely.
+Do not provide lengthy explanations unless specifically asked.
+""".strip()
+
+    async def inference_function(items: list, **hyperparams: Any) -> Any:
+        return await responses(
+            items,
+            model=model_name,
+            project_id=project_id,
+            system_instruction=system_prompt,
+            **hyperparams,
+        )
+
+    inference_pipeline = InferencePipeline(
+        model=model_name,
+        preprocessor=preprocessor,
+        inference_function=inference_function,
+        postprocessor=postprocessor,
+    )
+
+    # Step 5: Run the cloud-based evaluation
+    # Execute evaluation using Vertex AI with the inference pipeline
+    # - Uses score_type="all" to get both aggregate and per-item results
+    # - Limits to 10 items for quick demonstration and cost control
+    print(f"Running Vertex AI evaluation with model: {model_name}")
+    print(f"Project ID: {project_id}")
+    print("Evaluating 10 items from local dataset...")
+
+    results = evaluate(inference_pipeline, dataset, item_limit=10, score_type="all")
+    print(results)
+
+    # Step 6: Save results to file
+    # Export evaluation results as JSON for later analysis
+    output_file = output_dir / "vertex_messages_output.json"
+    with open(output_file, "w") as f:
+        json.dump(results, f, indent=4)
+        print(f"Results saved in {output_file}")
+
+
+# ============================================================================
+# Utility Functions
+# ============================================================================
+
+
+def setup_arguments() -> tuple[Path, str, str]:
+    """Parse command line arguments."""
+    import argparse
+    import os
+
+    parser = argparse.ArgumentParser(description="Run Vertex AI evaluation and save results.")
+    parser.add_argument(
+        "--output-dir",
+        type=str,
+        default=str(Path.cwd() / "results"),
+        help="Directory to save evaluation outputs (JSON).",
+    )
+    parser.add_argument(
+        "--model",
+        type=str,
+        required=True,
+        help="Gemini model to use for inference (e.g., gemini-2.0-flash-001)",
+    )
+    parser.add_argument(
+        "--project-id",
+        type=str,
+        help="Google Cloud Project ID (defaults to GOOGLE_CLOUD_PROJECT env var)",
+    )
+    args = parser.parse_args()
+
+    output_dir = Path(args.output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    # Handle project ID fallback
+    project_id = args.project_id or os.getenv("GOOGLE_CLOUD_PROJECT")
+    if not project_id:
+        raise ValueError(
+            "Project ID must be provided via --project-id or "
+            "GOOGLE_CLOUD_PROJECT environment variable"
+        )
+
+    return output_dir, str(args.model), str(project_id)
+
+
+if __name__ == "__main__":
+    main()

tutorials/notebooks/1-scoring.ipynb

@@ -0,0 +1,162 @@
+{
+ "cells": [
+  {
+   "metadata": {},
+   "cell_type": "markdown",
+   "source": "# Scoring Model Outputs with Scorebook\n\nThis notebook demonstrates how to use Scorebook's `score()` function to evaluate pre-generated model predictions.\n\n## When to use `score()`\n\n- You already have model predictions and want to compute metrics\n- You want to re-score existing results with different metrics\n- You're importing evaluation results from another framework\n"
+  },
+  {
+   "metadata": {},
+   "cell_type": "markdown",
+   "source": [
+    "## Setup\n",
+    "\n",
+    "First, let's import the necessary modules:"
+   ]
+  },
+  {
+   "metadata": {},
+   "cell_type": "code",
+   "source": [
+    "from pprint import pprint\n",
+    "from scorebook import score\n",
+    "from scorebook.metrics.accuracy import Accuracy"
+   ],
+   "outputs": [],
+   "execution_count": null
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Prepare Your Data\n",
+    "\n",
+    "The `score()` function expects a list of items, where each item is a dictionary with:\n",
+    "- `input`: The input to the model (optional, for reference)\n",
+    "- `output`: The model's prediction\n",
+    "- `label`: The ground truth answer"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "metadata": {},
+   "source": [
+    "# Example: Pre-generated model outputs\n",
+    "items = [\n",
+    "    {\n",
+    "        \"input\": \"What is 2 + 2?\",\n",
+    "        \"output\": \"4\",\n",
+    "        \"label\": \"4\"\n",
+    "    },\n",
+    "    {\n",
+    "        \"input\": \"What is the capital of France?\",\n",
+    "        \"output\": \"Paris\",\n",
+    "        \"label\": \"Paris\"\n",
+    "    },\n",
+    "    {\n",
+    "        \"input\": \"Who wrote Romeo and Juliet?\",\n",
+    "        \"output\": \"William Shakespeare\",\n",
+    "        \"label\": \"William Shakespeare\"\n",
+    "    },\n",
+    "    {\n",
+    "        \"input\": \"What is 5 * 6?\",\n",
+    "        \"output\": \"30\",\n",
+    "        \"label\": \"30\"\n",
+    "    },\n",
+    "    {\n",
+    "        \"input\": \"What is the largest planet in our solar system?\",\n",
+    "        \"output\": \"Jupiter\",\n",
+    "        \"label\": \"Jupiter\"\n",
+    "    },\n",
+    "]\n",
+    "\n",
+    "print(f\"Prepared {len(items)} items for scoring\")"
+   ],
+   "outputs": [],
+   "execution_count": null
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Score the Results\n",
+    "\n",
+    "Now we'll use the `score()` function to compute accuracy metrics:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "metadata": {},
+   "source": [
+    "results = score(\n",
+    "    items=items,\n",
+    "    metrics=Accuracy,\n",
+    "    dataset_name=\"basic_questions\",\n",
+    "    model_name=\"example-model\",\n",
+    "    upload_results=False,  # Set to True to upload to Trismik\n",
+    ")\n",
+    "\n",
+    "pprint(results)"
+   ],
+   "outputs": [],
+   "execution_count": null
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Understanding the Results\n",
+    "\n",
+    "The results dictionary contains:\n",
+    "- `aggregates`: Overall metrics (e.g., accuracy across all items)\n",
+    "- `items`: Per-item scores and predictions\n",
+    "- `metadata`: Information about the dataset and model"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "metadata": {},
+   "source": [
+    "# View aggregate metrics\n",
+    "print(\"\\nAggregate Metrics:\")\n",
+    "print(f\"\\nAccuracy: {results['aggregate_results'][0]['accuracy']}\")\n",
+    "\n",
+    "# View per-item scores\n",
+    "print(\"\\nPer-Item Scores:\")\n",
+    "for i, item in enumerate(results['item_results'][:3], 1):\n",
+    "    print(f\"\\nItem {i}:\")\n",
+    "    print(f\"    Output: {item['output']}\")\n",
+    "    print(f\"     Label: {item['label']}\")\n",
+    "    print(f\"  Accuracy: {item['accuracy']}\")"
+   ],
+   "outputs": [],
+   "execution_count": null
+  },
+  {
+   "cell_type": "markdown",
+   "source": "## Next Steps\n\n- Try the **Evaluate** notebook to learn how to run inference and scoring together\n- See the **Upload Results** notebook to upload your scores to Trismik's dashboard\n- Explore custom metrics in the Scorebook documentation",
+   "metadata": {}
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}