scorebook 0.0.13__py3-none-any.whl → 0.0.15__py3-none-any.whl

tutorials/examples/5-upload_results/3-uploading_your_results.py

@@ -0,0 +1,153 @@
+"""Tutorials - Upload Results - Example 3 - Uploading Pre-Scored Results."""
+
+from pathlib import Path
+from pprint import pprint
+from typing import Any
+
+from dotenv import load_dotenv
+
+from tutorials.utils import save_results_to_json, setup_logging
+
+from scorebook import login, upload_result
+
+
+def main() -> Any:
+    """Upload pre-scored results directly to Trismik's dashboard.
+
+    This example demonstrates how to upload results where metrics are ALREADY computed.
+    This is different from score() or evaluate() which compute metrics for you.
+
+    Use upload_result() when you:
+        - Already have metric scores calculated
+        - Used a custom evaluation framework that computed metrics
+        - Want to import historical evaluation data with existing scores
+        - Have results from external tools (e.g., other eval frameworks)
+
+    The key difference from Examples 1 & 2:
+        - Example 1 (score): You have outputs/labels → Scorebook computes metrics
+        - Example 2 (evaluate): Scorebook runs inference AND computes metrics
+        - Example 3 (upload_result): You have EVERYTHING including metrics → Just upload
+
+    Prerequisites:
+        - Valid Trismik API key set in TRISMIK_API_KEY environment variable
+        - A Trismik project ID
+    """
+
+    # Step 1: Log in with your Trismik API key
+    # login() reads TRISMIK_API_KEY from environment variables or .env file
+    login()
+
+    # Step 2: Format your pre-scored results
+    # This is the structure that upload_result() expects:
+    # - aggregate_results: List with one dict containing overall metrics
+    # - item_results: List of dicts with per-item data and metric scores
+
+    # Example: You already ran an evaluation with your custom framework
+    # and computed accuracy, f1_score, etc.
+    my_pre_scored_results = {
+        "aggregate_results": [
+            {
+                "dataset": "spanish_translation",
+                "accuracy": 0.8,  # Your pre-computed aggregate accuracy
+                "bleu_score": 0.75,  # Your pre-computed BLEU score
+                # Add any hyperparameters used (optional)
+                "temperature": 0.7,
+                "max_tokens": 100,
+            }
+        ],
+        "item_results": [
+            {
+                "id": 0,
+                "dataset": "spanish_translation",
+                "input": "Translate 'hello' to Spanish",
+                "output": "hola",
+                "label": "hola",
+                "accuracy": 1.0,  # Item-level metric scores
+                "bleu_score": 1.0,
+                "temperature": 0.7,
+                "max_tokens": 100,
+            },
+            {
+                "id": 1,
+                "dataset": "spanish_translation",
+                "input": "Translate 'goodbye' to Spanish",
+                "output": "adiós",
+                "label": "adiós",
+                "accuracy": 1.0,
+                "bleu_score": 0.95,
+                "temperature": 0.7,
+                "max_tokens": 100,
+            },
+            {
+                "id": 2,
+                "dataset": "spanish_translation",
+                "input": "Translate 'thank you' to Spanish",
+                "output": "gracias",
+                "label": "gracias",
+                "accuracy": 1.0,
+                "bleu_score": 1.0,
+                "temperature": 0.7,
+                "max_tokens": 100,
+            },
+            {
+                "id": 3,
+                "dataset": "spanish_translation",
+                "input": "Translate 'please' to Spanish",
+                "output": "por favor",
+                "label": "por favor",
+                "accuracy": 1.0,
+                "bleu_score": 1.0,
+                "temperature": 0.7,
+                "max_tokens": 100,
+            },
+            {
+                "id": 4,
+                "dataset": "spanish_translation",
+                "input": "Translate 'good morning' to Spanish",
+                "output": "buenos dias",  # Missing accent - wrong answer
+                "label": "buenos días",
+                "accuracy": 0.0,
+                "bleu_score": 0.85,
+                "temperature": 0.7,
+                "max_tokens": 100,
+            },
+        ],
+    }
+
+    # Step 3: Upload your pre-scored results directly
+    print("\nUploading pre-scored results to Trismik...")
+    print("Metrics are already computed - just uploading to dashboard.\n")
+
+    run_id = upload_result(
+        run_result=my_pre_scored_results,
+        experiment_id="Pre-Scored-Results-Example",
+        project_id="TRISMIK_PROJECT_ID", # TODO: ADD YOUR TRISMIK PROJECT ID
+        dataset_name="spanish_translation",
+        hyperparameters={
+            "temperature": 0.7,
+            "max_tokens": 100,
+        },
+        metadata={
+            "description": "Results with pre-computed metrics from custom framework",
+            "source": "Custom evaluation tool",
+            "evaluation_date": "2025-01-15",
+        },
+        model_name="my-custom-translator-v2",
+    )
+
+    print(f"\nResults uploaded successfully with run_id: {run_id}")
+
+    # Add run_id to results for reference
+    my_pre_scored_results["run_id"] = run_id
+
+    pprint(my_pre_scored_results)
+    return my_pre_scored_results
+
+
+if __name__ == "__main__":
+    load_dotenv()
+    log_file = setup_logging(experiment_id="3-uploading_your_results", base_dir=Path(__file__).parent)
+    output_dir = Path(__file__).parent / "results"
+    output_dir.mkdir(exist_ok=True)
+    results_dict = main()
+    save_results_to_json(results_dict, output_dir, "3-uploading_your_results_output.json")

tutorials/examples/6-providers/aws/__init__.py

@@ -0,0 +1 @@
+"""Example usage of the AWS API with the scorebook library."""

tutorials/examples/6-providers/aws/batch_example.py

@@ -0,0 +1,219 @@
+"""
+AWS Bedrock Batch Inference Example.
+
+This example demonstrates how to leverage AWS Bedrock's Model Invocation Jobs for
+cost-effective, large-scale model evaluation using Scorebook. It uses Claude models
+for batch processing with automatic S3 upload/download and job management.
+
+This example requires AWS CLI to be configured with appropriate credentials and
+permissions for Bedrock and S3. Set up your AWS profile and ensure you have
+the necessary IAM roles configured.
+
+Prerequisites:
+- AWS CLI configured with appropriate profile
+- S3 bucket with proper permissions
+- IAM role for Bedrock execution with S3 access
+- Minimum 100 items for batch processing (AWS requirement)
+"""
+
+import json
+from pathlib import Path
+from typing import Any, List, Optional
+
+from dotenv import load_dotenv
+
+from scorebook import EvalDataset, InferencePipeline, evaluate
+from scorebook.inference.clients.bedrock import batch
+from scorebook.metrics.accuracy import Accuracy
+
+
+def main() -> None:
+    """Run the AWS Bedrock batch inference example."""
+    # Load environment variables from .env file for configuration
+    load_dotenv()
+
+    args = setup_arguments()
+
+    # Step 1: Load the evaluation dataset
+    dataset = EvalDataset.from_json(
+        "examples/example_datasets/dataset.json",
+        metrics=[Accuracy],
+        input="question",
+        label="answer",
+    )
+
+    # Ensure minimum batch size requirement (AWS Bedrock requires 100+ items)
+    if len(dataset) < 100:
+        # Cycle through items to reach 100 - default minimum size of AWS batch jobs
+        original_items = dataset.items
+        expanded_items: List[Any] = []
+        while len(expanded_items) < 100:
+            items_needed = 100 - len(expanded_items)
+            expanded_items.extend(original_items[: min(len(original_items), items_needed)])
+
+        # Create new dataset with expanded items
+        # Items already have "input" and "label" columns from the original dataset
+        dataset = EvalDataset.from_list(
+            name=dataset.name,
+            metrics=[Accuracy],
+            items=expanded_items,
+            input="input",
+            label="label",
+        )
+
+    # Step 2: Define the preprocessing function for AWS Bedrock Batch API
+    def preprocessor(input_value: str) -> list:
+        """Pre-process dataset inputs into AWS Bedrock Batch API format."""
+        # Create the batch API request messages format for Bedrock
+        messages = [
+            {
+                "role": "system",
+                "content": "Answer the question directly and concisely as a single word",
+            },
+            {"role": "user", "content": input_value},
+        ]
+
+        return messages
+
+    # Step 3: Define the postprocessing function
+    def postprocessor(response: str) -> str:
+        """Post-process AWS Bedrock batch response to extract the answer."""
+        # The batch function returns the message content directly
+        return response.strip()
+
+    # Step 4: Create the inference pipeline for batch processing
+    async def inference_function(items: list, **hyperparams: Any) -> Any:
+        return await batch(
+            items,
+            model=args.model,
+            aws_region=args.aws_region,
+            aws_profile=args.aws_profile,
+            bucket=args.bucket,
+            input_prefix=args.input_prefix,
+            output_prefix=args.output_prefix,
+            role_arn=args.role_arn,
+            **hyperparams,
+        )
+
+    inference_pipeline = InferencePipeline(
+        model=args.model or "claude-model",
+        preprocessor=preprocessor,
+        inference_function=inference_function,
+        postprocessor=postprocessor,
+    )
+
+    # Step 5: Run the batch evaluation
+    print("Running AWS Bedrock Batch API evaluation")
+    print(f"Model: {args.model or 'Not specified'}")
+    print(f"AWS Region: {args.aws_region or 'Not specified'}")
+    print(f"AWS Profile: {args.aws_profile or 'Not specified'}")
+    print(f"S3 Bucket: {args.bucket or 'Not specified'}")
+    print(f"Role ARN: {args.role_arn or 'Not specified'}")
+    print(f"Processing {len(dataset)} items using batch inference...")
+    print("Note: Batch processing may take several minutes to complete.")
+
+    results = evaluate(inference_pipeline, dataset, return_items=True, return_output=True)
+    print("\nBatch evaluation completed!")
+    print(results)
+
+    # Step 6: Save results to file
+    output_file = args.output_dir / "bedrock_batch_output.json"
+    with open(output_file, "w") as f:
+        json.dump(results, f, indent=4)
+        print(f"Results saved in {output_file}")
+
+
+# ============================================================================
+# Utility Functions
+# ============================================================================
+
+
+class Args:
+    """Simple container for parsed arguments."""
+
+    def __init__(self) -> None:
+        """Parse command line arguments."""
+        self.output_dir: Path = Path(".")  # Will be overridden in setup_arguments
+        self.model: Optional[str] = None
+        self.aws_region: Optional[str] = None
+        self.aws_profile: Optional[str] = None
+        self.bucket: Optional[str] = None
+        self.input_prefix: Optional[str] = None
+        self.output_prefix: Optional[str] = None
+        self.role_arn: Optional[str] = None
+
+
+def setup_arguments() -> Args:
+    """Parse command line arguments."""
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Run AWS Bedrock Batch API evaluation and save results."
+    )
+
+    # Required argument
+    parser.add_argument(
+        "--output-dir",
+        type=str,
+        required=False,
+        default=str(Path.cwd() / "results"),
+        help="Directory to save evaluation outputs (JSON).",
+    )
+
+    # All optional AWS parameters
+    parser.add_argument(
+        "--model",
+        type=str,
+        help="Bedrock model ID (e.g., 'us.anthropic.claude-3-5-sonnet-20241022-v2:0')",
+    )
+    parser.add_argument(
+        "--aws-region",
+        type=str,
+        help="AWS region for Bedrock and S3 operations",
+    )
+    parser.add_argument(
+        "--aws-profile",
+        type=str,
+        help="AWS profile name for authentication",
+    )
+    parser.add_argument(
+        "--bucket",
+        type=str,
+        help="S3 bucket name for input/output data",
+    )
+    parser.add_argument(
+        "--input-prefix",
+        type=str,
+        help="S3 prefix for input data (e.g., 'batch/input/')",
+    )
+    parser.add_argument(
+        "--output-prefix",
+        type=str,
+        help="S3 prefix for output data (e.g., 'batch/output/')",
+    )
+    parser.add_argument(
+        "--role-arn",
+        type=str,
+        help="IAM role ARN for Bedrock execution",
+    )
+
+    parsed_args = parser.parse_args()
+
+    # Create Args object and populate
+    args = Args()
+    args.output_dir = Path(parsed_args.output_dir)
+    args.output_dir.mkdir(parents=True, exist_ok=True)
+
+    args.model = parsed_args.model
+    args.aws_region = parsed_args.aws_region
+    args.aws_profile = parsed_args.aws_profile
+    args.bucket = parsed_args.bucket
+    args.input_prefix = parsed_args.input_prefix
+    args.output_prefix = parsed_args.output_prefix
+    args.role_arn = parsed_args.role_arn
+
+    return args
+
+
+if __name__ == "__main__":
+    main()

tutorials/examples/6-providers/portkey/__init__.py

@@ -0,0 +1 @@
+"""Example usage of the Portkey API with the scorebook library."""

tutorials/examples/6-providers/portkey/batch_example.py

@@ -0,0 +1,120 @@
+"""
+Portkey Cloud Batch Inference Example.
+
+This example demonstrates how to leverage Portkey's Batch API for cost-effective,
+large-scale model evaluation using Scorebook. The backend provider of choice for this example is
+OpenAI, but it's easy to adapt to any other provider.
+
+This example requires a portkey account linked to an OpenAI account and
+a portkey API key set in environment variable PORTKEY_API_KEY .
+
+Compare with messages_example.py to understand the differences
+between real-time and batch processing approaches.
+"""
+
+import json
+from pathlib import Path
+from typing import Any
+
+from dotenv import load_dotenv
+
+from scorebook import EvalDataset, InferencePipeline, evaluate
+from scorebook.inference.clients.portkey import batch
+from scorebook.metrics.accuracy import Accuracy
+
+
+def main() -> None:
+    """Run the Portkey batch inference example."""
+    # Load environment variables from .env file for API keys
+    load_dotenv()
+
+    output_dir, model_name = setup_arguments()
+
+    # Step 1: Load the evaluation dataset
+    dataset = EvalDataset.from_json(
+        "examples/example_datasets/dataset.json", label="answer", metrics=[Accuracy]
+    )
+
+    # Step 2: Define the preprocessing function for batch API
+    def preprocessor(eval_item: dict) -> list:
+        """Pre-process dataset items into Portkey Batch API format."""
+        prompt = eval_item["question"]
+
+        # Create the batch API request messages format
+        messages = [
+            {
+                "role": "system",
+                "content": "Answer the question directly and concisely as a single word",
+            },
+            {"role": "user", "content": prompt},
+        ]
+
+        return messages
+
+    # Step 3: Define the postprocessing function
+    def postprocessor(response: str) -> str:
+        """Post-process Portkey batch response to extract the answer."""
+        # The batch function returns the message content directly
+        return response.strip()
+
+    # Step 4: Create the inference pipeline for batch processing
+
+    async def inference_function(items: list, **hyperparams: Any) -> Any:  # noqa
+        return await batch(items, model=model_name, **hyperparams)
+
+    inference_pipeline = InferencePipeline(
+        model=model_name,
+        preprocessor=preprocessor,
+        inference_function=inference_function,
+        postprocessor=postprocessor,
+    )
+
+    # Step 5: Run the batch evaluation
+    print(f"Running Portkey Batch API evaluation with model: {model_name}")
+    print(f"Processing {len(dataset)} items using batch inference...")
+    print("Note: Batch processing may take several minutes to complete.")
+
+    # For demonstration, limit to 25 items
+    results = evaluate(inference_pipeline, dataset, item_limit=25, score_type="all")
+    print("\nBatch evaluation completed!")
+    print(results)
+
+    # Step 6: Save results to file
+    output_file = output_dir / "portkey_batch_output.json"
+    with open(output_file, "w") as f:
+        json.dump(results, f, indent=4)
+        print(f"Results saved in {output_file}")
+
+
+# ============================================================================
+# Utility Functions
+# ============================================================================
+
+
+def setup_arguments() -> tuple[Path, str]:
+    """Parse command line arguments."""
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Run Portkey Batch API evaluation and save results."
+    )
+    parser.add_argument(
+        "--output-dir",
+        type=str,
+        default=str(Path.cwd() / "results"),
+        help="Directory to save evaluation outputs (JSON).",
+    )
+    parser.add_argument(
+        "--model",
+        type=str,
+        required=True,
+        help="Model to use for batch inference via Portkey (e.g., @openai/gpt-4.1-mini)",
+    )
+    args = parser.parse_args()
+    output_dir = Path(args.output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+    return output_dir, str(args.model)
+
+
+if __name__ == "__main__":
+    main()

tutorials/examples/6-providers/portkey/messages_example.py

@@ -0,0 +1,121 @@
+"""
+Portkey Cloud Model Inference Example.
+
+This example demonstrates how to evaluate language models using Portkey's inference
+services with Scorebook for real-time API calls.
+
+Prerequisites: PORTKEY_API_KEY environment variable and active Portkey account.
+"""
+
+import json
+from pathlib import Path
+from typing import Any
+
+from dotenv import load_dotenv
+
+from scorebook import EvalDataset, InferencePipeline, evaluate
+from scorebook.inference.clients.portkey import responses
+from scorebook.metrics.accuracy import Accuracy
+
+
+def main() -> None:
+    """Run the Portkey inference example."""
+    # Load environment variables from .env file for API keys
+    load_dotenv()
+
+    output_dir, model_name = setup_arguments()
+
+    # Step 1: Load the evaluation dataset
+    # Create an EvalDataset from local JSON file
+    # - Uses 'answer' field as ground truth labels
+    # - Configures Accuracy metric for evaluation
+    # - Loads from examples/example_datasets/dataset.json
+    dataset = EvalDataset.from_json(
+        "examples/example_datasets/dataset.json", label="answer", metrics=[Accuracy]
+    )
+
+    # Step 2: Define the preprocessing function
+    # Convert raw dataset items into Portkey API-compatible format
+    # This function formats the question for the cloud model
+    def preprocessor(eval_item: dict) -> list:
+        """Pre-process dataset items into Portkey messages format."""
+        prompt = eval_item["question"]
+
+        # Create a system message with instructions for direct answers
+        system_prompt = """
+Answer the question directly and concisely.
+Do not provide lengthy explanations unless specifically asked.
+""".strip()
+
+        # Format as messages for Portkey API
+        return [{"role": "system", "content": system_prompt}, {"role": "user", "content": prompt}]
+
+    # Step 3: Define the postprocessing function
+    # Extract the final answer from Portkey API response
+    # Handles response parsing and returns the response text
+    def postprocessor(response: Any) -> str:
+        """Post-process Portkey response to extract the answer."""
+        return str(response.choices[0].message.content.strip())
+
+    # Step 4: Create the inference pipeline for cloud-based evaluation
+    # Combine preprocessing, Portkey API inference, and postprocessing
+    # Uses scorebook's built-in Portkey responses function for API calls
+
+    async def inference_function(items: list, **hyperparams: Any) -> Any:
+        return await responses(items, model=model_name, **hyperparams)
+
+    inference_pipeline = InferencePipeline(
+        model=model_name,
+        preprocessor=preprocessor,
+        inference_function=inference_function,
+        postprocessor=postprocessor,
+    )
+
+    # Step 5: Run the cloud-based evaluation
+    # Execute evaluation using Portkey API with the inference pipeline
+    # - Uses score_type="all" to get both aggregate and per-item results
+    # - Limits to 10 items for quick demonstration and cost control
+    print(f"Running Portkey evaluation with model: {model_name}")
+    print("Evaluating 10 items from local dataset...")
+
+    results = evaluate(inference_pipeline, dataset, item_limit=10, score_type="all")
+    print(results)
+
+    # Step 6: Save results to file
+    # Export evaluation results as JSON for later analysis
+    output_file = output_dir / "portkey_messages_output.json"
+    with open(output_file, "w") as f:
+        json.dump(results, f, indent=4)
+        print(f"Results saved in {output_file}")
+
+
+# ============================================================================
+# Utility Functions
+# ============================================================================
+
+
+def setup_arguments() -> tuple[Path, str]:
+    """Parse command line arguments."""
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Run Portkey evaluation and save results.")
+    parser.add_argument(
+        "--output-dir",
+        type=str,
+        default=str(Path.cwd() / "results"),
+        help="Directory to save evaluation outputs (JSON).",
+    )
+    parser.add_argument(
+        "--model",
+        type=str,
+        required=True,
+        help="Model to use for inference via Portkey (e.g., @openai/gpt-4.1-mini)",
+    )
+    args = parser.parse_args()
+    output_dir = Path(args.output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+    return output_dir, str(args.model)
+
+
+if __name__ == "__main__":
+    main()

tutorials/examples/6-providers/vertex/__init__.py

@@ -0,0 +1 @@
+"""Example usage of the Google Cloud Platform with the scorebook library."""