scorebook 0.0.14__py3-none-any.whl → 0.0.16__py3-none-any.whl

tutorials/examples/6-providers/aws/batch_example.py

@@ -0,0 +1,219 @@
+"""
+AWS Bedrock Batch Inference Example.
+
+This example demonstrates how to leverage AWS Bedrock's Model Invocation Jobs for
+cost-effective, large-scale model evaluation using Scorebook. It uses Claude models
+for batch processing with automatic S3 upload/download and job management.
+
+This example requires AWS CLI to be configured with appropriate credentials and
+permissions for Bedrock and S3. Set up your AWS profile and ensure you have
+the necessary IAM roles configured.
+
+Prerequisites:
+- AWS CLI configured with appropriate profile
+- S3 bucket with proper permissions
+- IAM role for Bedrock execution with S3 access
+- Minimum 100 items for batch processing (AWS requirement)
+"""
+
+import json
+from pathlib import Path
+from typing import Any, List, Optional
+
+from dotenv import load_dotenv
+
+from scorebook import EvalDataset, InferencePipeline, evaluate
+from scorebook.inference.clients.bedrock import batch
+from scorebook.metrics.accuracy import Accuracy
+
+
+def main() -> None:
+    """Run the AWS Bedrock batch inference example."""
+    # Load environment variables from .env file for configuration
+    load_dotenv()
+
+    args = setup_arguments()
+
+    # Step 1: Load the evaluation dataset
+    dataset = EvalDataset.from_json(
+        "examples/example_datasets/dataset.json",
+        metrics=[Accuracy],
+        input="question",
+        label="answer",
+    )
+
+    # Ensure minimum batch size requirement (AWS Bedrock requires 100+ items)
+    if len(dataset) < 100:
+        # Cycle through items to reach 100 - default minimum size of AWS batch jobs
+        original_items = dataset.items
+        expanded_items: List[Any] = []
+        while len(expanded_items) < 100:
+            items_needed = 100 - len(expanded_items)
+            expanded_items.extend(original_items[: min(len(original_items), items_needed)])
+
+        # Create new dataset with expanded items
+        # Items already have "input" and "label" columns from the original dataset
+        dataset = EvalDataset.from_list(
+            name=dataset.name,
+            metrics=[Accuracy],
+            items=expanded_items,
+            input="input",
+            label="label",
+        )
+
+    # Step 2: Define the preprocessing function for AWS Bedrock Batch API
+    def preprocessor(input_value: str) -> list:
+        """Pre-process dataset inputs into AWS Bedrock Batch API format."""
+        # Create the batch API request messages format for Bedrock
+        messages = [
+            {
+                "role": "system",
+                "content": "Answer the question directly and concisely as a single word",
+            },
+            {"role": "user", "content": input_value},
+        ]
+
+        return messages
+
+    # Step 3: Define the postprocessing function
+    def postprocessor(response: str) -> str:
+        """Post-process AWS Bedrock batch response to extract the answer."""
+        # The batch function returns the message content directly
+        return response.strip()
+
+    # Step 4: Create the inference pipeline for batch processing
+    async def inference_function(items: list, **hyperparams: Any) -> Any:
+        return await batch(
+            items,
+            model=args.model,
+            aws_region=args.aws_region,
+            aws_profile=args.aws_profile,
+            bucket=args.bucket,
+            input_prefix=args.input_prefix,
+            output_prefix=args.output_prefix,
+            role_arn=args.role_arn,
+            **hyperparams,
+        )
+
+    inference_pipeline = InferencePipeline(
+        model=args.model or "claude-model",
+        preprocessor=preprocessor,
+        inference_function=inference_function,
+        postprocessor=postprocessor,
+    )
+
+    # Step 5: Run the batch evaluation
+    print("Running AWS Bedrock Batch API evaluation")
+    print(f"Model: {args.model or 'Not specified'}")
+    print(f"AWS Region: {args.aws_region or 'Not specified'}")
+    print(f"AWS Profile: {args.aws_profile or 'Not specified'}")
+    print(f"S3 Bucket: {args.bucket or 'Not specified'}")
+    print(f"Role ARN: {args.role_arn or 'Not specified'}")
+    print(f"Processing {len(dataset)} items using batch inference...")
+    print("Note: Batch processing may take several minutes to complete.")
+
+    results = evaluate(inference_pipeline, dataset, return_items=True, return_output=True)
+    print("\nBatch evaluation completed!")
+    print(results)
+
+    # Step 6: Save results to file
+    output_file = args.output_dir / "bedrock_batch_output.json"
+    with open(output_file, "w") as f:
+        json.dump(results, f, indent=4)
+        print(f"Results saved in {output_file}")
+
+
+# ============================================================================
+# Utility Functions
+# ============================================================================
+
+
+class Args:
+    """Simple container for parsed arguments."""
+
+    def __init__(self) -> None:
+        """Parse command line arguments."""
+        self.output_dir: Path = Path(".")  # Will be overridden in setup_arguments
+        self.model: Optional[str] = None
+        self.aws_region: Optional[str] = None
+        self.aws_profile: Optional[str] = None
+        self.bucket: Optional[str] = None
+        self.input_prefix: Optional[str] = None
+        self.output_prefix: Optional[str] = None
+        self.role_arn: Optional[str] = None
+
+
+def setup_arguments() -> Args:
+    """Parse command line arguments."""
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Run AWS Bedrock Batch API evaluation and save results."
+    )
+
+    # Required argument
+    parser.add_argument(
+        "--output-dir",
+        type=str,
+        required=False,
+        default=str(Path.cwd() / "results"),
+        help="Directory to save evaluation outputs (JSON).",
+    )
+
+    # All optional AWS parameters
+    parser.add_argument(
+        "--model",
+        type=str,
+        help="Bedrock model ID (e.g., 'us.anthropic.claude-3-5-sonnet-20241022-v2:0')",
+    )
+    parser.add_argument(
+        "--aws-region",
+        type=str,
+        help="AWS region for Bedrock and S3 operations",
+    )
+    parser.add_argument(
+        "--aws-profile",
+        type=str,
+        help="AWS profile name for authentication",
+    )
+    parser.add_argument(
+        "--bucket",
+        type=str,
+        help="S3 bucket name for input/output data",
+    )
+    parser.add_argument(
+        "--input-prefix",
+        type=str,
+        help="S3 prefix for input data (e.g., 'batch/input/')",
+    )
+    parser.add_argument(
+        "--output-prefix",
+        type=str,
+        help="S3 prefix for output data (e.g., 'batch/output/')",
+    )
+    parser.add_argument(
+        "--role-arn",
+        type=str,
+        help="IAM role ARN for Bedrock execution",
+    )
+
+    parsed_args = parser.parse_args()
+
+    # Create Args object and populate
+    args = Args()
+    args.output_dir = Path(parsed_args.output_dir)
+    args.output_dir.mkdir(parents=True, exist_ok=True)
+
+    args.model = parsed_args.model
+    args.aws_region = parsed_args.aws_region
+    args.aws_profile = parsed_args.aws_profile
+    args.bucket = parsed_args.bucket
+    args.input_prefix = parsed_args.input_prefix
+    args.output_prefix = parsed_args.output_prefix
+    args.role_arn = parsed_args.role_arn
+
+    return args
+
+
+if __name__ == "__main__":
+    main()

tutorials/examples/6-providers/portkey/__init__.py

@@ -0,0 +1 @@
+"""Example usage of the Portkey API with the scorebook library."""

tutorials/examples/6-providers/portkey/batch_example.py

@@ -0,0 +1,120 @@
+"""
+Portkey Cloud Batch Inference Example.
+
+This example demonstrates how to leverage Portkey's Batch API for cost-effective,
+large-scale model evaluation using Scorebook. The backend provider of choice for this example is
+OpenAI, but it's easy to adapt to any other provider.
+
+This example requires a portkey account linked to an OpenAI account and
+a portkey API key set in environment variable PORTKEY_API_KEY .
+
+Compare with messages_example.py to understand the differences
+between real-time and batch processing approaches.
+"""
+
+import json
+from pathlib import Path
+from typing import Any
+
+from dotenv import load_dotenv
+
+from scorebook import EvalDataset, InferencePipeline, evaluate
+from scorebook.inference.clients.portkey import batch
+from scorebook.metrics.accuracy import Accuracy
+
+
+def main() -> None:
+    """Run the Portkey batch inference example."""
+    # Load environment variables from .env file for API keys
+    load_dotenv()
+
+    output_dir, model_name = setup_arguments()
+
+    # Step 1: Load the evaluation dataset
+    dataset = EvalDataset.from_json(
+        "examples/example_datasets/dataset.json", label="answer", metrics=[Accuracy]
+    )
+
+    # Step 2: Define the preprocessing function for batch API
+    def preprocessor(eval_item: dict) -> list:
+        """Pre-process dataset items into Portkey Batch API format."""
+        prompt = eval_item["question"]
+
+        # Create the batch API request messages format
+        messages = [
+            {
+                "role": "system",
+                "content": "Answer the question directly and concisely as a single word",
+            },
+            {"role": "user", "content": prompt},
+        ]
+
+        return messages
+
+    # Step 3: Define the postprocessing function
+    def postprocessor(response: str) -> str:
+        """Post-process Portkey batch response to extract the answer."""
+        # The batch function returns the message content directly
+        return response.strip()
+
+    # Step 4: Create the inference pipeline for batch processing
+
+    async def inference_function(items: list, **hyperparams: Any) -> Any:  # noqa
+        return await batch(items, model=model_name, **hyperparams)
+
+    inference_pipeline = InferencePipeline(
+        model=model_name,
+        preprocessor=preprocessor,
+        inference_function=inference_function,
+        postprocessor=postprocessor,
+    )
+
+    # Step 5: Run the batch evaluation
+    print(f"Running Portkey Batch API evaluation with model: {model_name}")
+    print(f"Processing {len(dataset)} items using batch inference...")
+    print("Note: Batch processing may take several minutes to complete.")
+
+    # For demonstration, limit to 25 items
+    results = evaluate(inference_pipeline, dataset, item_limit=25, score_type="all")
+    print("\nBatch evaluation completed!")
+    print(results)
+
+    # Step 6: Save results to file
+    output_file = output_dir / "portkey_batch_output.json"
+    with open(output_file, "w") as f:
+        json.dump(results, f, indent=4)
+        print(f"Results saved in {output_file}")
+
+
+# ============================================================================
+# Utility Functions
+# ============================================================================
+
+
+def setup_arguments() -> tuple[Path, str]:
+    """Parse command line arguments."""
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Run Portkey Batch API evaluation and save results."
+    )
+    parser.add_argument(
+        "--output-dir",
+        type=str,
+        default=str(Path.cwd() / "results"),
+        help="Directory to save evaluation outputs (JSON).",
+    )
+    parser.add_argument(
+        "--model",
+        type=str,
+        required=True,
+        help="Model to use for batch inference via Portkey (e.g., @openai/gpt-4.1-mini)",
+    )
+    args = parser.parse_args()
+    output_dir = Path(args.output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+    return output_dir, str(args.model)
+
+
+if __name__ == "__main__":
+    main()

tutorials/examples/6-providers/portkey/messages_example.py

@@ -0,0 +1,121 @@
+"""
+Portkey Cloud Model Inference Example.
+
+This example demonstrates how to evaluate language models using Portkey's inference
+services with Scorebook for real-time API calls.
+
+Prerequisites: PORTKEY_API_KEY environment variable and active Portkey account.
+"""
+
+import json
+from pathlib import Path
+from typing import Any
+
+from dotenv import load_dotenv
+
+from scorebook import EvalDataset, InferencePipeline, evaluate
+from scorebook.inference.clients.portkey import responses
+from scorebook.metrics.accuracy import Accuracy
+
+
+def main() -> None:
+    """Run the Portkey inference example."""
+    # Load environment variables from .env file for API keys
+    load_dotenv()
+
+    output_dir, model_name = setup_arguments()
+
+    # Step 1: Load the evaluation dataset
+    # Create an EvalDataset from local JSON file
+    # - Uses 'answer' field as ground truth labels
+    # - Configures Accuracy metric for evaluation
+    # - Loads from examples/example_datasets/dataset.json
+    dataset = EvalDataset.from_json(
+        "examples/example_datasets/dataset.json", label="answer", metrics=[Accuracy]
+    )
+
+    # Step 2: Define the preprocessing function
+    # Convert raw dataset items into Portkey API-compatible format
+    # This function formats the question for the cloud model
+    def preprocessor(eval_item: dict) -> list:
+        """Pre-process dataset items into Portkey messages format."""
+        prompt = eval_item["question"]
+
+        # Create a system message with instructions for direct answers
+        system_prompt = """
+Answer the question directly and concisely.
+Do not provide lengthy explanations unless specifically asked.
+""".strip()
+
+        # Format as messages for Portkey API
+        return [{"role": "system", "content": system_prompt}, {"role": "user", "content": prompt}]
+
+    # Step 3: Define the postprocessing function
+    # Extract the final answer from Portkey API response
+    # Handles response parsing and returns the response text
+    def postprocessor(response: Any) -> str:
+        """Post-process Portkey response to extract the answer."""
+        return str(response.choices[0].message.content.strip())
+
+    # Step 4: Create the inference pipeline for cloud-based evaluation
+    # Combine preprocessing, Portkey API inference, and postprocessing
+    # Uses scorebook's built-in Portkey responses function for API calls
+
+    async def inference_function(items: list, **hyperparams: Any) -> Any:
+        return await responses(items, model=model_name, **hyperparams)
+
+    inference_pipeline = InferencePipeline(
+        model=model_name,
+        preprocessor=preprocessor,
+        inference_function=inference_function,
+        postprocessor=postprocessor,
+    )
+
+    # Step 5: Run the cloud-based evaluation
+    # Execute evaluation using Portkey API with the inference pipeline
+    # - Uses score_type="all" to get both aggregate and per-item results
+    # - Limits to 10 items for quick demonstration and cost control
+    print(f"Running Portkey evaluation with model: {model_name}")
+    print("Evaluating 10 items from local dataset...")
+
+    results = evaluate(inference_pipeline, dataset, item_limit=10, score_type="all")
+    print(results)
+
+    # Step 6: Save results to file
+    # Export evaluation results as JSON for later analysis
+    output_file = output_dir / "portkey_messages_output.json"
+    with open(output_file, "w") as f:
+        json.dump(results, f, indent=4)
+        print(f"Results saved in {output_file}")
+
+
+# ============================================================================
+# Utility Functions
+# ============================================================================
+
+
+def setup_arguments() -> tuple[Path, str]:
+    """Parse command line arguments."""
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Run Portkey evaluation and save results.")
+    parser.add_argument(
+        "--output-dir",
+        type=str,
+        default=str(Path.cwd() / "results"),
+        help="Directory to save evaluation outputs (JSON).",
+    )
+    parser.add_argument(
+        "--model",
+        type=str,
+        required=True,
+        help="Model to use for inference via Portkey (e.g., @openai/gpt-4.1-mini)",
+    )
+    args = parser.parse_args()
+    output_dir = Path(args.output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+    return output_dir, str(args.model)
+
+
+if __name__ == "__main__":
+    main()

tutorials/examples/6-providers/vertex/__init__.py

@@ -0,0 +1 @@
+"""Example usage of the Google Cloud Platform with the scorebook library."""

tutorials/examples/6-providers/vertex/batch_example.py

@@ -0,0 +1,166 @@
+"""
+Google Cloud Vertex AI Batch Inference Example.
+
+This example demonstrates how to leverage Google Cloud Vertex AI's Batch API for
+cost-effective, large-scale model evaluation using Scorebook. It uses Gemini models
+for batch processing with automatic GCS upload/download and job management.
+
+This example requires Google Cloud SDK (gsutil) to be installed and authenticated,
+and a Google Cloud project with Vertex AI enabled. Set the project ID in the
+GOOGLE_CLOUD_PROJECT environment variable or pass it as a command line argument.
+
+Compare with the Portkey batch example to understand the differences
+between different cloud providers' batch processing approaches.
+"""
+
+import json
+import os
+from pathlib import Path
+from typing import Any
+
+from dotenv import load_dotenv
+
+from scorebook import EvalDataset, InferencePipeline, evaluate
+from scorebook.inference.clients.vertex import batch
+from scorebook.metrics.accuracy import Accuracy
+
+
+def main() -> None:
+    """Run the Vertex AI batch inference example."""
+    # Load environment variables from .env file for configuration
+    load_dotenv()
+
+    output_dir, model_name, input_bucket, output_bucket, project_id = setup_arguments()
+
+    # Step 1: Load the evaluation dataset
+    dataset = EvalDataset.from_json(
+        "examples/example_datasets/dataset.json", label="answer", metrics=[Accuracy]
+    )
+
+    # Step 2: Define the preprocessing function for Vertex AI Batch API
+    def preprocessor(eval_item: dict) -> list:
+        """Pre-process dataset items into Vertex AI Batch API format."""
+        prompt = eval_item["question"]
+
+        # Create the batch API request messages format for Vertex AI
+        messages = [
+            {
+                "role": "system",
+                "content": "Answer the question directly and concisely as a single word",
+            },
+            {"role": "user", "content": prompt},
+        ]
+
+        return messages
+
+    # Step 3: Define the postprocessing function
+    def postprocessor(response: str) -> str:
+        """Post-process Vertex AI batch response to extract the answer."""
+        # The batch function returns the message content directly
+        return response.strip()
+
+    # Step 4: Create the inference pipeline for batch processing
+
+    async def inference_function(items: list, **hyperparams: Any) -> Any:  # noqa
+        return await batch(
+            items,
+            model=model_name,
+            project_id=project_id,
+            input_bucket=input_bucket,
+            output_bucket=output_bucket,
+            **hyperparams,
+        )
+
+    inference_pipeline = InferencePipeline(
+        model=model_name,
+        preprocessor=preprocessor,
+        inference_function=inference_function,
+        postprocessor=postprocessor,
+    )
+
+    # Step 5: Run the batch evaluation
+    print(f"Running Vertex AI Batch API evaluation with model: {model_name}")
+    print(f"Project ID: {project_id}")
+    print(f"Input bucket: {input_bucket}")
+    print(f"Output bucket: {output_bucket}")
+    print(f"Processing {len(dataset)} items using batch inference...")
+    print("Note: Batch processing may take several minutes to complete.")
+
+    # For demonstration, limit to 25 items
+    results = evaluate(inference_pipeline, dataset, item_limit=25, score_type="all")
+    print("\nBatch evaluation completed!")
+    print(results)
+
+    # Step 6: Save results to file
+    output_file = output_dir / "vertex_batch_output.json"
+    with open(output_file, "w") as f:
+        json.dump(results, f, indent=4)
+        print(f"Results saved in {output_file}")
+
+
+# ============================================================================
+# Utility Functions
+# ============================================================================
+
+
+def setup_arguments() -> tuple[Path, str, str, str, str]:
+    """Parse command line arguments."""
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Run Vertex AI Batch API evaluation and save results."
+    )
+    parser.add_argument(
+        "--output-dir",
+        type=str,
+        default=str(Path.cwd() / "results"),
+        help="Directory to save evaluation outputs (JSON).",
+    )
+    parser.add_argument(
+        "--model",
+        type=str,
+        required=True,
+        help="Gemini model to use for batch inference (e.g., gemini-2.0-flash-001)",
+    )
+    parser.add_argument(
+        "--input-bucket",
+        type=str,
+        required=True,
+        help="GCS bucket name for input data (without gs:// prefix)",
+    )
+    parser.add_argument(
+        "--output-bucket",
+        type=str,
+        required=True,
+        help="GCS bucket name for output data (without gs:// prefix)",
+    )
+    parser.add_argument(
+        "--project-id",
+        type=str,
+        help="Google Cloud Project ID (defaults to GOOGLE_CLOUD_PROJECT env var)",
+    )
+
+    args = parser.parse_args()
+
+    output_dir = Path(args.output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    # Handle project ID fallback
+    project_id = args.project_id or os.getenv("GOOGLE_CLOUD_PROJECT")
+    if not project_id:
+        raise ValueError(
+            "Project ID must be provided via --project-id or "
+            "GOOGLE_CLOUD_PROJECT environment variable"
+        )
+
+    return (
+        output_dir,
+        str(args.model),
+        str(args.input_bucket),
+        str(args.output_bucket),
+        str(project_id),
+    )
+
+
+if __name__ == "__main__":
+    main()