scorebook 0.0.13__py3-none-any.whl → 0.0.15__py3-none-any.whl

tutorials/examples/2-evaluate/4-evaluating_cloud_models_with_batching.py

@@ -0,0 +1,170 @@
+"""Tutorials - Evaluate - Example 4 - Evaluating Cloud Models with Batching."""
+
+import asyncio
+import json
+import tempfile
+from pathlib import Path
+from pprint import pprint
+from typing import Any, List
+
+from dotenv import load_dotenv
+from openai import AsyncOpenAI
+
+from tutorials.utils import save_results_to_json, setup_logging
+
+from scorebook import EvalDataset, evaluate_async
+
+
+async def main() -> Any:
+    """Run evaluation using OpenAI's Batch API.
+
+    This example demonstrates how to use OpenAI's Batch API for cost-effective,
+    large-scale model evaluation. The Batch API offers 50% cost savings compared
+    to standard API calls, with results typically delivered within 24 hours.
+
+    Prerequisites:
+        - OpenAI API key set in environment variable OPENAI_API_KEY
+    """
+
+    # Initialize OpenAI client
+    client = AsyncOpenAI()
+    model_name = "gpt-4o-mini"
+
+    # Define an async batch inference function
+    async def inference(inputs: List[Any], **hyperparameters: Any) -> List[Any]:
+        """Process inputs through OpenAI's Batch API.
+
+        Args:
+            inputs: Input values from an EvalDataset.
+            hyperparameters: Model hyperparameters including system_message and temperature.
+
+        Returns:
+            List of model outputs for all inputs.
+        """
+        # Step 1: Create batch requests in JSONL format
+        batch_requests = []
+        for idx, input_val in enumerate(inputs):
+            request = {
+                "custom_id": f"request-{idx}",
+                "method": "POST",
+                "url": "/v1/chat/completions",
+                "body": {
+                    "model": model_name,
+                    "messages": [
+                        {
+                            "role": "system",
+                            "content": hyperparameters.get(
+                                "system_message", "You are a helpful assistant."
+                            ),
+                        },
+                        {"role": "user", "content": str(input_val)},
+                    ],
+                    "temperature": hyperparameters.get("temperature", 0.7),
+                },
+            }
+            batch_requests.append(request)
+
+        # Step 2: Write requests to a temporary JSONL file
+        with tempfile.NamedTemporaryFile(mode="w", suffix=".jsonl", delete=False) as f:
+            for request in batch_requests:
+                f.write(json.dumps(request) + "\n")
+            temp_file_path = f.name
+
+        try:
+            # Step 3: Upload the batch file
+            print(f"Uploading batch file with {len(inputs)} requests...")
+            with open(temp_file_path, "rb") as f:
+                batch_file = await client.files.create(file=f, purpose="batch")
+
+            # Step 4: Create the batch job
+            print(f"Creating batch job...")
+            batch_job = await client.batches.create(
+                input_file_id=batch_file.id,
+                endpoint="/v1/chat/completions",
+                completion_window="24h",
+            )
+
+            # Step 5: Wait for batch completion (with polling)
+            print(f"Waiting for batch to complete (ID: {batch_job.id})...")
+            while batch_job.status not in ["completed", "failed", "cancelled"]:
+                await asyncio.sleep(10)  # Poll every 10 seconds
+                batch_job = await client.batches.retrieve(batch_job.id)
+                print(f"Status: {batch_job.status}")
+
+            if batch_job.status != "completed":
+                raise Exception(f"Batch job failed with status: {batch_job.status}")
+
+            # Step 6: Download and parse results
+            print("Batch completed! Downloading results...")
+            result_file_id = batch_job.output_file_id
+            result_content = await client.files.content(result_file_id)
+            result_text = result_content.text
+
+            # Step 7: Parse results and extract outputs
+            results_by_id = {}
+            for line in result_text.strip().split("\n"):
+                result = json.loads(line)
+                custom_id = result["custom_id"]
+                try:
+                    output = result["response"]["body"]["choices"][0]["message"]["content"]
+                    results_by_id[custom_id] = output.strip()
+                except (KeyError, IndexError):
+                    results_by_id[custom_id] = "Error: Failed to extract response"
+
+            # Step 8: Return outputs in original order
+            outputs = []
+            for idx in range(len(inputs)):
+                custom_id = f"request-{idx}"
+                outputs.append(results_by_id.get(custom_id, "Error: Missing response"))
+
+            return outputs
+
+        finally:
+            # Clean up the temporary file
+            Path(temp_file_path).unlink(missing_ok=True)
+
+    # Create a list of evaluation items
+    evaluation_items = [
+        {"question": "What is 2 + 2?", "answer": "4"},
+        {"question": "What is the capital of France?", "answer": "Paris"},
+        {"question": "Who wrote Romeo and Juliet?", "answer": "William Shakespeare"},
+    ]
+
+    # Create an evaluation dataset
+    evaluation_dataset = EvalDataset.from_list(
+        name="basic_questions",
+        metrics="accuracy",
+        items=evaluation_items,
+        input="question",
+        label="answer",
+    )
+
+    print(f"\nRunning OpenAI Batch API evaluation with model: {model_name}")
+    print("Note: Batch processing may take several minutes to complete.\n")
+
+    # Run evaluation
+    results = await evaluate_async(
+        inference,
+        evaluation_dataset,
+        hyperparameters={
+            "temperature": 0.7,
+            "system_message": "Answer the question directly and concisely",
+        },
+        return_aggregates=True,
+        return_items=True,
+        return_output=True,
+        upload_results=False,
+    )
+
+    print("\nBatch evaluation completed:\n")
+    pprint(results)
+    return results
+
+
+if __name__ == "__main__":
+    load_dotenv()
+    log_file = setup_logging(experiment_id="4-evaluating_cloud_models_with_batching", base_dir=Path(__file__).parent)
+    output_dir = Path(__file__).parent / "results"
+    output_dir.mkdir(exist_ok=True)
+    results_dict = asyncio.run(main())
+    save_results_to_json(results_dict, output_dir, "4-evaluating_cloud_models_with_batching_output.json")

tutorials/examples/2-evaluate/5-hyperparameter_sweeps.py

@@ -0,0 +1,122 @@
+"""Tutorials - Evaluate - Example 5 - Hyperparameter Sweeps."""
+
+from pathlib import Path
+from pprint import pprint
+from typing import Any, List
+
+import transformers
+from dotenv import load_dotenv
+
+from tutorials.utils import save_results_to_json, setup_logging
+
+from scorebook import EvalDataset, evaluate
+
+
+def main() -> Any:
+    """Run a Scorebook evaluation with a hyperparameter sweep.
+
+    This example demonstrates how Scorebook can automatically test multiple
+    hyperparameter configurations in a single evaluation.
+
+    How Hyperparameter Sweeping Works:
+    - Define hyperparameters with lists of values to test
+    - Scorebook generates all possible combinations (Cartesian product)
+    - Each configuration is evaluated separately on the same dataset
+
+    Example Hyperparameters:
+    - system_message: "Answer the question directly and concisely." (1 value)
+    - temperature: [0.6, 0.7, 0.8] (3 values)
+    - top_p: [0.7, 0.8, 0.9] (3 values)
+    - top_k: [10, 20, 30] (3 values)
+
+    Total configurations = 1 × 3 × 3 × 3 = 27 hyperparameter configurations
+    """
+
+    # Initialize HuggingFace model pipeline
+    model_name = "microsoft/Phi-4-mini-instruct"
+    pipeline = transformers.pipeline(
+        "text-generation",
+        model=model_name,
+        model_kwargs={"torch_dtype": "auto"},
+        device_map="auto",
+    )
+
+    # Define an inference function
+    def inference(inputs: List[Any], **hyperparameters: Any) -> List[Any]:
+        """Process inputs through the model.
+
+        Args:
+            inputs: Input values from an EvalDataset.
+            hyperparameters: Model hyperparameters including system_message, temperature, top_p, top_k.
+
+        Returns:
+            List of model outputs for all inputs.
+        """
+        outputs = []
+        for input_val in inputs:
+            # Preprocess: Build messages
+            messages = [
+                {"role": "system", "content": hyperparameters["system_message"]},
+                {"role": "user", "content": str(input_val)},
+            ]
+
+            # Run inference
+            result = pipeline(
+                messages,
+                temperature=hyperparameters["temperature"],
+                top_p=hyperparameters.get("top_p"),
+                top_k=hyperparameters.get("top_k"),
+            )
+
+            # Postprocess: Extract the answer
+            output = str(result[0]["generated_text"][-1]["content"])
+            outputs.append(output)
+
+        return outputs
+
+    # Create a list of evaluation items
+    evaluation_items = [
+        {"question": "What is 2 + 2?", "answer": "4"},
+        {"question": "What is the capital of France?", "answer": "Paris"},
+        {"question": "Who wrote Romeo and Juliet?", "answer": "William Shakespeare"},
+    ]
+
+    # Create an evaluation dataset
+    evaluation_dataset = EvalDataset.from_list(
+        name="basic_questions",
+        metrics="accuracy",
+        items=evaluation_items,
+        input="question",
+    label="answer",
+    )
+
+    # Define hyperparameters with lists of values to create a sweep
+    hyperparameters = {
+        "system_message": "Answer the question directly and concisely.",
+        "temperature": [0.6, 0.7, 0.8],
+        "top_p": [0.7, 0.8, 0.9],
+        "top_k": [10, 20, 30],
+    }
+
+    # Run evaluation across all hyperparameter combinations
+    results = evaluate(
+        inference,
+        evaluation_dataset,
+        hyperparameters=hyperparameters,
+        return_aggregates=True,
+        return_items=True,
+        return_output=True,
+        upload_results=False,
+    )
+
+    pprint(results)
+    return results
+
+
+if __name__ == "__main__":
+    load_dotenv()
+    log_file = setup_logging(experiment_id="5-hyperparameter_sweeps", base_dir=Path(__file__).parent)
+    output_dir = Path(__file__).parent / "results"
+    output_dir.mkdir(exist_ok=True)
+    results_dict = main()
+    save_results_to_json(results_dict, output_dir, "5-hyperparameter_sweeps_output.json")

tutorials/examples/2-evaluate/6-inference_pipelines.py

@@ -0,0 +1,141 @@
+"""Tutorials - Evaluate - Example 6 - Inference Pipelines."""
+
+from pprint import pprint
+from typing import Any, List
+
+import transformers
+from dotenv import load_dotenv
+from pathlib import Path
+
+from tutorials.utils import save_results_to_json, setup_logging
+
+from scorebook import EvalDataset, InferencePipeline, evaluate
+
+
+def main() -> Any:
+    """Run a simple Scorebook evaluation using an InferencePipeline.
+
+    This example demonstrates how to use Scorebook's InferencePipeline in evaluations.
+
+    Inference pipelines separate the evaluation workflow into three distinct stages:
+        1. Pre-processing: Convert raw dataset items into model-ready input format
+        2. Inference: Execute model predictions on preprocessed data
+        3. Post-processing: Extract final answers from raw model outputs
+
+    These stages can be encapsulated in reusable functions and used to create pipelines.
+    An inference pipeline can be passed into the evaluate function's inference parameter.
+    """
+
+    # === Pre-Processing ===
+
+    # The preprocessor function is responsible for mapping items in an Eval Dataset to model inputs
+    def preprocessor(input_value: str, **hyperparameter_config: Any) -> List[Any]:
+        """Convert an evaluation input to a valid model input.
+
+        Args:
+            input_value: The input value from the dataset.
+            hyperparameter_config: Model hyperparameters.
+
+        Returns:
+            A structured representation of an evaluation item for model input.
+        """
+        messages = [
+            {
+                "role": "system",
+                "content": hyperparameter_config["system_message"],
+            },
+            {"role": "user", "content": input_value},
+        ]
+
+        return messages
+
+    # === Inference ===
+
+    pipeline = transformers.pipeline(
+        "text-generation",
+        model="microsoft/Phi-4-mini-instruct",
+        model_kwargs={"torch_dtype": "auto"},
+        device_map="auto",
+    )
+
+    # An inference function for an InferencePipeline that returns a list of raw outputs
+    def inference(preprocessed_items: List[Any], **hyperparameter_config: Any) -> List[Any]:
+        """Run model inference on preprocessed eval items.
+
+        Args:
+            preprocessed_items: The list of evaluation items for an EvalDataset.
+            hyperparameter_config: Model hyperparameters.
+
+        Returns:
+            A list of model outputs for an EvalDataset.
+        """
+        return [
+            pipeline(model_input, temperature=hyperparameter_config["temperature"])
+            for model_input in preprocessed_items
+        ]
+
+    # === Post-Processing ===
+
+    # The postprocessor function parses model output for metric scoring
+    def postprocessor(model_output: Any, **hyperparameter_config: Any) -> str:
+        """Extract the final parsed answer from the model output.
+
+        Args:
+            model_output: An evaluation item from an EvalDataset.
+            hyperparameter_config: Model hyperparameters.
+
+        Returns:
+            Parsed answer from the model output to be used for scoring.
+        """
+        return str(model_output[0]["generated_text"][-1]["content"])
+
+    # === Evaluation With An InferencePipeline ===
+
+    # Step 1: Create an inference pipeline, using the 3 functions defined
+    inference_pipeline = InferencePipeline(
+        model="microsoft/Phi-4-mini-instruct",
+        preprocessor=preprocessor,
+        inference_function=inference,
+        postprocessor=postprocessor,
+    )
+
+    # Step 2: Create a list of evaluation items
+    evaluation_items = [
+        {"question": "What is 2 + 2?", "answer": "4"},
+        {"question": "What is the capital of France?", "answer": "Paris"},
+        {"question": "Who wrote Romeo and Juliet?", "answer": "William Shakespeare"},
+    ]
+
+    # Create an evaluation dataset
+    evaluation_dataset = EvalDataset.from_list(
+        name="basic_questions",
+        metrics="accuracy",
+        items=evaluation_items,
+        input="question",
+        label="answer",
+    )
+
+    # Step 3: Run the evaluation using the inference pipeline and dataset
+    results = evaluate(
+        inference_pipeline,
+        evaluation_dataset,
+        hyperparameters={
+            "temperature": 0.7,
+            "system_message": "Answer the question directly and concisely.",
+        },
+        return_items=True,     # Enable to include results for individual items in the dict returned.
+        return_output=True,    # Enable to include the model's output for individual items.
+        upload_results=False,  # Disable uploading for this example
+    )
+
+    pprint(results)
+    return results
+
+
+if __name__ == "__main__":
+    load_dotenv()
+    log_file = setup_logging(experiment_id="6-inference_output", base_dir=Path(__file__).parent)
+    output_dir = Path(__file__).parent / "results"
+    output_dir.mkdir(exist_ok=True)
+    results_dict = main()
+    save_results_to_json(results_dict, output_dir, "6-inference_output_output.json")

tutorials/examples/3-evaluation_datasets/1-evaluation_datasets_from_files.py

@@ -0,0 +1,110 @@
+"""Tutorials - Evaluation Datasets - Example 1 - Loading Datasets from Files."""
+
+from pathlib import Path
+from pprint import pprint
+from typing import Any, List
+
+import transformers
+from dotenv import load_dotenv
+
+from tutorials.utils import save_results_to_json, setup_logging
+
+from scorebook import EvalDataset, evaluate
+
+
+def main() -> Any:
+    """Run evaluations using datasets loaded from local files.
+
+    This example demonstrates how to load evaluation datasets from files:
+        - from_json: Load datasets from JSON files
+        - from_csv: Load datasets from CSV files
+
+    Both methods support loading data from local files with custom field mappings.
+    """
+
+    # Initialize HuggingFace model pipeline
+    model_name = "microsoft/Phi-4-mini-instruct"
+    pipeline = transformers.pipeline(
+        "text-generation",
+        model=model_name,
+        model_kwargs={"torch_dtype": "auto"},
+        device_map="auto",
+    )
+
+    # Define an inference function
+    def inference(inputs: List[Any], **hyperparameters: Any) -> List[Any]:
+        """Process inputs through the model.
+
+        Args:
+            inputs: Input values from an EvalDataset.
+            hyperparameters: Model hyperparameters.
+
+        Returns:
+            List of model outputs for all inputs.
+        """
+        outputs = []
+        for input_val in inputs:
+            # Build messages
+            messages = [
+                {
+                    "role": "system",
+                    "content": "Answer the question directly and concisely. Provide only the answer, no additional context or text.",
+                },
+                {"role": "user", "content": str(input_val)},
+            ]
+
+            # Run inference
+            result = pipeline(messages, temperature=0.7)
+
+            # Extract the answer
+            output = str(result[0]["generated_text"][-1]["content"])
+            outputs.append(output)
+
+        return outputs
+
+    # Construct paths to example data files
+    example_datasets_dir = Path(__file__).parent / "example_datasets"
+    json_path = example_datasets_dir / "basic_questions.json"
+    csv_path = example_datasets_dir / "basic_questions.csv"
+
+    # Load dataset from JSON file
+    json_dataset = EvalDataset.from_json(
+        name="basic_questions_json",
+        path=str(json_path),
+        metrics="accuracy",
+        input="question",
+        label="answer",
+    )
+    print(f"Loaded {json_dataset.name} from JSON file: {len(json_dataset.items)} items")
+
+    # Load dataset from CSV file
+    csv_dataset = EvalDataset.from_csv(
+        name="basic_questions_csv",
+        path=str(csv_path),
+        metrics="accuracy",
+        input="question",
+        label="answer",
+    )
+    print(f"Loaded {csv_dataset.name} from CSV file: {len(csv_dataset.items)} items")
+
+    # Run evaluation on both datasets
+    results = evaluate(
+        inference,
+        datasets=[json_dataset, csv_dataset],
+        return_aggregates=True,
+        return_items=True,
+        return_output=True,
+        upload_results=False,
+    )
+
+    pprint(results)
+    return results
+
+
+if __name__ == "__main__":
+    load_dotenv()
+    log_file = setup_logging(experiment_id="1-evaluation_datasets_from_files", base_dir=Path(__file__).parent)
+    output_dir = Path(__file__).parent / "results"
+    output_dir.mkdir(exist_ok=True)
+    results_dict = main()
+    save_results_to_json(results_dict, output_dir, "1-evaluation_datasets_from_files_output.json")

tutorials/examples/3-evaluation_datasets/2-evaluation_datasets_from_huggingface.py

@@ -0,0 +1,101 @@
+"""Tutorials - Evaluation Datasets - Example 2 - Loading from HuggingFace."""
+
+import asyncio
+from pathlib import Path
+from pprint import pprint
+from typing import Any, List
+
+from dotenv import load_dotenv
+from openai import AsyncOpenAI
+
+from tutorials.utils import save_results_to_json, setup_logging
+
+from scorebook import EvalDataset, evaluate_async
+
+
+async def main() -> Any:
+    """Run evaluations using datasets loaded from HuggingFace Hub.
+
+    This example demonstrates how to load evaluation datasets directly from
+    HuggingFace Hub using the from_huggingface method. This allows you to
+    easily evaluate on standard benchmarks and datasets.
+
+    We'll evaluate on the SimpleQA dataset, which tests factual question answering.
+
+    Prerequisites:
+        - OpenAI API key set in environment variable OPENAI_API_KEY
+    """
+
+    # Initialize OpenAI client
+    client = AsyncOpenAI()
+    model_name = "gpt-4o-mini"
+
+    # Define an async inference function
+    async def inference(inputs: List[Any], **hyperparameters: Any) -> List[Any]:
+        """Process inputs through OpenAI's API.
+
+        Args:
+            inputs: Input values from an EvalDataset.
+            hyperparameters: Model hyperparameters.
+
+        Returns:
+            List of model outputs for all inputs.
+        """
+        outputs = []
+        for input_val in inputs:
+            # Build messages for OpenAI API
+            messages = [
+                {
+                    "role": "system",
+                    "content": "Answer the question directly and concisely. Provide only the answer, no additional context or text.",
+                },
+                {"role": "user", "content": str(input_val)},
+            ]
+
+            # Call OpenAI API
+            try:
+                response = await client.chat.completions.create(
+                    model=model_name,
+                    messages=messages,
+                    temperature=0.7,
+                )
+                output = response.choices[0].message.content.strip()
+            except Exception as e:
+                output = f"Error: {str(e)}"
+
+            outputs.append(output)
+
+        return outputs
+
+    # Load dataset from HuggingFace Hub
+    simple_qa = EvalDataset.from_huggingface(
+        path="basicv8vc/SimpleQA",
+        metrics="accuracy",
+        input="problem",
+        label="answer",
+        split="test",
+    )
+    print(f"Loaded {simple_qa.name} from HuggingFace Hub: {len(simple_qa.items)} items")
+
+    # Run evaluation with a sample to avoid long runtime
+    results = await evaluate_async(
+        inference,
+        simple_qa,
+        sample_size=10,  # Sample 10 items for quick testing
+        return_aggregates=True,
+        return_items=True,
+        return_output=True,
+        upload_results=False,
+    )
+
+    pprint(results)
+    return results
+
+
+if __name__ == "__main__":
+    load_dotenv()
+    log_file = setup_logging(experiment_id="2-evaluation_datasets_from_huggingface", base_dir=Path(__file__).parent)
+    output_dir = Path(__file__).parent / "results"
+    output_dir.mkdir(exist_ok=True)
+    results_dict = asyncio.run(main())
+    save_results_to_json(results_dict, output_dir, "2-evaluation_datasets_from_huggingface_output.json")