scorebook 0.0.14__py3-none-any.whl → 0.0.16__py3-none-any.whl

tutorials/examples/2-evaluate/5-hyperparameter_sweeps.py

@@ -0,0 +1,122 @@
+"""Tutorials - Evaluate - Example 5 - Hyperparameter Sweeps."""
+
+from pathlib import Path
+from pprint import pprint
+from typing import Any, List
+
+import transformers
+from dotenv import load_dotenv
+
+from tutorials.utils import save_results_to_json, setup_logging
+
+from scorebook import EvalDataset, evaluate
+
+
+def main() -> Any:
+    """Run a Scorebook evaluation with a hyperparameter sweep.
+
+    This example demonstrates how Scorebook can automatically test multiple
+    hyperparameter configurations in a single evaluation.
+
+    How Hyperparameter Sweeping Works:
+    - Define hyperparameters with lists of values to test
+    - Scorebook generates all possible combinations (Cartesian product)
+    - Each configuration is evaluated separately on the same dataset
+
+    Example Hyperparameters:
+    - system_message: "Answer the question directly and concisely." (1 value)
+    - temperature: [0.6, 0.7, 0.8] (3 values)
+    - top_p: [0.7, 0.8, 0.9] (3 values)
+    - top_k: [10, 20, 30] (3 values)
+
+    Total configurations = 1 × 3 × 3 × 3 = 27 hyperparameter configurations
+    """
+
+    # Initialize HuggingFace model pipeline
+    model_name = "microsoft/Phi-4-mini-instruct"
+    pipeline = transformers.pipeline(
+        "text-generation",
+        model=model_name,
+        model_kwargs={"torch_dtype": "auto"},
+        device_map="auto",
+    )
+
+    # Define an inference function
+    def inference(inputs: List[Any], **hyperparameters: Any) -> List[Any]:
+        """Process inputs through the model.
+
+        Args:
+            inputs: Input values from an EvalDataset.
+            hyperparameters: Model hyperparameters including system_message, temperature, top_p, top_k.
+
+        Returns:
+            List of model outputs for all inputs.
+        """
+        outputs = []
+        for input_val in inputs:
+            # Preprocess: Build messages
+            messages = [
+                {"role": "system", "content": hyperparameters["system_message"]},
+                {"role": "user", "content": str(input_val)},
+            ]
+
+            # Run inference
+            result = pipeline(
+                messages,
+                temperature=hyperparameters["temperature"],
+                top_p=hyperparameters.get("top_p"),
+                top_k=hyperparameters.get("top_k"),
+            )
+
+            # Postprocess: Extract the answer
+            output = str(result[0]["generated_text"][-1]["content"])
+            outputs.append(output)
+
+        return outputs
+
+    # Create a list of evaluation items
+    evaluation_items = [
+        {"question": "What is 2 + 2?", "answer": "4"},
+        {"question": "What is the capital of France?", "answer": "Paris"},
+        {"question": "Who wrote Romeo and Juliet?", "answer": "William Shakespeare"},
+    ]
+
+    # Create an evaluation dataset
+    evaluation_dataset = EvalDataset.from_list(
+        name="basic_questions",
+        metrics="accuracy",
+        items=evaluation_items,
+        input="question",
+    label="answer",
+    )
+
+    # Define hyperparameters with lists of values to create a sweep
+    hyperparameters = {
+        "system_message": "Answer the question directly and concisely.",
+        "temperature": [0.6, 0.7, 0.8],
+        "top_p": [0.7, 0.8, 0.9],
+        "top_k": [10, 20, 30],
+    }
+
+    # Run evaluation across all hyperparameter combinations
+    results = evaluate(
+        inference,
+        evaluation_dataset,
+        hyperparameters=hyperparameters,
+        return_aggregates=True,
+        return_items=True,
+        return_output=True,
+        upload_results=False,
+    )
+
+    pprint(results)
+    return results
+
+
+if __name__ == "__main__":
+    load_dotenv()
+    log_file = setup_logging(experiment_id="5-hyperparameter_sweeps", base_dir=Path(__file__).parent)
+    output_dir = Path(__file__).parent / "results"
+    output_dir.mkdir(exist_ok=True)
+    results_dict = main()
+    save_results_to_json(results_dict, output_dir, "5-hyperparameter_sweeps_output.json")

tutorials/examples/2-evaluate/6-inference_pipelines.py

@@ -0,0 +1,141 @@
+"""Tutorials - Evaluate - Example 6 - Inference Pipelines."""
+
+from pprint import pprint
+from typing import Any, List
+
+import transformers
+from dotenv import load_dotenv
+from pathlib import Path
+
+from tutorials.utils import save_results_to_json, setup_logging
+
+from scorebook import EvalDataset, InferencePipeline, evaluate
+
+
+def main() -> Any:
+    """Run a simple Scorebook evaluation using an InferencePipeline.
+
+    This example demonstrates how to use Scorebook's InferencePipeline in evaluations.
+
+    Inference pipelines separate the evaluation workflow into three distinct stages:
+        1. Pre-processing: Convert raw dataset items into model-ready input format
+        2. Inference: Execute model predictions on preprocessed data
+        3. Post-processing: Extract final answers from raw model outputs
+
+    These stages can be encapsulated in reusable functions and used to create pipelines.
+    An inference pipeline can be passed into the evaluate function's inference parameter.
+    """
+
+    # === Pre-Processing ===
+
+    # The preprocessor function is responsible for mapping items in an Eval Dataset to model inputs
+    def preprocessor(input_value: str, **hyperparameter_config: Any) -> List[Any]:
+        """Convert an evaluation input to a valid model input.
+
+        Args:
+            input_value: The input value from the dataset.
+            hyperparameter_config: Model hyperparameters.
+
+        Returns:
+            A structured representation of an evaluation item for model input.
+        """
+        messages = [
+            {
+                "role": "system",
+                "content": hyperparameter_config["system_message"],
+            },
+            {"role": "user", "content": input_value},
+        ]
+
+        return messages
+
+    # === Inference ===
+
+    pipeline = transformers.pipeline(
+        "text-generation",
+        model="microsoft/Phi-4-mini-instruct",
+        model_kwargs={"torch_dtype": "auto"},
+        device_map="auto",
+    )
+
+    # An inference function for an InferencePipeline that returns a list of raw outputs
+    def inference(preprocessed_items: List[Any], **hyperparameter_config: Any) -> List[Any]:
+        """Run model inference on preprocessed eval items.
+
+        Args:
+            preprocessed_items: The list of evaluation items for an EvalDataset.
+            hyperparameter_config: Model hyperparameters.
+
+        Returns:
+            A list of model outputs for an EvalDataset.
+        """
+        return [
+            pipeline(model_input, temperature=hyperparameter_config["temperature"])
+            for model_input in preprocessed_items
+        ]
+
+    # === Post-Processing ===
+
+    # The postprocessor function parses model output for metric scoring
+    def postprocessor(model_output: Any, **hyperparameter_config: Any) -> str:
+        """Extract the final parsed answer from the model output.
+
+        Args:
+            model_output: An evaluation item from an EvalDataset.
+            hyperparameter_config: Model hyperparameters.
+
+        Returns:
+            Parsed answer from the model output to be used for scoring.
+        """
+        return str(model_output[0]["generated_text"][-1]["content"])
+
+    # === Evaluation With An InferencePipeline ===
+
+    # Step 1: Create an inference pipeline, using the 3 functions defined
+    inference_pipeline = InferencePipeline(
+        model="microsoft/Phi-4-mini-instruct",
+        preprocessor=preprocessor,
+        inference_function=inference,
+        postprocessor=postprocessor,
+    )
+
+    # Step 2: Create a list of evaluation items
+    evaluation_items = [
+        {"question": "What is 2 + 2?", "answer": "4"},
+        {"question": "What is the capital of France?", "answer": "Paris"},
+        {"question": "Who wrote Romeo and Juliet?", "answer": "William Shakespeare"},
+    ]
+
+    # Create an evaluation dataset
+    evaluation_dataset = EvalDataset.from_list(
+        name="basic_questions",
+        metrics="accuracy",
+        items=evaluation_items,
+        input="question",
+        label="answer",
+    )
+
+    # Step 3: Run the evaluation using the inference pipeline and dataset
+    results = evaluate(
+        inference_pipeline,
+        evaluation_dataset,
+        hyperparameters={
+            "temperature": 0.7,
+            "system_message": "Answer the question directly and concisely.",
+        },
+        return_items=True,     # Enable to include results for individual items in the dict returned.
+        return_output=True,    # Enable to include the model's output for individual items.
+        upload_results=False,  # Disable uploading for this example
+    )
+
+    pprint(results)
+    return results
+
+
+if __name__ == "__main__":
+    load_dotenv()
+    log_file = setup_logging(experiment_id="6-inference_output", base_dir=Path(__file__).parent)
+    output_dir = Path(__file__).parent / "results"
+    output_dir.mkdir(exist_ok=True)
+    results_dict = main()
+    save_results_to_json(results_dict, output_dir, "6-inference_output_output.json")

tutorials/examples/3-evaluation_datasets/1-evaluation_datasets_from_files.py

@@ -0,0 +1,110 @@
+"""Tutorials - Evaluation Datasets - Example 1 - Loading Datasets from Files."""
+
+from pathlib import Path
+from pprint import pprint
+from typing import Any, List
+
+import transformers
+from dotenv import load_dotenv
+
+from tutorials.utils import save_results_to_json, setup_logging
+
+from scorebook import EvalDataset, evaluate
+
+
+def main() -> Any:
+    """Run evaluations using datasets loaded from local files.
+
+    This example demonstrates how to load evaluation datasets from files:
+        - from_json: Load datasets from JSON files
+        - from_csv: Load datasets from CSV files
+
+    Both methods support loading data from local files with custom field mappings.
+    """
+
+    # Initialize HuggingFace model pipeline
+    model_name = "microsoft/Phi-4-mini-instruct"
+    pipeline = transformers.pipeline(
+        "text-generation",
+        model=model_name,
+        model_kwargs={"torch_dtype": "auto"},
+        device_map="auto",
+    )
+
+    # Define an inference function
+    def inference(inputs: List[Any], **hyperparameters: Any) -> List[Any]:
+        """Process inputs through the model.
+
+        Args:
+            inputs: Input values from an EvalDataset.
+            hyperparameters: Model hyperparameters.
+
+        Returns:
+            List of model outputs for all inputs.
+        """
+        outputs = []
+        for input_val in inputs:
+            # Build messages
+            messages = [
+                {
+                    "role": "system",
+                    "content": "Answer the question directly and concisely. Provide only the answer, no additional context or text.",
+                },
+                {"role": "user", "content": str(input_val)},
+            ]
+
+            # Run inference
+            result = pipeline(messages, temperature=0.7)
+
+            # Extract the answer
+            output = str(result[0]["generated_text"][-1]["content"])
+            outputs.append(output)
+
+        return outputs
+
+    # Construct paths to example data files
+    example_datasets_dir = Path(__file__).parent / "example_datasets"
+    json_path = example_datasets_dir / "basic_questions.json"
+    csv_path = example_datasets_dir / "basic_questions.csv"
+
+    # Load dataset from JSON file
+    json_dataset = EvalDataset.from_json(
+        name="basic_questions_json",
+        path=str(json_path),
+        metrics="accuracy",
+        input="question",
+        label="answer",
+    )
+    print(f"Loaded {json_dataset.name} from JSON file: {len(json_dataset.items)} items")
+
+    # Load dataset from CSV file
+    csv_dataset = EvalDataset.from_csv(
+        name="basic_questions_csv",
+        path=str(csv_path),
+        metrics="accuracy",
+        input="question",
+        label="answer",
+    )
+    print(f"Loaded {csv_dataset.name} from CSV file: {len(csv_dataset.items)} items")
+
+    # Run evaluation on both datasets
+    results = evaluate(
+        inference,
+        datasets=[json_dataset, csv_dataset],
+        return_aggregates=True,
+        return_items=True,
+        return_output=True,
+        upload_results=False,
+    )
+
+    pprint(results)
+    return results
+
+
+if __name__ == "__main__":
+    load_dotenv()
+    log_file = setup_logging(experiment_id="1-evaluation_datasets_from_files", base_dir=Path(__file__).parent)
+    output_dir = Path(__file__).parent / "results"
+    output_dir.mkdir(exist_ok=True)
+    results_dict = main()
+    save_results_to_json(results_dict, output_dir, "1-evaluation_datasets_from_files_output.json")

tutorials/examples/3-evaluation_datasets/2-evaluation_datasets_from_huggingface.py

@@ -0,0 +1,101 @@
+"""Tutorials - Evaluation Datasets - Example 2 - Loading from HuggingFace."""
+
+import asyncio
+from pathlib import Path
+from pprint import pprint
+from typing import Any, List
+
+from dotenv import load_dotenv
+from openai import AsyncOpenAI
+
+from tutorials.utils import save_results_to_json, setup_logging
+
+from scorebook import EvalDataset, evaluate_async
+
+
+async def main() -> Any:
+    """Run evaluations using datasets loaded from HuggingFace Hub.
+
+    This example demonstrates how to load evaluation datasets directly from
+    HuggingFace Hub using the from_huggingface method. This allows you to
+    easily evaluate on standard benchmarks and datasets.
+
+    We'll evaluate on the SimpleQA dataset, which tests factual question answering.
+
+    Prerequisites:
+        - OpenAI API key set in environment variable OPENAI_API_KEY
+    """
+
+    # Initialize OpenAI client
+    client = AsyncOpenAI()
+    model_name = "gpt-4o-mini"
+
+    # Define an async inference function
+    async def inference(inputs: List[Any], **hyperparameters: Any) -> List[Any]:
+        """Process inputs through OpenAI's API.
+
+        Args:
+            inputs: Input values from an EvalDataset.
+            hyperparameters: Model hyperparameters.
+
+        Returns:
+            List of model outputs for all inputs.
+        """
+        outputs = []
+        for input_val in inputs:
+            # Build messages for OpenAI API
+            messages = [
+                {
+                    "role": "system",
+                    "content": "Answer the question directly and concisely. Provide only the answer, no additional context or text.",
+                },
+                {"role": "user", "content": str(input_val)},
+            ]
+
+            # Call OpenAI API
+            try:
+                response = await client.chat.completions.create(
+                    model=model_name,
+                    messages=messages,
+                    temperature=0.7,
+                )
+                output = response.choices[0].message.content.strip()
+            except Exception as e:
+                output = f"Error: {str(e)}"
+
+            outputs.append(output)
+
+        return outputs
+
+    # Load dataset from HuggingFace Hub
+    simple_qa = EvalDataset.from_huggingface(
+        path="basicv8vc/SimpleQA",
+        metrics="accuracy",
+        input="problem",
+        label="answer",
+        split="test",
+    )
+    print(f"Loaded {simple_qa.name} from HuggingFace Hub: {len(simple_qa.items)} items")
+
+    # Run evaluation with a sample to avoid long runtime
+    results = await evaluate_async(
+        inference,
+        simple_qa,
+        sample_size=10,  # Sample 10 items for quick testing
+        return_aggregates=True,
+        return_items=True,
+        return_output=True,
+        upload_results=False,
+    )
+
+    pprint(results)
+    return results
+
+
+if __name__ == "__main__":
+    load_dotenv()
+    log_file = setup_logging(experiment_id="2-evaluation_datasets_from_huggingface", base_dir=Path(__file__).parent)
+    output_dir = Path(__file__).parent / "results"
+    output_dir.mkdir(exist_ok=True)
+    results_dict = asyncio.run(main())
+    save_results_to_json(results_dict, output_dir, "2-evaluation_datasets_from_huggingface_output.json")

tutorials/examples/3-evaluation_datasets/3-evaluation_datasets_from_huggingface_with_yaml_configs.py

@@ -0,0 +1,110 @@
+"""Tutorials - Evaluation Datasets - Example 3 - Loading from YAML Config."""
+
+import asyncio
+from pathlib import Path
+from pprint import pprint
+from typing import Any, List
+
+from dotenv import load_dotenv
+from openai import AsyncOpenAI
+
+from tutorials.utils import save_results_to_json, setup_logging
+
+from scorebook import EvalDataset, evaluate_async
+
+
+async def main() -> Any:
+    """Run evaluations using datasets loaded from YAML configuration files.
+
+    This example demonstrates how to use YAML configuration files to define
+    dataset loading parameters. YAML configs are useful for:
+        - Storing dataset configurations in version control
+        - Reusing the same dataset configuration across projects
+        - Defining complex prompt templates and field mappings
+
+    The YAML files contain:
+        - HuggingFace dataset path and split information
+        - Metrics to use for evaluation
+        - Jinja2 templates for input and label formatting
+        - Metadata about the dataset
+
+    Prerequisites:
+        - OpenAI API key set in environment variable OPENAI_API_KEY
+    """
+
+    # Initialize OpenAI client
+    client = AsyncOpenAI()
+    model_name = "gpt-4o-mini"
+
+    # Define an async inference function
+    async def inference(inputs: List[Any], **hyperparameters: Any) -> List[Any]:
+        """Process inputs through OpenAI's API.
+
+        Args:
+            inputs: Input values from an EvalDataset.
+            hyperparameters: Model hyperparameters.
+
+        Returns:
+            List of model outputs for all inputs.
+        """
+        outputs = []
+        for input_val in inputs:
+            # Build messages for OpenAI API
+            messages = [
+                {
+                    "role": "system",
+                    "content": "Answer the multiple choice question by selecting the correct letter (A, B, C, D, etc.). Provide ONLY the letter of your answer, no additional text or explanation.",
+                },
+                {"role": "user", "content": str(input_val)},
+            ]
+
+            # Call OpenAI API
+            try:
+                response = await client.chat.completions.create(
+                    model=model_name,
+                    messages=messages,
+                    temperature=0.7,
+                )
+                output = response.choices[0].message.content.strip()
+            except Exception as e:
+                output = f"Error: {str(e)}"
+
+            outputs.append(output)
+
+        return outputs
+
+    # Construct paths to YAML config files
+    yaml_configs_dir = Path(__file__).parent / "example_yaml_configs"
+    cais_mmlu_yaml = yaml_configs_dir / "Cais-MMLU.yaml"
+    tiger_mmlu_pro_yaml = yaml_configs_dir / "TIGER-Lab-MMLU-Pro.yaml"
+
+    # Load Cais-MMLU dataset from YAML configuration
+    cais_mmlu = EvalDataset.from_yaml(str(cais_mmlu_yaml))
+    print(f"Loaded {cais_mmlu.name} from YAML config: {len(cais_mmlu.items)} items")
+
+    # Load TIGER-Lab MMLU-Pro dataset from YAML configuration
+    tiger_mmlu_pro = EvalDataset.from_yaml(str(tiger_mmlu_pro_yaml))
+    print(f"Loaded {tiger_mmlu_pro.name} from YAML config: {len(tiger_mmlu_pro.items)} items")
+
+    # Run evaluation on both datasets
+    results = await evaluate_async(
+        inference,
+        datasets=[cais_mmlu, tiger_mmlu_pro],
+        sample_size=5,  # Sample 5 items from each dataset for quick testing
+        return_aggregates=True,
+        return_items=True,
+        return_output=True,
+        upload_results=False,
+    )
+
+    pprint(results)
+    return results
+
+
+if __name__ == "__main__":
+    load_dotenv()
+    log_file = setup_logging(experiment_id="3-evaluation_datasets_from_yaml", base_dir=Path(__file__).parent)
+    output_dir = Path(__file__).parent / "results"
+    output_dir.mkdir(exist_ok=True)
+    results_dict = asyncio.run(main())
+    save_results_to_json(results_dict, output_dir, "3-evaluation_datasets_from_yaml_output.json")

tutorials/examples/3-evaluation_datasets/example_datasets/basic_questions.csv

@@ -0,0 +1,11 @@
+question,answer
+What is 2 + 2?,4
+What is the capital of France?,Paris
+Who wrote Romeo and Juliet?,William Shakespeare
+What is 5 * 6?,30
+What is the largest planet in our solar system?,Jupiter
+Who painted the Mona Lisa?,Leonardo da Vinci
+What is the square root of 64?,8
+What is the capital of Japan?,Tokyo
+Who invented the telephone?,Alexander Graham Bell
+What is 12 - 7?,5

tutorials/examples/3-evaluation_datasets/example_datasets/basic_questions.json

@@ -0,0 +1,42 @@
+[
+    {
+        "question": "What is 2 + 2?",
+        "answer": "4"
+    },
+    {
+        "question": "What is the capital of France?",
+        "answer": "Paris"
+    },
+    {
+        "question": "Who wrote Romeo and Juliet?",
+        "answer": "William Shakespeare"
+    },
+    {
+        "question": "What is 5 * 6?",
+        "answer": "30"
+    },
+    {
+        "question": "What is the largest planet in our solar system?",
+        "answer": "Jupiter"
+    },
+    {
+        "question": "Who painted the Mona Lisa?",
+        "answer": "Leonardo da Vinci"
+    },
+    {
+        "question": "What is the square root of 64?",
+        "answer": "8"
+    },
+    {
+        "question": "What is the capital of Japan?",
+        "answer": "Tokyo"
+    },
+    {
+        "question": "Who invented the telephone?",
+        "answer": "Alexander Graham Bell"
+    },
+    {
+        "question": "What is 12 - 7?",
+        "answer": "5"
+    }
+]

tutorials/examples/3-evaluation_datasets/example_yaml_configs/Cais-MMLU.yaml

@@ -0,0 +1,19 @@
+path: "cais/mmlu"
+name: "Cais-MMLU"
+split: "test"
+config: "all"
+metrics:
+  - "accuracy"
+
+templates:
+  input: |
+    {{ question }}
+
+    A. {{ choices[0] }}
+    B. {{ choices[1] }}
+    C. {{ choices[2] }}
+    D. {{ choices[3] }}
+  label: "{{ answer }}"
+
+metadata:
+  description: "MMLU multiple choice questions from Cais"

tutorials/examples/3-evaluation_datasets/example_yaml_configs/TIGER-Lab-MMLU-Pro.yaml

@@ -0,0 +1,18 @@
+path: "TIGER-Lab/MMLU-Pro"
+name: "TIGER-Lab/MMLU-Pro"
+split: "validation"
+config: "default"
+metrics:
+  - "accuracy"
+
+templates:
+  input: |
+    {{ question }}
+    Options:
+    {% for option in options %}
+    {{ number_to_letter(loop.index0) }} : {{ option }}
+    {% endfor %}
+  label: "{{ answer }}"
+
+metadata:
+  description: "MMLU-Pro multiple choice questions"