scorebook 0.0.13__py3-none-any.whl → 0.0.15__py3-none-any.whl

tutorials/examples/3-evaluation_datasets/3-evaluation_datasets_from_huggingface_with_yaml_configs.py

@@ -0,0 +1,110 @@
+"""Tutorials - Evaluation Datasets - Example 3 - Loading from YAML Config."""
+
+import asyncio
+from pathlib import Path
+from pprint import pprint
+from typing import Any, List
+
+from dotenv import load_dotenv
+from openai import AsyncOpenAI
+
+from tutorials.utils import save_results_to_json, setup_logging
+
+from scorebook import EvalDataset, evaluate_async
+
+
+async def main() -> Any:
+    """Run evaluations using datasets loaded from YAML configuration files.
+
+    This example demonstrates how to use YAML configuration files to define
+    dataset loading parameters. YAML configs are useful for:
+        - Storing dataset configurations in version control
+        - Reusing the same dataset configuration across projects
+        - Defining complex prompt templates and field mappings
+
+    The YAML files contain:
+        - HuggingFace dataset path and split information
+        - Metrics to use for evaluation
+        - Jinja2 templates for input and label formatting
+        - Metadata about the dataset
+
+    Prerequisites:
+        - OpenAI API key set in environment variable OPENAI_API_KEY
+    """
+
+    # Initialize OpenAI client
+    client = AsyncOpenAI()
+    model_name = "gpt-4o-mini"
+
+    # Define an async inference function
+    async def inference(inputs: List[Any], **hyperparameters: Any) -> List[Any]:
+        """Process inputs through OpenAI's API.
+
+        Args:
+            inputs: Input values from an EvalDataset.
+            hyperparameters: Model hyperparameters.
+
+        Returns:
+            List of model outputs for all inputs.
+        """
+        outputs = []
+        for input_val in inputs:
+            # Build messages for OpenAI API
+            messages = [
+                {
+                    "role": "system",
+                    "content": "Answer the multiple choice question by selecting the correct letter (A, B, C, D, etc.). Provide ONLY the letter of your answer, no additional text or explanation.",
+                },
+                {"role": "user", "content": str(input_val)},
+            ]
+
+            # Call OpenAI API
+            try:
+                response = await client.chat.completions.create(
+                    model=model_name,
+                    messages=messages,
+                    temperature=0.7,
+                )
+                output = response.choices[0].message.content.strip()
+            except Exception as e:
+                output = f"Error: {str(e)}"
+
+            outputs.append(output)
+
+        return outputs
+
+    # Construct paths to YAML config files
+    yaml_configs_dir = Path(__file__).parent / "example_yaml_configs"
+    cais_mmlu_yaml = yaml_configs_dir / "Cais-MMLU.yaml"
+    tiger_mmlu_pro_yaml = yaml_configs_dir / "TIGER-Lab-MMLU-Pro.yaml"
+
+    # Load Cais-MMLU dataset from YAML configuration
+    cais_mmlu = EvalDataset.from_yaml(str(cais_mmlu_yaml))
+    print(f"Loaded {cais_mmlu.name} from YAML config: {len(cais_mmlu.items)} items")
+
+    # Load TIGER-Lab MMLU-Pro dataset from YAML configuration
+    tiger_mmlu_pro = EvalDataset.from_yaml(str(tiger_mmlu_pro_yaml))
+    print(f"Loaded {tiger_mmlu_pro.name} from YAML config: {len(tiger_mmlu_pro.items)} items")
+
+    # Run evaluation on both datasets
+    results = await evaluate_async(
+        inference,
+        datasets=[cais_mmlu, tiger_mmlu_pro],
+        sample_size=5,  # Sample 5 items from each dataset for quick testing
+        return_aggregates=True,
+        return_items=True,
+        return_output=True,
+        upload_results=False,
+    )
+
+    pprint(results)
+    return results
+
+
+if __name__ == "__main__":
+    load_dotenv()
+    log_file = setup_logging(experiment_id="3-evaluation_datasets_from_yaml", base_dir=Path(__file__).parent)
+    output_dir = Path(__file__).parent / "results"
+    output_dir.mkdir(exist_ok=True)
+    results_dict = asyncio.run(main())
+    save_results_to_json(results_dict, output_dir, "3-evaluation_datasets_from_yaml_output.json")

tutorials/examples/3-evaluation_datasets/example_datasets/basic_questions.csv

@@ -0,0 +1,11 @@
+question,answer
+What is 2 + 2?,4
+What is the capital of France?,Paris
+Who wrote Romeo and Juliet?,William Shakespeare
+What is 5 * 6?,30
+What is the largest planet in our solar system?,Jupiter
+Who painted the Mona Lisa?,Leonardo da Vinci
+What is the square root of 64?,8
+What is the capital of Japan?,Tokyo
+Who invented the telephone?,Alexander Graham Bell
+What is 12 - 7?,5

tutorials/examples/3-evaluation_datasets/example_datasets/basic_questions.json

@@ -0,0 +1,42 @@
+[
+    {
+        "question": "What is 2 + 2?",
+        "answer": "4"
+    },
+    {
+        "question": "What is the capital of France?",
+        "answer": "Paris"
+    },
+    {
+        "question": "Who wrote Romeo and Juliet?",
+        "answer": "William Shakespeare"
+    },
+    {
+        "question": "What is 5 * 6?",
+        "answer": "30"
+    },
+    {
+        "question": "What is the largest planet in our solar system?",
+        "answer": "Jupiter"
+    },
+    {
+        "question": "Who painted the Mona Lisa?",
+        "answer": "Leonardo da Vinci"
+    },
+    {
+        "question": "What is the square root of 64?",
+        "answer": "8"
+    },
+    {
+        "question": "What is the capital of Japan?",
+        "answer": "Tokyo"
+    },
+    {
+        "question": "Who invented the telephone?",
+        "answer": "Alexander Graham Bell"
+    },
+    {
+        "question": "What is 12 - 7?",
+        "answer": "5"
+    }
+]

tutorials/examples/3-evaluation_datasets/example_yaml_configs/Cais-MMLU.yaml

@@ -0,0 +1,19 @@
+path: "cais/mmlu"
+name: "Cais-MMLU"
+split: "test"
+config: "all"
+metrics:
+  - "accuracy"
+
+templates:
+  input: |
+    {{ question }}
+
+    A. {{ choices[0] }}
+    B. {{ choices[1] }}
+    C. {{ choices[2] }}
+    D. {{ choices[3] }}
+  label: "{{ answer }}"
+
+metadata:
+  description: "MMLU multiple choice questions from Cais"

tutorials/examples/3-evaluation_datasets/example_yaml_configs/TIGER-Lab-MMLU-Pro.yaml

@@ -0,0 +1,18 @@
+path: "TIGER-Lab/MMLU-Pro"
+name: "TIGER-Lab/MMLU-Pro"
+split: "validation"
+config: "default"
+metrics:
+  - "accuracy"
+
+templates:
+  input: |
+    {{ question }}
+    Options:
+    {% for option in options %}
+    {{ number_to_letter(loop.index0) }} : {{ option }}
+    {% endfor %}
+  label: "{{ answer }}"
+
+metadata:
+  description: "MMLU-Pro multiple choice questions"

tutorials/examples/4-adaptive_evaluations/1-adaptive_evaluation.py

@@ -0,0 +1,114 @@
+"""Tutorials - Adaptive Evaluations - Example 1 - Adaptive Evaluation."""
+
+import asyncio
+import string
+from pathlib import Path
+from pprint import pprint
+from typing import Any, List
+
+from dotenv import load_dotenv
+from openai import AsyncOpenAI
+
+from tutorials.utils import save_results_to_json, setup_logging
+
+from scorebook import evaluate_async, login
+
+
+async def main() -> Any:
+    """Run an adaptive evaluation using Trismik's adaptive testing.
+
+    This example demonstrates how to use Trismik's adaptive evaluation feature.
+    Adaptive evaluations use Item Response Theory (IRT) to efficiently estimate
+    model capabilities by selecting questions based on previous responses.
+
+    Benefits of adaptive evaluation:
+        - More efficient: Fewer questions needed to assess capability
+        - Precise measurement: Better statistical confidence intervals
+        - Optimal difficulty: Questions adapt to model's skill level
+
+    Prerequisites:
+        - Valid Trismik API key set in TRISMIK_API_KEY environment variable
+        - A Trismik project ID
+        - OpenAI API key set in OPENAI_API_KEY environment variable
+    """
+
+    # Initialize OpenAI client
+    client = AsyncOpenAI()
+    model_name = "gpt-4o-mini"
+
+    # Define an async inference function
+    async def inference(inputs: List[Any], **hyperparameters: Any) -> List[Any]:
+        """Process inputs through OpenAI's API.
+
+        Args:
+            inputs: Input values from an EvalDataset. For adaptive MMLU-Pro,
+                   each input is a dict with 'question' and 'options' keys.
+            hyperparameters: Model hyperparameters.
+
+        Returns:
+            List of model outputs for all inputs.
+        """
+        outputs = []
+        for input_val in inputs:
+            # Handle dict input from adaptive dataset
+            if isinstance(input_val, dict):
+                prompt = input_val.get("question", "")
+                if "options" in input_val:
+                    prompt += "\nOptions:\n" + "\n".join(
+                        f"{letter}: {choice}"
+                        for letter, choice in zip(string.ascii_uppercase, input_val["options"])
+                    )
+            else:
+                prompt = str(input_val)
+
+            # Build messages for OpenAI API
+            messages = [
+                {
+                    "role": "system",
+                    "content": "Answer the question with a single letter representing the correct answer from the list of choices. Do not provide any additional explanation or output beyond the single letter.",
+                },
+                {"role": "user", "content": prompt},
+            ]
+
+            # Call OpenAI API
+            try:
+                response = await client.chat.completions.create(
+                    model=model_name,
+                    messages=messages,
+                    temperature=0.7,
+                )
+                output = response.choices[0].message.content.strip()
+            except Exception as e:
+                output = f"Error: {str(e)}"
+
+            outputs.append(output)
+
+        return outputs
+
+    # Step 1: Log in with your Trismik API key
+    # login() reads TRISMIK_API_KEY from environment variables or .env file
+    login()
+
+    # Step 2: Run adaptive evaluation
+    results = await evaluate_async(
+        inference,
+        datasets="trismik/headQA:adaptive",  # Adaptive datasets have the ":adaptive" suffix
+        experiment_id="Adaptive-Head-QA-Evaluation",
+        project_id='TRISMIK-PROJECT-ID',
+        return_dict=True,
+        return_aggregates=True,
+        return_items=True,
+        return_output=True,
+    )
+
+    pprint(results)
+    return results
+
+
+if __name__ == "__main__":
+    load_dotenv()
+    log_file = setup_logging(experiment_id="1-adaptive_evaluation", base_dir=Path(__file__).parent)
+    output_dir = Path(__file__).parent / "results"
+    output_dir.mkdir(exist_ok=True)
+    results_dict = asyncio.run(main())
+    save_results_to_json(results_dict, output_dir, "1-adaptive_evaluation_output.json")

tutorials/examples/4-adaptive_evaluations/2-adaptive_dataset_splits.py

@@ -0,0 +1,106 @@
+"""Tutorials - Adaptive Evaluations - Example 2 - Adaptive Dataset Splits."""
+
+import asyncio
+import string
+from pathlib import Path
+from pprint import pprint
+from typing import Any, List
+
+from dotenv import load_dotenv
+from openai import AsyncOpenAI
+
+from tutorials.utils import save_results_to_json, setup_logging
+
+from scorebook import evaluate_async, login
+
+
+async def main() -> Any:
+    """
+
+    Prerequisites:
+        - Valid Trismik API key set in TRISMIK_API_KEY environment variable
+        - A Trismik project ID
+        - OpenAI API key set in OPENAI_API_KEY environment variable
+    """
+
+    # Initialize OpenAI client
+    client = AsyncOpenAI()
+    model_name = "gpt-4o-mini"
+
+    # Define an async inference function
+    async def inference(inputs: List[Any], **hyperparameters: Any) -> List[Any]:
+        """Process inputs through OpenAI's API.
+
+        Args:
+            inputs: Input values from an EvalDataset. For adaptive headQA,
+                   each input is a dict with 'question' and 'options' keys.
+            hyperparameters: Model hyperparameters.
+
+        Returns:
+            List of model outputs for all inputs.
+        """
+        outputs = []
+        for input_val in inputs:
+            # Handle dict input from adaptive dataset
+            if isinstance(input_val, dict):
+                prompt = input_val.get("question", "")
+                if "options" in input_val:
+                    prompt += "\nOptions:\n" + "\n".join(
+                        f"{letter}: {choice}"
+                        for letter, choice in zip(string.ascii_uppercase, input_val["options"])
+                    )
+            else:
+                prompt = str(input_val)
+
+            # Build messages for OpenAI API
+            messages = [
+                {
+                    "role": "system",
+                    "content": "Answer the question with a single letter representing the correct answer from the list of choices. Do not provide any additional explanation or output beyond the single letter.",
+                },
+                {"role": "user", "content": prompt},
+            ]
+
+            # Call OpenAI API
+            try:
+                response = await client.chat.completions.create(
+                    model=model_name,
+                    messages=messages,
+                    temperature=0.7,
+                )
+                output = response.choices[0].message.content.strip()
+            except Exception as e:
+                output = f"Error: {str(e)}"
+
+            outputs.append(output)
+
+        return outputs
+
+    # Step 1: Log in with your Trismik API key
+    # login() reads TRISMIK_API_KEY from environment variables or .env file
+    login()
+
+    # Step 2: Run adaptive evaluation
+    results = await evaluate_async(
+        inference,
+        datasets="trismik/headQA:adaptive",  # Adaptive datasets have the ":adaptive" suffix
+        split="test",                        # Specify the test split for evaluation
+        experiment_id="Adaptive-Head-QA-Test-Evaluation",
+        project_id='TRISMIK-PROJECT-ID',
+        return_dict=True,
+        return_aggregates=True,
+        return_items=True,
+        return_output=True,
+    )
+
+    pprint(results)
+    return results
+
+
+if __name__ == "__main__":
+    load_dotenv()
+    log_file = setup_logging(experiment_id="2-adaptive_dataset_splits", base_dir=Path(__file__).parent)
+    output_dir = Path(__file__).parent / "results"
+    output_dir.mkdir(exist_ok=True)
+    results_dict = asyncio.run(main())
+    save_results_to_json(results_dict, output_dir, "2-adaptive_dataset_splits_output.json")

tutorials/examples/5-upload_results/1-uploading_score_results.py

@@ -0,0 +1,92 @@
+"""Tutorials - Upload Results - Example 1 - Uploading score() Results."""
+
+from pathlib import Path
+from pprint import pprint
+from typing import Any
+
+from dotenv import load_dotenv
+
+from tutorials.utils import save_results_to_json, setup_logging
+
+from scorebook import login, score
+from scorebook.metrics.accuracy import Accuracy
+
+
+def main() -> Any:
+    """Score pre-computed outputs and upload results to Trismik's dashboard.
+
+    This example demonstrates how to upload score() results to Trismik.
+    The score() function is used when you already have model outputs and
+    want to score them against labels.
+
+    Use score() when you want to:
+        - Score pre-computed model outputs
+        - Re-score existing results with different metrics
+        - Upload scoring results without re-running inference
+
+    Prerequisites:
+        - Valid Trismik API key set in TRISMIK_API_KEY environment variable
+        - A Trismik project ID
+    """
+
+    # Prepare items with pre-computed outputs and labels
+    items = [
+        {
+            "input": "What is 2 + 2?",
+            "output": "4",
+            "label": "4"
+        },
+        {
+            "input": "What is the capital of France?",
+            "output": "Paris",
+            "label": "Paris"
+        },
+        {
+            "input": "Who wrote Romeo and Juliet?",
+            "output": "William Shakespeare",
+            "label": "William Shakespeare"
+        },
+        {
+            "input": "What is 5 * 6?",
+            "output": "30",
+            "label": "30"
+        },
+        {
+            "input": "What is the largest planet in our solar system?",
+            "output": "Jupiter",
+            "label": "Jupiter"
+        },
+    ]
+
+    # Step 1: Log in with your Trismik API key
+    # login() reads TRISMIK_API_KEY from environment variables or .env file
+    login()
+
+    # Step 2: Score the outputs and upload results
+    # When you provide experiment_id and project_id, results are automatically uploaded
+    results = score(
+        items=items,
+        metrics=Accuracy,
+        dataset_name="basic_questions",
+        model_name="gpt-4o-mini",
+        experiment_id="Score-Upload-Example",
+        project_id="TRISMIK_PROJECT_ID", # TODO: ADD YOUR TRISMIK PROJECT ID
+        metadata={
+            "description": "Example demonstrating score() result uploading",
+            "note": "These are pre-computed outputs",
+        },
+        upload_results=True,  # Explicitly enable uploading
+    )
+
+    print("\nResults uploaded successfully!")
+    pprint(results)
+    return results
+
+
+if __name__ == "__main__":
+    load_dotenv()
+    log_file = setup_logging(experiment_id="1-uploading_score_results", base_dir=Path(__file__).parent)
+    output_dir = Path(__file__).parent / "results"
+    output_dir.mkdir(exist_ok=True)
+    results_dict = main()
+    save_results_to_json(results_dict, output_dir, "1-uploading_score_results_output.json")

tutorials/examples/5-upload_results/2-uploading_evaluate_results.py

@@ -0,0 +1,117 @@
+"""Tutorials - Upload Results - Example 2 - Uploading evaluate() Results."""
+
+from pathlib import Path
+from pprint import pprint
+from typing import Any, List
+
+import transformers
+from dotenv import load_dotenv
+
+from tutorials.utils import save_results_to_json, setup_logging
+
+from scorebook import EvalDataset, evaluate, login
+
+
+def main() -> Any:
+    """Run an evaluation and upload results to Trismik's dashboard.
+
+    This example demonstrates how to upload evaluate() results to Trismik.
+    The evaluate() function runs inference on a dataset and automatically
+    uploads the results when you provide experiment_id and project_id.
+
+    Use evaluate() when you want to:
+        - Run inference AND score in one step
+        - Track full evaluation runs with hyperparameters
+        - Compare different models on the same dataset
+
+    Prerequisites:
+        - Valid Trismik API key set in TRISMIK_API_KEY environment variable
+        - A Trismik project ID
+    """
+
+    # Initialize HuggingFace model pipeline
+    model_name = "microsoft/Phi-4-mini-instruct"
+    pipeline = transformers.pipeline(
+        "text-generation",
+        model=model_name,
+        model_kwargs={"torch_dtype": "auto"},
+        device_map="auto",
+    )
+
+    # Define an inference function
+    def inference(inputs: List[Any], **hyperparameters: Any) -> List[Any]:
+        """Process inputs through the model.
+
+        Args:
+            inputs: Input values from an EvalDataset.
+            hyperparameters: Model hyperparameters.
+
+        Returns:
+            List of model outputs for all inputs.
+        """
+        outputs = []
+        for input_val in inputs:
+            # Build messages
+            messages = [
+                {"role": "system", "content": hyperparameters["system_message"]},
+                {"role": "user", "content": str(input_val)},
+            ]
+
+            # Run inference
+            result = pipeline(messages)
+
+            # Extract the answer
+            output = str(result[0]["generated_text"][-1]["content"])
+            outputs.append(output)
+
+        return outputs
+
+    # Load evaluation dataset
+    dataset_path = Path(__file__).parent.parent / "3-evaluation_datasets" / "example_datasets" / "basic_questions.json"
+    dataset = EvalDataset.from_json(
+        path=str(dataset_path),
+        metrics="accuracy",
+        input="question",
+        label="answer",
+    )
+
+    # Step 1: Log in with your Trismik API key
+    # login() reads TRISMIK_API_KEY from environment variables or .env file
+    login()
+
+
+    # Step 2: Run evaluation with result uploading
+    # When you provide experiment_id and project_id, results are automatically uploaded
+    print(f"\nRunning evaluation with model: {model_name}")
+    print("Results will be uploaded to Trismik dashboard.\n")
+
+    results = evaluate(
+        inference,
+        dataset,
+        hyperparameters={
+            "system_message": "Answer the question directly and concisely.",
+        },
+        experiment_id="Uploading-Results-Example",  # Creates/uses this experiment
+        project_id="TRISMIK_PROJECT_ID", # TODO: ADD YOUR TRISMIK PROJECT ID
+        metadata={
+            "model": model_name,
+            "description": "Example evaluation demonstrating result uploading",
+        },
+        return_aggregates=True,
+        return_items=True,
+        return_output=True,
+    )
+
+    print("\nResults uploaded successfully!")
+    pprint(results)
+
+    return results
+
+
+if __name__ == "__main__":
+    load_dotenv()
+    log_file = setup_logging(experiment_id="2-uploading_evaluate_results", base_dir=Path(__file__).parent)
+    output_dir = Path(__file__).parent / "results"
+    output_dir.mkdir(exist_ok=True)
+    results_dict = main()
+    save_results_to_json(results_dict, output_dir, "2-uploading_evaluate_results_output.json")