scorebook 0.0.14__py3-none-any.whl → 0.0.16__py3-none-any.whl

tutorials/examples/4-adaptive_evaluations/1-adaptive_evaluation.py

@@ -0,0 +1,114 @@
+"""Tutorials - Adaptive Evaluations - Example 1 - Adaptive Evaluation."""
+
+import asyncio
+import string
+from pathlib import Path
+from pprint import pprint
+from typing import Any, List
+
+from dotenv import load_dotenv
+from openai import AsyncOpenAI
+
+from tutorials.utils import save_results_to_json, setup_logging
+
+from scorebook import evaluate_async, login
+
+
+async def main() -> Any:
+    """Run an adaptive evaluation using Trismik's adaptive testing.
+
+    This example demonstrates how to use Trismik's adaptive evaluation feature.
+    Adaptive evaluations use Item Response Theory (IRT) to efficiently estimate
+    model capabilities by selecting questions based on previous responses.
+
+    Benefits of adaptive evaluation:
+        - More efficient: Fewer questions needed to assess capability
+        - Precise measurement: Better statistical confidence intervals
+        - Optimal difficulty: Questions adapt to model's skill level
+
+    Prerequisites:
+        - Valid Trismik API key set in TRISMIK_API_KEY environment variable
+        - A Trismik project ID
+        - OpenAI API key set in OPENAI_API_KEY environment variable
+    """
+
+    # Initialize OpenAI client
+    client = AsyncOpenAI()
+    model_name = "gpt-4o-mini"
+
+    # Define an async inference function
+    async def inference(inputs: List[Any], **hyperparameters: Any) -> List[Any]:
+        """Process inputs through OpenAI's API.
+
+        Args:
+            inputs: Input values from an EvalDataset. For adaptive MMLU-Pro,
+                   each input is a dict with 'question' and 'options' keys.
+            hyperparameters: Model hyperparameters.
+
+        Returns:
+            List of model outputs for all inputs.
+        """
+        outputs = []
+        for input_val in inputs:
+            # Handle dict input from adaptive dataset
+            if isinstance(input_val, dict):
+                prompt = input_val.get("question", "")
+                if "options" in input_val:
+                    prompt += "\nOptions:\n" + "\n".join(
+                        f"{letter}: {choice}"
+                        for letter, choice in zip(string.ascii_uppercase, input_val["options"])
+                    )
+            else:
+                prompt = str(input_val)
+
+            # Build messages for OpenAI API
+            messages = [
+                {
+                    "role": "system",
+                    "content": "Answer the question with a single letter representing the correct answer from the list of choices. Do not provide any additional explanation or output beyond the single letter.",
+                },
+                {"role": "user", "content": prompt},
+            ]
+
+            # Call OpenAI API
+            try:
+                response = await client.chat.completions.create(
+                    model=model_name,
+                    messages=messages,
+                    temperature=0.7,
+                )
+                output = response.choices[0].message.content.strip()
+            except Exception as e:
+                output = f"Error: {str(e)}"
+
+            outputs.append(output)
+
+        return outputs
+
+    # Step 1: Log in with your Trismik API key
+    # login() reads TRISMIK_API_KEY from environment variables or .env file
+    login()
+
+    # Step 2: Run adaptive evaluation
+    results = await evaluate_async(
+        inference,
+        datasets="trismik/headQA:adaptive",  # Adaptive datasets have the ":adaptive" suffix
+        experiment_id="Adaptive-Head-QA-Evaluation",
+        project_id='TRISMIK-PROJECT-ID',
+        return_dict=True,
+        return_aggregates=True,
+        return_items=True,
+        return_output=True,
+    )
+
+    pprint(results)
+    return results
+
+
+if __name__ == "__main__":
+    load_dotenv()
+    log_file = setup_logging(experiment_id="1-adaptive_evaluation", base_dir=Path(__file__).parent)
+    output_dir = Path(__file__).parent / "results"
+    output_dir.mkdir(exist_ok=True)
+    results_dict = asyncio.run(main())
+    save_results_to_json(results_dict, output_dir, "1-adaptive_evaluation_output.json")

tutorials/examples/4-adaptive_evaluations/2-adaptive_dataset_splits.py

@@ -0,0 +1,106 @@
+"""Tutorials - Adaptive Evaluations - Example 2 - Adaptive Dataset Splits."""
+
+import asyncio
+import string
+from pathlib import Path
+from pprint import pprint
+from typing import Any, List
+
+from dotenv import load_dotenv
+from openai import AsyncOpenAI
+
+from tutorials.utils import save_results_to_json, setup_logging
+
+from scorebook import evaluate_async, login
+
+
+async def main() -> Any:
+    """
+
+    Prerequisites:
+        - Valid Trismik API key set in TRISMIK_API_KEY environment variable
+        - A Trismik project ID
+        - OpenAI API key set in OPENAI_API_KEY environment variable
+    """
+
+    # Initialize OpenAI client
+    client = AsyncOpenAI()
+    model_name = "gpt-4o-mini"
+
+    # Define an async inference function
+    async def inference(inputs: List[Any], **hyperparameters: Any) -> List[Any]:
+        """Process inputs through OpenAI's API.
+
+        Args:
+            inputs: Input values from an EvalDataset. For adaptive headQA,
+                   each input is a dict with 'question' and 'options' keys.
+            hyperparameters: Model hyperparameters.
+
+        Returns:
+            List of model outputs for all inputs.
+        """
+        outputs = []
+        for input_val in inputs:
+            # Handle dict input from adaptive dataset
+            if isinstance(input_val, dict):
+                prompt = input_val.get("question", "")
+                if "options" in input_val:
+                    prompt += "\nOptions:\n" + "\n".join(
+                        f"{letter}: {choice}"
+                        for letter, choice in zip(string.ascii_uppercase, input_val["options"])
+                    )
+            else:
+                prompt = str(input_val)
+
+            # Build messages for OpenAI API
+            messages = [
+                {
+                    "role": "system",
+                    "content": "Answer the question with a single letter representing the correct answer from the list of choices. Do not provide any additional explanation or output beyond the single letter.",
+                },
+                {"role": "user", "content": prompt},
+            ]
+
+            # Call OpenAI API
+            try:
+                response = await client.chat.completions.create(
+                    model=model_name,
+                    messages=messages,
+                    temperature=0.7,
+                )
+                output = response.choices[0].message.content.strip()
+            except Exception as e:
+                output = f"Error: {str(e)}"
+
+            outputs.append(output)
+
+        return outputs
+
+    # Step 1: Log in with your Trismik API key
+    # login() reads TRISMIK_API_KEY from environment variables or .env file
+    login()
+
+    # Step 2: Run adaptive evaluation
+    results = await evaluate_async(
+        inference,
+        datasets="trismik/headQA:adaptive",  # Adaptive datasets have the ":adaptive" suffix
+        split="test",                        # Specify the test split for evaluation
+        experiment_id="Adaptive-Head-QA-Test-Evaluation",
+        project_id='TRISMIK-PROJECT-ID',
+        return_dict=True,
+        return_aggregates=True,
+        return_items=True,
+        return_output=True,
+    )
+
+    pprint(results)
+    return results
+
+
+if __name__ == "__main__":
+    load_dotenv()
+    log_file = setup_logging(experiment_id="2-adaptive_dataset_splits", base_dir=Path(__file__).parent)
+    output_dir = Path(__file__).parent / "results"
+    output_dir.mkdir(exist_ok=True)
+    results_dict = asyncio.run(main())
+    save_results_to_json(results_dict, output_dir, "2-adaptive_dataset_splits_output.json")

tutorials/examples/5-upload_results/1-uploading_score_results.py

@@ -0,0 +1,92 @@
+"""Tutorials - Upload Results - Example 1 - Uploading score() Results."""
+
+from pathlib import Path
+from pprint import pprint
+from typing import Any
+
+from dotenv import load_dotenv
+
+from tutorials.utils import save_results_to_json, setup_logging
+
+from scorebook import login, score
+from scorebook.metrics.accuracy import Accuracy
+
+
+def main() -> Any:
+    """Score pre-computed outputs and upload results to Trismik's dashboard.
+
+    This example demonstrates how to upload score() results to Trismik.
+    The score() function is used when you already have model outputs and
+    want to score them against labels.
+
+    Use score() when you want to:
+        - Score pre-computed model outputs
+        - Re-score existing results with different metrics
+        - Upload scoring results without re-running inference
+
+    Prerequisites:
+        - Valid Trismik API key set in TRISMIK_API_KEY environment variable
+        - A Trismik project ID
+    """
+
+    # Prepare items with pre-computed outputs and labels
+    items = [
+        {
+            "input": "What is 2 + 2?",
+            "output": "4",
+            "label": "4"
+        },
+        {
+            "input": "What is the capital of France?",
+            "output": "Paris",
+            "label": "Paris"
+        },
+        {
+            "input": "Who wrote Romeo and Juliet?",
+            "output": "William Shakespeare",
+            "label": "William Shakespeare"
+        },
+        {
+            "input": "What is 5 * 6?",
+            "output": "30",
+            "label": "30"
+        },
+        {
+            "input": "What is the largest planet in our solar system?",
+            "output": "Jupiter",
+            "label": "Jupiter"
+        },
+    ]
+
+    # Step 1: Log in with your Trismik API key
+    # login() reads TRISMIK_API_KEY from environment variables or .env file
+    login()
+
+    # Step 2: Score the outputs and upload results
+    # When you provide experiment_id and project_id, results are automatically uploaded
+    results = score(
+        items=items,
+        metrics=Accuracy,
+        dataset_name="basic_questions",
+        model_name="gpt-4o-mini",
+        experiment_id="Score-Upload-Example",
+        project_id="TRISMIK_PROJECT_ID", # TODO: ADD YOUR TRISMIK PROJECT ID
+        metadata={
+            "description": "Example demonstrating score() result uploading",
+            "note": "These are pre-computed outputs",
+        },
+        upload_results=True,  # Explicitly enable uploading
+    )
+
+    print("\nResults uploaded successfully!")
+    pprint(results)
+    return results
+
+
+if __name__ == "__main__":
+    load_dotenv()
+    log_file = setup_logging(experiment_id="1-uploading_score_results", base_dir=Path(__file__).parent)
+    output_dir = Path(__file__).parent / "results"
+    output_dir.mkdir(exist_ok=True)
+    results_dict = main()
+    save_results_to_json(results_dict, output_dir, "1-uploading_score_results_output.json")

tutorials/examples/5-upload_results/2-uploading_evaluate_results.py

@@ -0,0 +1,117 @@
+"""Tutorials - Upload Results - Example 2 - Uploading evaluate() Results."""
+
+from pathlib import Path
+from pprint import pprint
+from typing import Any, List
+
+import transformers
+from dotenv import load_dotenv
+
+from tutorials.utils import save_results_to_json, setup_logging
+
+from scorebook import EvalDataset, evaluate, login
+
+
+def main() -> Any:
+    """Run an evaluation and upload results to Trismik's dashboard.
+
+    This example demonstrates how to upload evaluate() results to Trismik.
+    The evaluate() function runs inference on a dataset and automatically
+    uploads the results when you provide experiment_id and project_id.
+
+    Use evaluate() when you want to:
+        - Run inference AND score in one step
+        - Track full evaluation runs with hyperparameters
+        - Compare different models on the same dataset
+
+    Prerequisites:
+        - Valid Trismik API key set in TRISMIK_API_KEY environment variable
+        - A Trismik project ID
+    """
+
+    # Initialize HuggingFace model pipeline
+    model_name = "microsoft/Phi-4-mini-instruct"
+    pipeline = transformers.pipeline(
+        "text-generation",
+        model=model_name,
+        model_kwargs={"torch_dtype": "auto"},
+        device_map="auto",
+    )
+
+    # Define an inference function
+    def inference(inputs: List[Any], **hyperparameters: Any) -> List[Any]:
+        """Process inputs through the model.
+
+        Args:
+            inputs: Input values from an EvalDataset.
+            hyperparameters: Model hyperparameters.
+
+        Returns:
+            List of model outputs for all inputs.
+        """
+        outputs = []
+        for input_val in inputs:
+            # Build messages
+            messages = [
+                {"role": "system", "content": hyperparameters["system_message"]},
+                {"role": "user", "content": str(input_val)},
+            ]
+
+            # Run inference
+            result = pipeline(messages)
+
+            # Extract the answer
+            output = str(result[0]["generated_text"][-1]["content"])
+            outputs.append(output)
+
+        return outputs
+
+    # Load evaluation dataset
+    dataset_path = Path(__file__).parent.parent / "3-evaluation_datasets" / "example_datasets" / "basic_questions.json"
+    dataset = EvalDataset.from_json(
+        path=str(dataset_path),
+        metrics="accuracy",
+        input="question",
+        label="answer",
+    )
+
+    # Step 1: Log in with your Trismik API key
+    # login() reads TRISMIK_API_KEY from environment variables or .env file
+    login()
+
+
+    # Step 2: Run evaluation with result uploading
+    # When you provide experiment_id and project_id, results are automatically uploaded
+    print(f"\nRunning evaluation with model: {model_name}")
+    print("Results will be uploaded to Trismik dashboard.\n")
+
+    results = evaluate(
+        inference,
+        dataset,
+        hyperparameters={
+            "system_message": "Answer the question directly and concisely.",
+        },
+        experiment_id="Uploading-Results-Example",  # Creates/uses this experiment
+        project_id="TRISMIK_PROJECT_ID", # TODO: ADD YOUR TRISMIK PROJECT ID
+        metadata={
+            "model": model_name,
+            "description": "Example evaluation demonstrating result uploading",
+        },
+        return_aggregates=True,
+        return_items=True,
+        return_output=True,
+    )
+
+    print("\nResults uploaded successfully!")
+    pprint(results)
+
+    return results
+
+
+if __name__ == "__main__":
+    load_dotenv()
+    log_file = setup_logging(experiment_id="2-uploading_evaluate_results", base_dir=Path(__file__).parent)
+    output_dir = Path(__file__).parent / "results"
+    output_dir.mkdir(exist_ok=True)
+    results_dict = main()
+    save_results_to_json(results_dict, output_dir, "2-uploading_evaluate_results_output.json")

tutorials/examples/5-upload_results/3-uploading_your_results.py

@@ -0,0 +1,153 @@
+"""Tutorials - Upload Results - Example 3 - Uploading Pre-Scored Results."""
+
+from pathlib import Path
+from pprint import pprint
+from typing import Any
+
+from dotenv import load_dotenv
+
+from tutorials.utils import save_results_to_json, setup_logging
+
+from scorebook import login, upload_result
+
+
+def main() -> Any:
+    """Upload pre-scored results directly to Trismik's dashboard.
+
+    This example demonstrates how to upload results where metrics are ALREADY computed.
+    This is different from score() or evaluate() which compute metrics for you.
+
+    Use upload_result() when you:
+        - Already have metric scores calculated
+        - Used a custom evaluation framework that computed metrics
+        - Want to import historical evaluation data with existing scores
+        - Have results from external tools (e.g., other eval frameworks)
+
+    The key difference from Examples 1 & 2:
+        - Example 1 (score): You have outputs/labels → Scorebook computes metrics
+        - Example 2 (evaluate): Scorebook runs inference AND computes metrics
+        - Example 3 (upload_result): You have EVERYTHING including metrics → Just upload
+
+    Prerequisites:
+        - Valid Trismik API key set in TRISMIK_API_KEY environment variable
+        - A Trismik project ID
+    """
+
+    # Step 1: Log in with your Trismik API key
+    # login() reads TRISMIK_API_KEY from environment variables or .env file
+    login()
+
+    # Step 2: Format your pre-scored results
+    # This is the structure that upload_result() expects:
+    # - aggregate_results: List with one dict containing overall metrics
+    # - item_results: List of dicts with per-item data and metric scores
+
+    # Example: You already ran an evaluation with your custom framework
+    # and computed accuracy, f1_score, etc.
+    my_pre_scored_results = {
+        "aggregate_results": [
+            {
+                "dataset": "spanish_translation",
+                "accuracy": 0.8,  # Your pre-computed aggregate accuracy
+                "bleu_score": 0.75,  # Your pre-computed BLEU score
+                # Add any hyperparameters used (optional)
+                "temperature": 0.7,
+                "max_tokens": 100,
+            }
+        ],
+        "item_results": [
+            {
+                "id": 0,
+                "dataset": "spanish_translation",
+                "input": "Translate 'hello' to Spanish",
+                "output": "hola",
+                "label": "hola",
+                "accuracy": 1.0,  # Item-level metric scores
+                "bleu_score": 1.0,
+                "temperature": 0.7,
+                "max_tokens": 100,
+            },
+            {
+                "id": 1,
+                "dataset": "spanish_translation",
+                "input": "Translate 'goodbye' to Spanish",
+                "output": "adiós",
+                "label": "adiós",
+                "accuracy": 1.0,
+                "bleu_score": 0.95,
+                "temperature": 0.7,
+                "max_tokens": 100,
+            },
+            {
+                "id": 2,
+                "dataset": "spanish_translation",
+                "input": "Translate 'thank you' to Spanish",
+                "output": "gracias",
+                "label": "gracias",
+                "accuracy": 1.0,
+                "bleu_score": 1.0,
+                "temperature": 0.7,
+                "max_tokens": 100,
+            },
+            {
+                "id": 3,
+                "dataset": "spanish_translation",
+                "input": "Translate 'please' to Spanish",
+                "output": "por favor",
+                "label": "por favor",
+                "accuracy": 1.0,
+                "bleu_score": 1.0,
+                "temperature": 0.7,
+                "max_tokens": 100,
+            },
+            {
+                "id": 4,
+                "dataset": "spanish_translation",
+                "input": "Translate 'good morning' to Spanish",
+                "output": "buenos dias",  # Missing accent - wrong answer
+                "label": "buenos días",
+                "accuracy": 0.0,
+                "bleu_score": 0.85,
+                "temperature": 0.7,
+                "max_tokens": 100,
+            },
+        ],
+    }
+
+    # Step 3: Upload your pre-scored results directly
+    print("\nUploading pre-scored results to Trismik...")
+    print("Metrics are already computed - just uploading to dashboard.\n")
+
+    run_id = upload_result(
+        run_result=my_pre_scored_results,
+        experiment_id="Pre-Scored-Results-Example",
+        project_id="TRISMIK_PROJECT_ID", # TODO: ADD YOUR TRISMIK PROJECT ID
+        dataset_name="spanish_translation",
+        hyperparameters={
+            "temperature": 0.7,
+            "max_tokens": 100,
+        },
+        metadata={
+            "description": "Results with pre-computed metrics from custom framework",
+            "source": "Custom evaluation tool",
+            "evaluation_date": "2025-01-15",
+        },
+        model_name="my-custom-translator-v2",
+    )
+
+    print(f"\nResults uploaded successfully with run_id: {run_id}")
+
+    # Add run_id to results for reference
+    my_pre_scored_results["run_id"] = run_id
+
+    pprint(my_pre_scored_results)
+    return my_pre_scored_results
+
+
+if __name__ == "__main__":
+    load_dotenv()
+    log_file = setup_logging(experiment_id="3-uploading_your_results", base_dir=Path(__file__).parent)
+    output_dir = Path(__file__).parent / "results"
+    output_dir.mkdir(exist_ok=True)
+    results_dict = main()
+    save_results_to_json(results_dict, output_dir, "3-uploading_your_results_output.json")

tutorials/examples/6-providers/aws/__init__.py

@@ -0,0 +1 @@
+"""Example usage of the AWS API with the scorebook library."""