scorebook 0.0.13__py3-none-any.whl → 0.0.15__py3-none-any.whl

tutorials/examples/1-score/3-scoring_model_f1.py

@@ -0,0 +1,64 @@
+"""Tutorials - Score - Example 3 - F1 Metric Scoring."""
+
+from pathlib import Path
+from pprint import pprint
+from typing import Any
+
+from dotenv import load_dotenv
+
+from tutorials.utils import save_results_to_json, setup_logging
+from scorebook import score
+from scorebook.metrics.f1 import F1
+
+
+def main() -> Any:
+    """Score pre-computed model predictions using F1 metric.
+
+    This example demonstrates how to score NER (Named Entity Recognition)
+    predictions using the F1 metric with different averaging methods.
+    """
+
+    # Sample NER predictions (in CoNLL format with BIO tags)
+    model_predictions = [
+        {"output": "O", "label": "O"},
+        {"output": "B-PER", "label": "B-PER"},
+        {"output": "I-PER", "label": "I-PER"},
+        {"output": "O", "label": "O"},
+        {"output": "B-LOC", "label": "B-LOC"},
+        {"output": "O", "label": "O"},
+        {"output": "B-ORG", "label": "B-LOC"},  # Misclassification
+        {"output": "O", "label": "B-MISC"},     # Missed entity
+        {"output": "B-PER", "label": "B-PER"},
+        {"output": "O", "label": "O"},
+    ]
+
+    print(f"Scoring {len(model_predictions)} NER predictions\n")
+
+    # Score with all averaging methods at once
+    print("All averaging methods:")
+    results_all = score(
+        items=model_predictions,
+        metrics=F1(average="all"),
+        upload_results=False,
+    )
+    pprint(results_all["aggregate_results"])
+
+    # Score with specific combination of methods
+    print("\nMicro and weighted averaging:")
+    results_combo = score(
+        items=model_predictions,
+        metrics=F1(average=["micro", "weighted"]),
+        upload_results=False,
+    )
+    pprint(results_combo["aggregate_results"])
+
+    return results_all
+
+
+if __name__ == "__main__":
+    load_dotenv()
+    log_file = setup_logging(experiment_id="3-scoring_f1_metric", base_dir=Path(__file__).parent)
+    output_dir = Path(__file__).parent / "results"
+    output_dir.mkdir(exist_ok=True)
+    results_dict = main()
+    save_results_to_json(results_dict, output_dir, "3-scoring_f1_metric_output.json")

tutorials/examples/1-score/4-scoring_model_rouge.py

@@ -0,0 +1,64 @@
+"""Tutorials - Score - Example 4 - Scoring Models with ROUGE."""
+
+from pathlib import Path
+from pprint import pprint
+from typing import Any
+
+from dotenv import load_dotenv
+
+from tutorials.utils import save_results_to_json, setup_logging
+
+from scorebook import score
+from scorebook.metrics.rouge import ROUGE
+
+
+def main() -> Any:
+    """Score text generation predictions using ROUGE metric.
+
+    This example demonstrates how to score generated summaries
+    against reference summaries using ROUGE scores.
+    """
+
+    # Prepare a list of items with generated summaries and reference summaries
+    model_predictions = [
+        {
+            "output": "A woman donated her kidney to a stranger. This sparked a chain of six kidney transplants.",
+            "label": "Zully Broussard decided to give a kidney to a stranger. A new computer program helped her donation spur transplants for six kidney patients.",
+        },
+        {
+            "output": "Scientists discovered a new species of frog in the Amazon rainforest. The frog has unique markings that distinguish it from other species.",
+            "label": "A new frog species with distinctive blue and yellow stripes was found in the Amazon. Researchers say this discovery highlights the biodiversity of the region.",
+        },
+        {
+            "output": "The technology company released its quarterly earnings report showing strong growth.",
+            "label": "Tech giant announces record quarterly revenue driven by cloud services and AI products.",
+        },
+    ]
+
+    # Score the predictions against labels using the ROUGE metric
+    results = score(
+        items=model_predictions,
+        metrics=ROUGE(rouge_types=["rouge1", "rougeL"], use_stemmer=True),
+        upload_results=False,  # Disable uploading for this example
+    )
+
+    print("\nResults:")
+    pprint(results)
+
+    # Display individual item scores
+    print("\n\nIndividual ROUGE Scores:")
+    for i, item_score in enumerate(results["item_results"]):
+        print(f"\nItem {i+1}:")
+        print(f"  ROUGE-1 F1: {item_score['rouge1']:.4f}")
+        print(f"  ROUGE-L F1: {item_score['rougeL']:.4f}")
+
+    return results
+
+
+if __name__ == "__main__":
+    load_dotenv()
+    log_file = setup_logging(experiment_id="4-scoring_model_rouge", base_dir=Path(__file__).parent)
+    output_dir = Path(__file__).parent / "results"
+    output_dir.mkdir(exist_ok=True)
+    results_dict = main()
+    save_results_to_json(results_dict, output_dir, "4-scoring_model_rouge_output.json")

tutorials/examples/1-score/5-scoring_model_exact_match.py

@@ -0,0 +1,84 @@
+"""Tutorials - Score - Example 5 - Scoring Models with Exact Match."""
+
+from pathlib import Path
+from pprint import pprint
+from typing import Any
+
+from dotenv import load_dotenv
+
+from tutorials.utils import save_results_to_json, setup_logging
+
+from scorebook import score
+from scorebook.metrics.exactmatch import ExactMatch
+
+
+def main() -> Any:
+    """Score text predictions using Exact Match metric.
+
+    This example demonstrates how to compare model outputs against
+    reference labels using exact string matching with configurable
+    preprocessing options.
+    """
+
+    # Prepare a list of items with model outputs and expected labels
+    # Note: outputs may have different casing or extra whitespace
+    model_predictions = [
+        {"output": "Paris", "label": "Paris"},           # Exact match
+        {"output": "LONDON", "label": "London"},         # Different case
+        {"output": "  Berlin  ", "label": "Berlin"},     # Extra whitespace
+        {"output": " NEW YORK ", "label": "new york"},   # Both case and whitespace
+        {"output": "Tokyo", "label": "Kyoto"},           # No match
+    ]
+
+    print(f"Scoring {len(model_predictions)} predictions\n")
+
+    # Score with default settings (case_insensitive=True, strip=True)
+    print("Default settings (case_insensitive=True, strip=True):")
+    results_default = score(
+        items=model_predictions,
+        metrics=ExactMatch(),
+        upload_results=False,
+    )
+    pprint(results_default["aggregate_results"])
+    print(f"Item matches: {[item['exact_match'] for item in results_default['item_results']]}")
+
+    # Score with case-sensitive matching
+    print("\nCase-sensitive matching (case_insensitive=False, strip=True):")
+    results_case_sensitive = score(
+        items=model_predictions,
+        metrics=ExactMatch(case_insensitive=False),
+        upload_results=False,
+    )
+    pprint(results_case_sensitive["aggregate_results"])
+    print(f"Item matches: {[item['exact_match'] for item in results_case_sensitive['item_results']]}")
+
+    # Score without stripping whitespace
+    print("\nWithout stripping (case_insensitive=True, strip=False):")
+    results_no_strip = score(
+        items=model_predictions,
+        metrics=ExactMatch(strip=False),
+        upload_results=False,
+    )
+    pprint(results_no_strip["aggregate_results"])
+    print(f"Item matches: {[item['exact_match'] for item in results_no_strip['item_results']]}")
+
+    # Score with strict matching (no preprocessing)
+    print("\nStrict matching (case_insensitive=False, strip=False):")
+    results_strict = score(
+        items=model_predictions,
+        metrics=ExactMatch(case_insensitive=False, strip=False),
+        upload_results=False,
+    )
+    pprint(results_strict["aggregate_results"])
+    print(f"Item matches: {[item['exact_match'] for item in results_strict['item_results']]}")
+
+    return results_default
+
+
+if __name__ == "__main__":
+    load_dotenv()
+    log_file = setup_logging(experiment_id="5-scoring_model_exact_match", base_dir=Path(__file__).parent)
+    output_dir = Path(__file__).parent / "results"
+    output_dir.mkdir(exist_ok=True)
+    results_dict = main()
+    save_results_to_json(results_dict, output_dir, "5-scoring_model_exact_match_output.json")

tutorials/examples/1-score/6-scoring_with_bertscore.py

@@ -0,0 +1,57 @@
+"""Tutorials - Score - Example 6 - Scoring with BertScore."""
+
+from pathlib import Path
+from pprint import pprint
+from typing import Any
+
+from dotenv import load_dotenv
+
+from scorebook.metrics.bertscore import BertScore
+
+from tutorials.utils import save_results_to_json, setup_logging
+from scorebook import score
+
+
+def main() -> Any:
+    """Score pre-computed model predictions using Scorebook.
+
+    This example demonstrates how to score generated model predictions.
+    """
+
+    # Prepare a list of items with generated summaries and reference summaries
+    model_predictions = [
+        {
+            "output": "A woman donated her kidney to a stranger. This sparked a chain of six kidney transplants.",
+            "label": "Zully Broussard decided to give a kidney to a stranger. A new computer program helped her donation spur transplants for six kidney patients.",
+        },
+        {
+            "output": "Scientists discovered a new species of frog in the Amazon rainforest. The frog has unique markings that distinguish it from other species.",
+            "label": "A new frog species with distinctive blue and yellow stripes was found in the Amazon. Researchers say this discovery highlights the biodiversity of the region.",
+        },
+        {
+            "output": "The technology company released its quarterly earnings report showing strong growth.",
+            "label": "Tech giant announces record quarterly revenue driven by cloud services and AI products.",
+        },
+    ]
+
+    # Score the predictions against labels using the BertScore metric
+    results = score(
+        items=model_predictions,
+        metrics=BertScore,
+        upload_results=False,  # Disable uploading for this example
+    )
+
+    print("\nResults:")
+    pprint(results)
+
+    return results
+
+
+if __name__ == "__main__":
+    load_dotenv()
+    log_file = setup_logging(experiment_id="6-scoring_model_bertscore", base_dir=Path(__file__).parent)
+    output_dir = Path(__file__).parent / "results"
+    output_dir.mkdir(exist_ok=True)
+    results_dict = main()
+    save_results_to_json(results_dict, output_dir, "6-scoring_model_bertscore_output.json")
+

tutorials/examples/2-evaluate/1-evaluating_local_models.py

@@ -0,0 +1,106 @@
+"""Tutorials - Evaluate - Example 1 - Evaluating Local Models."""
+
+from pathlib import Path
+from pprint import pprint
+from typing import Any, List
+
+import transformers
+from dotenv import load_dotenv
+
+from tutorials.utils import save_results_to_json, setup_logging
+
+from scorebook import EvalDataset, evaluate
+
+
+def main() -> Any:
+    """Run a simple Scorebook evaluation on a local model.
+
+    This example demonstrates the fundamental workflow for evaluating a model using Scorebook.
+
+    It shows how to:
+        1. Create an evaluation dataset from a list of evaluation items
+        2. Define an inference function using Hugging Face's transformers library
+        3. Run the evaluation and collect results
+
+    This serves as a starting point for understanding Scorebook's core evaluation capabilities.
+    """
+
+    # Create a list of evaluation items
+    evaluation_items = [
+        {"question": "What is 2 + 2?", "answer": "4"},
+        {"question": "What is the capital of France?", "answer": "Paris"},
+        {"question": "Who wrote Romeo and Juliet?", "answer": "William Shakespeare"},
+    ]
+
+    # Create an evaluation dataset
+    evaluation_dataset = EvalDataset.from_list(
+        name="basic_questions",    # Dataset name
+        metrics="accuracy",        # Metric/Metrics used to calculate scores
+        items=evaluation_items,    # List of evaluation items
+        input="question",          # Key for the input field in evaluation items
+        label="answer",            # Key for the label field in evaluation items
+    )
+
+    # Create a model
+    pipeline = transformers.pipeline(
+        "text-generation",
+        model="microsoft/Phi-4-mini-instruct",
+        model_kwargs={"torch_dtype": "auto"},
+        device_map="auto",
+    )
+
+    # Define an inference function
+    def inference(inputs: List[Any], **hyperparameters: Any) -> List[Any]:
+        """Return a list of model outputs for a list of inputs.
+
+        Args:
+            inputs: Input values from an EvalDataset.
+            hyperparameters: Model hyperparameters.
+
+        Returns:
+            The model outputs for a list of inputs.
+        """
+        inference_outputs = []
+        for model_input in inputs:
+
+            # Wrap inputs in the model's message format
+            messages = [
+                {
+                    "role": "system",
+                    "content": hyperparameters.get("system_message"),
+                },
+                {"role": "user", "content": model_input},
+            ]
+
+            # Run inference on the item
+            output = pipeline(messages, temperature=hyperparameters.get("temperature"))
+
+            # Extract and collect the output generated from the model's response
+            inference_outputs.append(output[0]["generated_text"][-1]["content"])
+
+        return inference_outputs
+
+    # Evaluate a model against an evaluation dataset
+    results = evaluate(
+        inference,
+        evaluation_dataset,
+        hyperparameters={
+            "temperature": 0.7,
+            "system_message": "Answer the question directly and concisely.",
+        },
+        return_items=True,
+        upload_results=False,  # Disable uploading for this example
+    )
+
+    print("\nEvaluation Results:")
+    pprint(results)
+    return results
+
+
+if __name__ == "__main__":
+    load_dotenv()
+    log_file = setup_logging(experiment_id="1-evaluating_local_models", base_dir=Path(__file__).parent)
+    output_dir = Path(__file__).parent / "results"
+    output_dir.mkdir(exist_ok=True)
+    results_dict = main()
+    save_results_to_json(results_dict, output_dir, "1-evaluating_local_models_output.json")

tutorials/examples/2-evaluate/2-evaluating_local_models_with_batching.py

@@ -0,0 +1,108 @@
+"""Tutorials - Evaluate - Example 2 - Evaluating Local Models with Batching."""
+
+from pathlib import Path
+from pprint import pprint
+from typing import Any, List
+
+import transformers
+from dotenv import load_dotenv
+
+from tutorials.utils import save_results_to_json, setup_logging
+
+from scorebook import EvalDataset, evaluate
+
+
+def main() -> Any:
+    """Run a Scorebook evaluation using local batch inference.
+
+    This example demonstrates how to perform batch inference locally.
+
+    This approach offers several benefits:
+        1. Improved throughput by processing multiple items in parallel
+        2. Better GPU utilization through batched tensor operations
+        3. More efficient memory usage compared to sequential processing
+    """
+
+    # Initialize the pipeline with appropriate settings for batch processing
+    model_name = "google/flan-t5-small"
+
+    # Task is text2text-generation for seq2seq models
+    pipeline = transformers.pipeline(
+        "text2text-generation",
+        model=model_name,
+        torch_dtype="auto",
+        device_map="auto",  # will pick up gpu if available
+    )
+
+    # Define a batch inference function
+    def inference(inputs: List[Any], **hyperparameters: Any) -> List[Any]:
+        """Process multiple inputs through the model in batches.
+
+        Args:
+            inputs: Input values from an EvalDataset.
+            hyperparameters: Model hyperparameters including batch_size and max_new_tokens.
+
+        Returns:
+            List of model outputs for all inputs.
+        """
+        # Preprocess: Convert inputs to strings
+        preprocessed_inputs = [str(input_val) for input_val in inputs]
+
+        # Run batch inference
+        raw_results = pipeline(
+            preprocessed_inputs,
+            batch_size=hyperparameters["batch_size"],
+            max_new_tokens=hyperparameters["max_new_tokens"],
+            pad_token_id=pipeline.tokenizer.eos_token_id,
+        )
+
+        # Postprocess: Extract and clean the generated text
+        final_outputs = [str(result["generated_text"]).strip() for result in raw_results]
+
+        return final_outputs
+
+    # Create a list of evaluation items
+    evaluation_items = [
+        {"question": "What is 2 + 2?", "answer": "4"},
+        {"question": "What is the capital of France?", "answer": "Paris"},
+        {"question": "Who wrote Romeo and Juliet?", "answer": "William Shakespeare"},
+    ]
+
+    # Create an evaluation dataset
+    evaluation_dataset = EvalDataset.from_list(
+        name="basic_questions",    # Dataset name
+        metrics="accuracy",        # Metric/Metrics used to calculate scores
+        items=evaluation_items,    # List of evaluation items
+        input="question",          # Key for the input field in evaluation items
+        label="answer",            # Key for the label field in evaluation items
+    )
+
+
+    # Define hyperparameters
+    hyperparameters = {
+        "max_new_tokens": 128,
+        "batch_size": 2,
+    }
+
+    # Run the evaluation with batch inference
+    results = evaluate(
+        inference,
+        evaluation_dataset,
+        hyperparameters=hyperparameters,
+        return_aggregates=True,  # Include aggregate results for each configuration
+        return_items=True,       # Include results for individual items
+        return_output=True,      # Include model outputs for debugging
+        upload_results=False,    # Disable uploading for this example
+    )
+
+    pprint(results)
+    return results
+
+
+if __name__ == "__main__":
+    load_dotenv()
+    log_file = setup_logging(experiment_id="2-evaluating_local_models_with_batching", base_dir=Path(__file__).parent)
+    output_dir = Path(__file__).parent / "results"
+    output_dir.mkdir(exist_ok=True)
+    results_dict = main()
+    save_results_to_json(results_dict, output_dir, "2-evaluating_local_models_with_batching_output.json")

tutorials/examples/2-evaluate/3-evaluating_cloud_models.py

@@ -0,0 +1,109 @@
+"""Tutorials - Evaluate - Example 3 - Evaluating Cloud Models."""
+
+import asyncio
+from pathlib import Path
+from pprint import pprint
+from typing import Any, List
+
+from dotenv import load_dotenv
+from openai import AsyncOpenAI
+
+from tutorials.utils import save_results_to_json, setup_logging
+
+from scorebook import EvalDataset, evaluate_async
+
+
+async def main() -> Any:
+    """Run an evaluation using a cloud-hosted model.
+
+    This example demonstrates how to evaluate cloud-hosted models using OpenAI's API directly.
+
+    Prerequisites:
+        - OpenAI API key set in environment variable OPENAI_API_KEY
+    """
+
+    # Initialize OpenAI client
+    client = AsyncOpenAI()
+    model_name = "gpt-4o-mini"
+
+    # Define an async inference function
+    async def inference(inputs: List[Any], **hyperparameters: Any) -> List[Any]:
+        """Process inputs through OpenAI's API.
+
+        Args:
+            inputs: Input values from an EvalDataset.
+            hyperparameters: Model hyperparameters including system_message and temperature.
+
+        Returns:
+            List of model outputs for all inputs.
+        """
+        outputs = []
+        for input_val in inputs:
+            # Build messages for OpenAI API
+            messages = [
+                {
+                    "role": "system",
+                    "content": hyperparameters.get(
+                        "system_message", "You are a helpful assistant."
+                    ),
+                },
+                {"role": "user", "content": str(input_val)},
+            ]
+
+            # Call OpenAI API
+            try:
+                response = await client.chat.completions.create(
+                    model=model_name,
+                    messages=messages,
+                    temperature=hyperparameters.get("temperature", 0.7),
+                )
+                output = response.choices[0].message.content.strip()
+            except Exception as e:
+                output = f"Error: {str(e)}"
+
+            outputs.append(output)
+
+        return outputs
+
+    # Create a list of evaluation items
+    evaluation_items = [
+        {"question": "What is 2 + 2?", "answer": "4"},
+        {"question": "What is the capital of France?", "answer": "Paris"},
+        {"question": "Who wrote Romeo and Juliet?", "answer": "William Shakespeare"},
+    ]
+
+    # Create an evaluation dataset
+    evaluation_dataset = EvalDataset.from_list(
+        name="basic_questions",    # Dataset name
+        metrics="accuracy",        # Metric/Metrics used to calculate scores
+        items=evaluation_items,    # List of evaluation items
+        input="question",          # Key for the input field in evaluation items
+        label="answer",            # Key for the label field in evaluation items
+    )
+
+    # Run evaluation
+    results = await evaluate_async(
+        inference,
+        evaluation_dataset,
+        hyperparameters={
+            "system_message": (
+                "Answer the question directly. Provide only the answer, without context."
+            ),
+            "temperature": 0.7,
+        },
+        return_items=True,
+        return_output=True,
+        upload_results=False,
+    )
+
+    pprint(results)
+    return results
+
+
+if __name__ == "__main__":
+    load_dotenv()
+    log_file = setup_logging(experiment_id="3-evaluating_cloud_models", base_dir=Path(__file__).parent)
+    output_dir = Path(__file__).parent / "results"
+    output_dir.mkdir(exist_ok=True)
+    results_dict = asyncio.run(main())
+    save_results_to_json(results_dict, output_dir, "3-evaluating_cloud_models_output.json")