scorebook 0.0.1__py3-none-any.whl

scorebook/utils/__init__.py

@@ -0,0 +1,8 @@
+"""Utility functions and common helpers for the Scorebook framework."""
+
+from scorebook.utils.async_utils import is_awaitable
+from scorebook.utils.io_helpers import validate_path
+from scorebook.utils.progress_bars import evaluation_progress
+from scorebook.utils.transform_helpers import expand_dict
+
+__all__ = ["is_awaitable", "validate_path", "expand_dict", "evaluation_progress"]

scorebook/utils/async_utils.py

@@ -0,0 +1,27 @@
+"""Async utilities for handling callable objects and coroutines."""
+
+import asyncio
+from typing import Callable
+
+
+def is_awaitable(obj: Callable) -> bool:
+    """
+    Check if a callable object is awaitable.
+
+    This handles both coroutine functions and callable instances (like classes
+    with __call__ methods) that may return coroutines.
+
+    Args:
+        obj: The callable object to check
+
+    Returns:
+        True if the object is awaitable, False otherwise
+    """
+    if asyncio.iscoroutinefunction(obj):
+        return True
+
+    # Check if it's a callable instance with an awaitable __call__ method
+    if hasattr(obj, "__call__") and asyncio.iscoroutinefunction(obj.__call__):
+        return True
+
+    return False

scorebook/utils/io_helpers.py

@@ -0,0 +1,28 @@
+"""Input/output helper functions for Scorebook."""
+
+from pathlib import Path
+from typing import Optional
+
+
+def validate_path(file_path: str, expected_suffix: Optional[str] = None) -> Path:
+    """Validate that a file path exists and optionally check its suffix.
+
+    Args:
+        file_path: Path to the file as string or Path object
+        expected_suffix: Optional file extension to validate (e.g. ".json", ".csv")
+
+    Returns:
+        Path object for the validated file path
+
+    Raises:
+        FileNotFoundError: If the file does not exist
+        ValueError: If the file has the wrong extension
+    """
+    path = Path(file_path)
+    if not path.exists():
+        raise FileNotFoundError(f"File not found: {file_path}")
+
+    if expected_suffix and path.suffix.lower() != expected_suffix.lower():
+        raise ValueError(f"File must have {expected_suffix} extension, got: {path.suffix}")
+
+    return path

scorebook/utils/mappers.py

@@ -0,0 +1,36 @@
+"""Utility functions for mapping and converting data types in Scorebook."""
+
+from typing import Any, Literal
+
+ClassificationResult = Literal["true_positive", "false_positive", "true_negative", "false_negative"]
+
+
+def to_binary(value: Any) -> int:
+    """Transform various input types to binary (0/1) classification value."""
+    if value is None:
+        return 0
+    if isinstance(value, str):
+        if value.upper() in ["A", "1", "TRUE", "YES", "Y"]:
+            return 1
+        return 0
+    return 1 if value else 0
+
+
+def to_binary_classification(prediction: Any, reference: Any) -> ClassificationResult:
+    """
+    Determine classification result based on prediction and reference values.
+
+    Args:
+        prediction: Predicted value (will be converted to binary)
+        reference: Reference/true value (will be converted to binary)
+
+    Returns:
+        Classification result as one of: "true_positive", "false_positive",
+                                       "true_negative", "false_negative"
+    """
+    pred_binary = to_binary(prediction)
+    ref_binary = to_binary(reference)
+
+    if pred_binary == 1:
+        return "true_positive" if ref_binary == 1 else "false_positive"
+    return "false_negative" if ref_binary == 1 else "true_negative"

scorebook/utils/progress_bars.py

@@ -0,0 +1,89 @@
+"""Progress bar utilities for evaluation tracking."""
+
+from contextlib import contextmanager
+from typing import Any, Generator, List, Optional
+
+from tqdm import tqdm
+
+
+class EvaluationProgressBars:
+    """Manages nested progress bars for evaluation tracking."""
+
+    def __init__(self, datasets: List[Any], hyperparam_count: int) -> None:
+        """Initialize progress bar manager.
+
+        Args:
+            datasets: List of datasets being evaluated
+            hyperparam_count: Number of hyperparameter configurations per dataset
+        """
+        self.datasets = datasets
+        self.hyperparam_count = hyperparam_count
+        self.dataset_pbar: Optional[tqdm] = None
+        self.hyperparam_pbar: Optional[tqdm] = None
+
+    def start_dataset_progress(self) -> None:
+        """Start the outer progress bar for datasets."""
+        self.dataset_pbar = tqdm(
+            total=len(self.datasets),
+            desc="Datasets   ",
+            unit="dataset",
+            position=0,
+            leave=True,
+            ncols=80,
+            bar_format="{desc} {percentage:3.0f}%|{bar:40}| {n_fmt}/{total_fmt}",
+        )
+
+    def update_dataset_progress(self) -> None:
+        """Update the dataset progress bar."""
+        if self.dataset_pbar:
+            self.dataset_pbar.update(1)
+
+    def close_dataset_progress(self) -> None:
+        """Close the dataset progress bar."""
+        if self.dataset_pbar:
+            self.dataset_pbar.close()
+            self.dataset_pbar = None
+
+    @contextmanager
+    def hyperparam_progress_context(self) -> Generator[tqdm, None, None]:
+        """Context manager for hyperparameter progress bar."""
+        self.hyperparam_pbar = tqdm(
+            total=self.hyperparam_count,
+            desc="Hyperparams",
+            unit="config",
+            position=1,
+            leave=False,
+            ncols=80,
+            bar_format="{desc} {percentage:3.0f}%|{bar:40}| {n_fmt}/{total_fmt}",
+        )
+        try:
+            yield self.hyperparam_pbar
+        finally:
+            self.hyperparam_pbar.close()
+            self.hyperparam_pbar = None
+
+    def update_hyperparam_progress(self) -> None:
+        """Update the hyperparameter progress bar."""
+        if self.hyperparam_pbar:
+            self.hyperparam_pbar.update(1)
+
+
+@contextmanager
+def evaluation_progress(
+    datasets: List[Any], hyperparam_count: int
+) -> Generator[EvaluationProgressBars, None, None]:
+    """Context manager for evaluation progress bars.
+
+    Args:
+        datasets: List of datasets being evaluated
+        hyperparam_count: Number of hyperparameter configurations per dataset
+
+    Yields:
+        EvaluationProgressBars: Progress bar manager instance
+    """
+    progress_bars = EvaluationProgressBars(datasets, hyperparam_count)
+    progress_bars.start_dataset_progress()
+    try:
+        yield progress_bars
+    finally:
+        progress_bars.close_dataset_progress()

scorebook/utils/transform_helpers.py

@@ -0,0 +1,25 @@
+"""Utility functions for transforming and manipulating data structures."""
+
+from itertools import product
+
+
+def expand_dict(data: dict) -> list[dict]:
+    """Expand a dictionary with list values into multiple dictionaries.
+
+    Takes a dictionary that may contain list values and expands it into a list of dictionaries,
+    where each dictionary represents one possible combination of values from the lists.
+    Non-list values remain constant across all generated dictionaries.
+
+    Args:
+        data: A dictionary potentially containing list values to be expanded
+
+    Returns:
+        A list of dictionaries representing all possible combinations of the input values
+    """
+    fixed = {k: v for k, v in data.items() if not isinstance(v, list)}
+    expandables = {k: v for k, v in data.items() if isinstance(v, list)}
+
+    keys, values = zip(*expandables.items()) if expandables else ([], [])
+    combinations = product(*values) if values else [()]
+
+    return [{**fixed, **dict(zip(keys, combo))} for combo in combinations]

scorebook-0.0.1.dist-info/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Trismik Ltd
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

scorebook-0.0.1.dist-info/METADATA

@@ -0,0 +1,376 @@
+Metadata-Version: 2.3
+Name: scorebook
+Version: 0.0.1
+Summary: A Python project for LLM evaluation.
+Author: Euan Campbell
+Author-email: euan@trismik.com
+Requires-Python: >=3.9
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Provides-Extra: examples
+Provides-Extra: openai
+Provides-Extra: portkey
+Requires-Dist: accelerate ; extra == "examples"
+Requires-Dist: datasets (>=3.6.0)
+Requires-Dist: notebook ; extra == "examples"
+Requires-Dist: openai ; extra == "openai"
+Requires-Dist: portkey-ai ; extra == "portkey"
+Requires-Dist: python-dotenv ; extra == "openai"
+Requires-Dist: python-dotenv ; extra == "portkey"
+Requires-Dist: torch ; extra == "examples"
+Requires-Dist: torchaudio ; extra == "examples"
+Requires-Dist: torchvision ; extra == "examples"
+Requires-Dist: transformers ; extra == "examples"
+Description-Content-Type: text/markdown
+
+# Scorebook
+
+**A Python library for LLM evaluation**
+
+<p align="center">
+  <img alt="Dynamic TOML Badge" src="https://img.shields.io/badge/dynamic/toml?url=https%3A%2F%2Fraw.githubusercontent.com%2Ftrismik%2Fscorebook%2Frefs%2Fheads%2Fmain%2Fpyproject.toml&query=tool.poetry.version&style=flat&label=version">
+  <img alt="Python Version" src="https://img.shields.io/badge/python-3.9%2B-blue">
+  <img alt="License" src="https://img.shields.io/badge/license-MIT-green">
+</p>
+
+Scorebook is a flexible and extensible framework for evaluating Large Language Models (LLMs). It provides clear contracts for data loading, model inference, and metrics computation, making it easy to run comprehensive evaluations across different datasets, models, and metrics.
+
+## ✨ Key Features
+
+- **🔌 Flexible Data Loading**: Support for Hugging Face datasets, CSV, JSON, and Python lists
+- **🚀 Model Agnostic**: Works with any model or inference provider
+- **📊 Extensible Metric Engine**: Use the metrics we provide or implement your own
+- **🔄 Automated Sweeping**: Test multiple model configurations automatically
+- **📈 Rich Results**: Export results to JSON, CSV, or structured formats like pandas DataFrames
+
+## 🚀 Quick Start
+
+### Installation
+
+```bash
+pip install scorebook
+```
+
+For OpenAI integration:
+```bash
+pip install scorebook[openai]
+```
+
+For local model examples:
+```bash
+pip install scorebook[examples]
+```
+
+### Basic Usage
+
+```python
+from scorebook import EvalDataset, evaluate
+from scorebook.metrics import Accuracy
+
+# 1. Create an evaluation dataset
+data = [
+    {"question": "What is 2 + 2?", "answer": "4"},
+    {"question": "What is the capital of France?", "answer": "Paris"},
+    {"question": "Who wrote Romeo and Juliet?", "answer": "William Shakespeare"}
+]
+
+dataset = EvalDataset.from_list(
+    name="basic_qa",
+    label="answer",
+    metrics=[Accuracy],
+    data=data
+)
+
+# 2. Define your inference function
+def my_inference_function(items, **hyperparameters):
+    # Your model logic here
+    predictions = []
+    for item in items:
+        # Process each item and generate prediction
+        prediction = your_model.predict(item["question"])
+        predictions.append(prediction)
+    return predictions
+
+# 3. Run evaluation
+results = evaluate(my_inference_function, dataset)
+print(results)
+```
+
+## 📊 Core Components
+
+### 1. Evaluation Datasets
+
+Scorebook supports multiple data sources through the `EvalDataset` class:
+
+#### From Hugging Face
+```python
+dataset = EvalDataset.from_huggingface(
+    "TIGER-Lab/MMLU-Pro",
+    label="answer",
+    metrics=[Accuracy],
+    split="validation"
+)
+```
+
+#### From CSV
+```python
+dataset = EvalDataset.from_csv(
+    "dataset.csv",
+    label="answer",
+    metrics=[Accuracy]
+)
+```
+
+#### From JSON
+```python
+dataset = EvalDataset.from_json(
+    "dataset.json",
+    label="answer",
+    metrics=[Accuracy]
+)
+```
+
+#### From Python List
+```python
+dataset = EvalDataset.from_list(
+    name="custom_dataset",
+    label="answer",
+    metrics=[Accuracy],
+    data=[{"question": "...", "answer": "..."}]
+)
+```
+
+### 2. Model Integration
+
+Scorebook offers two approaches for model integration:
+
+#### Inference Functions
+A single function that handles the complete pipeline:
+
+```python
+def inference_function(eval_items, **hyperparameters):
+    results = []
+    for item in eval_items:
+        # 1. Preprocessing
+        prompt = format_prompt(item)
+
+        # 2. Inference
+        output = model.generate(prompt)
+
+        # 3. Postprocessing
+        prediction = extract_answer(output)
+        results.append(prediction)
+
+    return results
+```
+
+#### Inference Pipelines
+Modular approach with separate stages:
+
+```python
+from scorebook.types.inference_pipeline import InferencePipeline
+
+def preprocessor(item):
+    return {"messages": [{"role": "user", "content": item["question"]}]}
+
+def inference_function(processed_items, **hyperparameters):
+    return [model.generate(item) for item in processed_items]
+
+def postprocessor(output):
+    return output.strip()
+
+pipeline = InferencePipeline(
+    model="my-model",
+    preprocessor=preprocessor,
+    inference_function=inference_function,
+    postprocessor=postprocessor
+)
+
+results = evaluate(pipeline, dataset)
+```
+
+### 3. Metrics System
+
+#### Built-in Metrics
+- **Accuracy**: Percentage of correct predictions
+- **Precision**: Accuracy of positive predictions
+
+```python
+from scorebook.metrics import Accuracy, Precision
+
+dataset = EvalDataset.from_list(
+    name="test",
+    label="answer",
+    metrics=[Accuracy, Precision],  # Multiple metrics
+    data=data
+)
+```
+
+#### Custom Metrics
+Create custom metrics by extending `MetricBase`:
+
+```python
+from scorebook.metrics import MetricBase, MetricRegistry
+
+@MetricRegistry.register()
+class F1Score(MetricBase):
+    @staticmethod
+    def score(outputs, labels):
+        # Calculate F1 score
+        item_scores = [calculate_f1_item(o, l) for o, l in zip(outputs, labels)]
+        aggregate_score = {"f1": sum(item_scores) / len(item_scores)}
+        return aggregate_score, item_scores
+
+# Use by string name or class
+dataset = EvalDataset.from_list(..., metrics=["f1score"])
+# or
+dataset = EvalDataset.from_list(..., metrics=[F1Score])
+```
+
+### 4. Hyperparameter Sweeping
+
+Test multiple configurations automatically:
+
+```python
+hyperparameters = {
+    "temperature": [0.7, 0.9, 1.0],
+    "max_tokens": [50, 100, 150],
+    "top_p": [0.8, 0.9]
+}
+
+results = evaluate(
+    inference_function,
+    dataset,
+    hyperparameters=hyperparameters,
+    score_type="all"
+)
+
+# Results include all combinations: 3 × 3 × 2 = 18 configurations
+```
+
+### 5. Results and Export
+
+Control result format with `score_type`:
+
+```python
+# Only aggregate scores (default)
+results = evaluate(model, dataset, score_type="aggregate")
+
+# Only per-item scores
+results = evaluate(model, dataset, score_type="item")
+
+# Both aggregate and per-item
+results = evaluate(model, dataset, score_type="all")
+```
+
+Export results:
+
+```python
+# Get EvalResult objects for advanced usage
+results = evaluate(model, dataset, return_type="object")
+
+# Export to files
+for result in results:
+    result.to_json("results.json")
+    result.to_csv("results.csv")
+```
+
+## 🔧 OpenAI Integration
+
+Scorebook includes built-in OpenAI support for both single requests and batch processing:
+
+```python
+from scorebook.inference.openai import responses, batch
+from scorebook.types.inference_pipeline import InferencePipeline
+
+# For single requests
+pipeline = InferencePipeline(
+    model="gpt-4o-mini",
+    preprocessor=format_for_openai,
+    inference_function=responses,
+    postprocessor=extract_response
+)
+
+# For batch processing (more efficient for large datasets)
+batch_pipeline = InferencePipeline(
+    model="gpt-4o-mini",
+    preprocessor=format_for_openai,
+    inference_function=batch,
+    postprocessor=extract_response
+)
+```
+
+## 📋 Examples
+
+The `examples/` directory contains comprehensive examples:
+
+- **`basic_example.py`**: Local model evaluation with Hugging Face
+- **`openai_responses_api.py`**: OpenAI API integration
+- **`openai_batch_api.py`**: OpenAI Batch API for large-scale evaluation
+- **`hyperparam_sweep.py`**: Hyperparameter optimization
+- **`scorebook_showcase.ipynb`**: Interactive Jupyter notebook tutorial
+
+Run an example:
+
+```bash
+cd examples/
+python basic_example.py --output-dir ./my_results
+```
+
+## 🏗️ Architecture
+
+Scorebook follows a modular architecture:
+
+```
+┌─────────────────┐    ┌──────────────┐    ┌─────────────────┐
+│   EvalDataset   │    │  Inference   │    │     Metrics     │
+│                 │    │   Pipeline   │    │                 │
+│ • Data Loading  │    │              │    │ • Accuracy      │
+│ • HF Integration│    │ • Preprocess │    │ • Precision     │
+│ • CSV/JSON      │    │ • Inference  │    │ • Custom        │
+│ • Validation    │    │ • Postprocess│    │ • Registry      │
+└─────────────────┘    └──────────────┘    └─────────────────┘
+        │                       │                       │
+        └───────────────────────┼───────────────────────┘
+                                │
+                   ┌─────────────────────┐
+                   │     evaluate()      │
+                   │                     │
+                   │ • Orchestration     │
+                   │ • Progress Tracking │
+                   │ • Result Formatting │
+                   │ • Export Options    │
+                   └─────────────────────┘
+```
+
+## 🎯 Use Cases
+
+Scorebook is designed for:
+
+- **🏆 Model Benchmarking**: Compare different models on standard datasets
+- **⚙️ Hyperparameter Optimization**: Find optimal model configurations
+- **📊 Dataset Analysis**: Understand model performance across different data types
+- **🔄 A/B Testing**: Compare model versions or approaches
+- **🔬 Research Experiments**: Reproducible evaluation workflows
+- **📈 Production Monitoring**: Track model performance over time
+
+## 🤝 Contributing
+
+We welcome contributions! Please see [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines.
+
+## 📄 License
+
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
+
+## 🏢 About
+
+Scorebook is developed by [Trismik](https://trismik.com) to speed up your LLM evaluation.
+
+---
+
+*For more examples and detailed documentation, check out the Jupyter notebook in `examples/scorebook_showcase.ipynb`*
+

scorebook-0.0.1.dist-info/RECORD

@@ -0,0 +1,24 @@
+scorebook/__init__.py,sha256=cYv8bT3_7o2MTxPVKiv51DcpaPtH_A9qOH5yF_FULZo,336
+scorebook/evaluator.py,sha256=Ce4KerLVPlaF63xng9RKH9M1l-ldo3mdrd3T2dBs_YE,8908
+scorebook/inference/__init__.py,sha256=sU_ZSN9eO7ajZ-QklNpx8_gf3jCdDn69J-SfU0z07-E,333
+scorebook/inference/openai.py,sha256=XD1dbPrEHQJVXOMtqCt9a0yQ-qR381N5XXhCrgz8jio,5826
+scorebook/inference/portkey.py,sha256=OHSS-sa2aLxuO6fEfG8MsPlhXc_95_-6j7ImbCkY8KE,5952
+scorebook/metrics/__init__.py,sha256=be_riJNojebXw2xfkMsHHjl3HFKgk9jQWlLkXJHhheI,782
+scorebook/metrics/accuracy.py,sha256=5KQ4hfOn9M94sB7WsXUelJWJiuKfoCGQEl5q5q9vNfo,1467
+scorebook/metrics/metric_base.py,sha256=I3L0DGcRojFp93UGFnXG1tZ2UK9ilTcXXJG6lj5ddXA,857
+scorebook/metrics/metric_registry.py,sha256=jWwt9P3zvtFLlEYrd60v7LS7X251nZczouE02zcCxWg,3402
+scorebook/metrics/precision.py,sha256=AaYPYYKnY74Nwqp_p3jd2Ewf3VHNOJjoRWf5fhb-tXk,563
+scorebook/types/__init__.py,sha256=xQMOae_fIBbeyeuqoa7SbNwjxAiVinPBbckOcUzo57U,358
+scorebook/types/eval_dataset.py,sha256=TeIeVHQ597NxedxaTEXohZO8gR5iAiDtJbCja_u69EI,11703
+scorebook/types/eval_result.py,sha256=y0vLN6RMgiz1lyai5ltmzDibBHE25-k9bTrQ7U27RZ8,4552
+scorebook/types/inference_pipeline.py,sha256=M3JgchpcVdhRJPzn3mh5ys6iivSt8eBmHIj4F5LcFYU,3167
+scorebook/utils/__init__.py,sha256=DmhS61OZ2nNWkGxDfVrMBwwiH7dmLAbg3MHuNgaXhQg,382
+scorebook/utils/async_utils.py,sha256=OeNvMrOT9P4rIyaCf5IbR3ZIFMtEzXgoAArNbINRtMU,728
+scorebook/utils/io_helpers.py,sha256=ksOJ9ILcZqqt-HwRUYy1NMQbS6RuMh8i2ZzUADLMlQ8,913
+scorebook/utils/mappers.py,sha256=OcUnPBrnSUxZNhAzJhVmVWUWmqIKFXLTrK-xLi6_SUg,1259
+scorebook/utils/progress_bars.py,sha256=BlKqYlXDbik5eUn5nf5f7QnMvnTj8CU_CfXKxCWp3Ww,2909
+scorebook/utils/transform_helpers.py,sha256=UnVLtFvcJrtmBEmLsuA4rrX4iJlNUKxm2DkIOGLl-2o,1030
+scorebook-0.0.1.dist-info/LICENSE,sha256=JLH1g9FhxHZf6CBCeQ_xAisPtICVObuNGW1bLPiTYEs,1068
+scorebook-0.0.1.dist-info/METADATA,sha256=oiwYbuJkRVkoFZkIAQej09LdG5xBLxhKPy2ozWTV-_w,10976
+scorebook-0.0.1.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+scorebook-0.0.1.dist-info/RECORD,,

scorebook-0.0.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: poetry-core 2.1.3
+Root-Is-Purelib: true
+Tag: py3-none-any