scorebook 0.0.12__py3-none-any.whl → 0.0.14__py3-none-any.whl

scorebook-0.0.14.dist-info/METADATA

@@ -0,0 +1,292 @@
+Metadata-Version: 2.4
+Name: scorebook
+Version: 0.0.14
+Summary: A Python project for LLM evaluation.
+License-File: LICENSE
+Author: Euan Campbell
+Author-email: euan@trismik.com
+Requires-Python: >=3.9, <3.14
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Provides-Extra: bedrock
+Provides-Extra: examples
+Provides-Extra: openai
+Provides-Extra: portkey
+Provides-Extra: vertex
+Requires-Dist: accelerate ; extra == "examples"
+Requires-Dist: boto3 (==1.40.0) ; extra == "bedrock"
+Requires-Dist: datasets (>=3.6.0)
+Requires-Dist: fsspec[gcs] ; extra == "vertex"
+Requires-Dist: google-cloud-storage ; extra == "vertex"
+Requires-Dist: google-genai ; extra == "vertex"
+Requires-Dist: ipywidgets (>=8.0.0)
+Requires-Dist: notebook (>=7.4.5,<8.0.0)
+Requires-Dist: notebook ; extra == "examples"
+Requires-Dist: openai ; extra == "openai"
+Requires-Dist: pandas ; extra == "vertex"
+Requires-Dist: portkey-ai ; extra == "portkey"
+Requires-Dist: python-dotenv ; extra == "bedrock"
+Requires-Dist: python-dotenv ; extra == "openai"
+Requires-Dist: python-dotenv ; extra == "portkey"
+Requires-Dist: python-dotenv ; extra == "vertex"
+Requires-Dist: torch ; extra == "examples"
+Requires-Dist: torchaudio ; extra == "examples"
+Requires-Dist: torchvision ; extra == "examples"
+Requires-Dist: transformers ; extra == "examples"
+Requires-Dist: trismik (==1.0.2)
+Description-Content-Type: text/markdown
+
+<h1 align="center">Scorebook</h1>
+
+<p align="center"><strong>A Python library for Model evaluation</strong></p>
+
+<p align="center">
+  <img alt="Dynamic TOML Badge" src="https://img.shields.io/badge/dynamic/toml?url=https%3A%2F%2Fraw.githubusercontent.com%2Ftrismik%2Fscorebook%2Frefs%2Fheads%2Fmain%2Fpyproject.toml&query=tool.poetry.version&style=flat&label=version">
+  <img alt="Python Version" src="https://img.shields.io/badge/python-3.9%2B-blue">
+  <a href="https://docs.trismik.com/scorebook/introduction-to-scorebook/" target="_blank" rel="noopener">
+    <img alt="Documentation" src="https://img.shields.io/badge/docs-Scorebook-blue?style=flat">
+  </a>
+  <img alt="License" src="https://img.shields.io/badge/license-MIT-green">
+</p>
+
+Scorebook provides a flexible and extensible framework for evaluating models such as large language models (LLMs). Easily evaluate any model using evaluation datasets from Hugging Face such as MMLU-Pro, HellaSwag, and CommonSenseQA, or with data from any other source. Evaluations calculate scores for any number of specified metrics such as accuracy, precision, and recall, as well as any custom defined metrics, including LLM as a judge (LLMaJ).
+
+## Use Cases
+
+Scorebook's evaluations can be used for:
+
+- **Model Benchmarking**: Compare different models on standard datasets.
+- **Model Optimization**: Find optimal model configurations.
+- **Iterative Experimentation**: Reproducible evaluation workflows.
+
+## Key Features
+
+- **Model Agnostic**: Evaluate any model, running locally or deployed on the cloud.
+- **Dataset Agnostic**: Create evaluation datasets from Hugging Face datasets or any other source.
+- **Extensible Metric Engine**: Use the Scorebook's built-in or implement your own.
+- **Hyperparameter Sweeping**: Evaluate over multiple model hyperparameter configurations.
+- **Adaptive Evaluations**: Run Trismik's ultra-fast [adaptive evaluations](https://docs.trismik.com/adaptiveTesting/adaptive-testing-introduction/).
+- **Trismik Integration**: Upload evaluations to [Trismik's platform](https://www.trismik.com/).
+
+## Installation
+
+```bash
+pip install scorebook
+```
+
+## Scoring Models Output
+
+Scorebooks score function can be used to evaluate pre-generated model outputs.
+
+### Score Example
+```python
+from scorebook import score
+from scorebook.metrics import Accuracy
+
+# 1. Prepare a list of generated model outputs and labels
+model_predictions = [
+    {"input": "What is 2 + 2?", "output": "4", "label": "4"},
+    {"input": "What is the capital of France?", "output": "London", "label": "Paris"},
+    {"input": "Who wrote Romeo and Juliette?", "output": "William Shakespeare", "label": "William Shakespeare"},
+    {"input": "What is the chemical symbol for gold?", "output": "Au", "label": "Au"},
+]
+
+# 2. Score the model's predictions against labels using metrics
+results = score(
+    items = model_predictions,
+    metrics = Accuracy,
+)
+```
+
+### Score Results:
+```json
+{
+    "aggregate_results": [
+        {
+            "dataset": "scored_items",
+            "accuracy": 0.75
+        }
+    ],
+    "item_results": [
+        {
+            "id": 0,
+            "dataset": "scored_items",
+            "input": "What is 2 + 2?",
+            "output": "4",
+            "label": "4",
+            "accuracy": true
+        }
+        // ... additional items
+    ]
+}
+```
+
+## _Classical_ Evaluations
+
+Running a classical evaluation in Scorebook executes model inference on every item in the dataset, then scores the generated outputs using the dataset’s specified metrics to quantify model performance.
+
+### Classical Evaluation example:
+```python
+from scorebook import evaluate, EvalDataset
+from scorebook.metrics import Accuracy
+
+# 1. Create an evaluation dataset
+evaluation_items = [
+    {"question": "What is 2 + 2?", "answer": "4"},
+    {"question": "What is the capital of France?", "answer": "Paris"},
+    {"question": "Who wrote Romeo and Juliet?", "answer": "William Shakespeare"}
+]
+
+evaluation_dataset = EvalDataset.from_list(
+    name = "basic_questions",
+    items = evaluation_items,
+    input = "question",
+    label = "answer",
+    metrics = Accuracy,
+)
+
+# 2. Define an inference function - This is a pseudocode example
+def inference_function(inputs: List[Any], **hyperparameters):
+
+    # Create or call a model
+    model = Model()
+    model.temperature = hyperparameters.get("temperature")
+
+    # Call model inference
+    model_outputs = model(inputs)
+
+    # Return outputs
+    return model_outputs
+
+# 3. Run evaluation
+evaluation_results = evaluate(
+    inference_function,
+    evaluation_dataset,
+    hyperparameters = {"temperature": 0.7}
+)
+```
+
+### Evaluation Results:
+```json
+{
+    "aggregate_results": [
+        {
+            "dataset": "basic_questions",
+            "temperature": 0.7,
+            "accuracy": 1.0,
+            "run_completed": true
+        }
+    ],
+    "item_results": [
+        {
+            "id": 0,
+            "dataset": "basic_questions",
+            "input": "What is 2 + 2?",
+            "output": "4",
+            "label": "4",
+            "temperature": 0.7,
+            "accuracy": true
+        }
+        // ... additional items
+    ]
+}
+```
+
+### _Adaptive_ Evaluations with `evaluate`
+
+To run an adaptive evaluation, use a Trismik adaptive dataset
+The CAT algorithm dynamically selects items to estimate the model’s ability (θ) with minimal standard error and fewest questions.
+
+### Adaptive Evaluation Example
+```python
+from scorebook import evaluate, login
+
+# 1. Log in with your Trismik API key
+login("TRISMIK_API_KEY")
+
+# 2. Define an inference function
+def inference_function(inputs: List[Any], **hyperparameters):
+
+    # Create or call a model
+    model = Model()
+
+    # Call model inference
+    outputs = model(inputs)
+
+    # Return outputs
+    return outputs
+
+# 3. Run an adaptive evaluation
+results = evaluate(
+    inference_function,
+    datasets = "trismik/headQA:adaptive",    # Adaptive datasets have the ":adaptive" suffix
+    project_id = "TRISMIK_PROJECT_ID",       # Required: Create a project on your Trismik dashboard
+    experiment_id = "TRISMIK_EXPERIMENT_ID", # Optional: An identifier to upload this run under
+)
+```
+
+### Adaptive Evaluation Results
+```json
+{
+    "aggregate_results": [
+        {
+            "dataset": "trismik/headQA:adaptive",
+            "experiment_id": "TRISMIK_EXPERIMENT_ID",
+            "project_id": "TRISMIK_PROJECT_ID",
+            "run_id": "RUN_ID",
+            "score": {
+                "theta": 1.2,
+                "std_error": 0.20
+            },
+            "responses": null
+        }
+    ],
+    "item_results": []
+}
+```
+
+## Metrics
+
+| Metric     | Sync/Async | Aggregate Scores                                 | Item Scores                             |
+|------------|------------|--------------------------------------------------|-----------------------------------------|
+| `Accuracy` | Sync       | `Float`: Percentage of correct outputs           | `Boolean`: Exact match between output and label |
+
+
+## Tutorials
+
+For local more detailed and runnable examples:
+```bash
+pip install scorebook[examples]
+```
+
+The `tutorials/` directory contains comprehensive tutorials as notebooks and code examples:
+
+- **`tutorials/notebooks`**: Interactive Jupyter Notebooks showcasing Scorebook's capabilities.
+- **`tutorials/examples`**: Runnable Python examples incrementally implementing Scorebook's features.
+
+**Run a notebook:**
+```bash
+jupyter notebook tutorials/notebooks
+```
+
+**Run an example:**
+```bash
+python3 tutorials/examples/1-score/1-scoring_model_accuracy.py
+```
+
+## Contributing
+
+We welcome contributions! Please see [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines.
+
+## License
+
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
+
+## About
+
+Scorebook is developed by [Trismik](https://trismik.com) to simplify and speed up your LLM evaluations.
+

scorebook-0.0.14.dist-info/RECORD

@@ -0,0 +1,53 @@
+scorebook/__init__.py,sha256=S2JaZZsx76p0EjXtKz4UPdSzuO60jAjOvooYP-idBu8,1144
+scorebook/cli/__init__.py,sha256=E89jR1DljFSHhfjEGSRKLgz0KhxGyRQ9a3vpUOmQL9o,32
+scorebook/cli/auth.py,sha256=VGS5T0CSeS0n_7bntNggrYx-vDwxJJHdYxbKedFAq74,2939
+scorebook/cli/main.py,sha256=cEvShENl6L6feX_sa7FGNTeoz5UtwqzwenmcHaON1hg,1589
+scorebook/dashboard/__init__.py,sha256=36DxO3oXVcZ2I6kizLFCcJkLBpXOU8UIXFT_ZjeFTB4,50
+scorebook/dashboard/create_project.py,sha256=RK90aMN0_XVM-DnawTY_b59yPJaRnpb_GoidCqXB5Vw,2845
+scorebook/dashboard/credentials.py,sha256=Q_khY5AX3fnyWshHe6LaesBHcCmNBse6a_XFGT8OOaw,3474
+scorebook/dashboard/upload_results.py,sha256=sdgOEf0C7QLt7t2QiXvSoceQpAiiPmlG_4SFEEzVPlc,9738
+scorebook/eval_datasets/__init__.py,sha256=wsmFNyuZJdBxjokcKG4NRfuUzPZKuzsKX3aG21zfFV4,39
+scorebook/eval_datasets/eval_dataset.py,sha256=xnG7VaceWUmg8Wrk2IGnVFZs9umzmZrW8F7THvtWMqs,28041
+scorebook/evaluate/__init__.py,sha256=Qqe-l4y3Nu81Fdx83RbtCQESoXC0XukBgOC3DPSWZZA,39
+scorebook/evaluate/_async/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scorebook/evaluate/_async/evaluate_async.py,sha256=G0RB_A1f5mQ42D82DnxkzAZhyV5kgbxi9Lr7qKaKUyY,16590
+scorebook/evaluate/_sync/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scorebook/evaluate/_sync/evaluate.py,sha256=OIUsW2U1IrdwYIIPsfpTCOfJDAYJ6BYl-6pQQiafSNE,16364
+scorebook/evaluate/evaluate_helpers.py,sha256=NnanxLEeHwoZNztGXQJc6u_WqKfDkn1vYmck2BrKF-c,17028
+scorebook/exceptions.py,sha256=3sxCWhFqYgXiWNUAMRR2ggLfqvbDI8e5vLjnT9V7X1M,3649
+scorebook/inference/__init__.py,sha256=gGuZG1rdpxKYC54q0eAS6oTHQbRYhgxlBeAqonqHvRU,60
+scorebook/inference/clients/__init__.py,sha256=VaLW7mi4tywJtR3Q9wr2pPci8NlEQ3bJanZyM5S81Z4,51
+scorebook/inference/clients/bedrock.py,sha256=bsnz0IB6ufjZVPd6syD3yVaOelerB5G_YAmPAVqmBmI,10071
+scorebook/inference/clients/openai.py,sha256=JPPcJdxYwwZNUfXCGTRRzzUUA8I8WiV3bu6-pgS1_UE,9043
+scorebook/inference/clients/portkey.py,sha256=RCuEZB8xNAVeGEt3IJ0esh_wqreZNB3jrDKiWH6miV0,5949
+scorebook/inference/clients/vertex.py,sha256=g6oNnag0qcLOYCtQ4SXAXfnqKtvPAVdigB--I7wU1yM,9871
+scorebook/inference/inference_pipeline.py,sha256=1qSmfI4fBJFS3EcAhRlA-f4-8aI6wDiupSJu-vNXoYI,5571
+scorebook/metrics/__init__.py,sha256=bsEq15LpFt3h0AQQFbnvL4CU7KpIpifVdJAsfduPGXk,48
+scorebook/metrics/accuracy.py,sha256=5KQ4hfOn9M94sB7WsXUelJWJiuKfoCGQEl5q5q9vNfo,1467
+scorebook/metrics/metric_base.py,sha256=I3L0DGcRojFp93UGFnXG1tZ2UK9ilTcXXJG6lj5ddXA,857
+scorebook/metrics/metric_registry.py,sha256=YcbKGf2kPMQqyqJ9NYVq_-J19rARXSo22HjTW5WU-QU,3404
+scorebook/metrics/precision.py,sha256=AaYPYYKnY74Nwqp_p3jd2Ewf3VHNOJjoRWf5fhb-tXk,563
+scorebook/score/__init__.py,sha256=CqkslUvOw8QfCCbSfwZgGrbmXeSLpZqIVo4ntrctYuY,66
+scorebook/score/_async/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scorebook/score/_async/score_async.py,sha256=SatV9hEUT8MAru2ACSyM03weKX6VTFx7crW59_uX0L8,6155
+scorebook/score/_sync/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scorebook/score/_sync/score.py,sha256=nANQbuyYyIaWnoTQzyGMwPZRMFP6MmyIyHb1GO1mktQ,6101
+scorebook/score/score_helpers.py,sha256=Gjx2Lgd94ISvunb5CHj-tDWYVEOVj9ySjjVYnnhpk_Q,7086
+scorebook/settings.py,sha256=qZrNiki6rFXn43udmhjQSmdDKOEaX62WYoEs2Rbggr0,720
+scorebook/types.py,sha256=2lv1YUky7aDGIEPjgj18aKTpBMdmqD01TKLbwli19pQ,4904
+scorebook/utils/__init__.py,sha256=oBTybVHI5EdHIgzb0TeoAnSLMQdUh20Ww6vcL9542Pk,72
+scorebook/utils/async_utils.py,sha256=2ewk_VOePib8z7DTRl-pZQBGzVI3L3JvnEuYW-DTkRA,1325
+scorebook/utils/common_helpers.py,sha256=lJIqO9XGf1T3S3rdGBTjZJ1BzVPvaU_XTONEfPApnEM,1218
+scorebook/utils/io_helpers.py,sha256=ORO6DwtXOKWJq9v_isuunUrz0viE3xy2qYO4lrgU-TM,1437
+scorebook/utils/jinja_helpers.py,sha256=ksIKHiKdj8N0o7ZJZGasfbSNoAY6K5d9X_KM6mcKYD4,4208
+scorebook/utils/mappers.py,sha256=OcUnPBrnSUxZNhAzJhVmVWUWmqIKFXLTrK-xLi6_SUg,1259
+scorebook/utils/mock_llm/__init__.py,sha256=dK70wNVBKk4hv1o3fceDTBG1_maFbkMvoOtTriPCe78,1293
+scorebook/utils/mock_llm/data/mock_llm_data.json,sha256=b28j7OCR0igpP0rkXDJAR2NWIiuVkOaAkzB-Miv665Y,381567
+scorebook/utils/progress_bars.py,sha256=gdT6dJ9LMLYzs7TospP3wQNY9htm_FhVLdX0ueluC6E,31890
+scorebook/utils/render_template.py,sha256=NOaZt-N1WcR5MA7at1XxzD-4sFMFKo9X0k7fKq6oSSM,1654
+scorebook/utils/transform_helpers.py,sha256=UnVLtFvcJrtmBEmLsuA4rrX4iJlNUKxm2DkIOGLl-2o,1030
+scorebook-0.0.14.dist-info/METADATA,sha256=jPqVszfpCiAKf3yt45XD6lXfIJL1-TFvSMDVGrIoCPs,9491
+scorebook-0.0.14.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+scorebook-0.0.14.dist-info/entry_points.txt,sha256=9gNd3Q0MEozhJ7fog-Q-Z_PrcGMnF-404Jon40MH2_U,53
+scorebook-0.0.14.dist-info/licenses/LICENSE,sha256=JLH1g9FhxHZf6CBCeQ_xAisPtICVObuNGW1bLPiTYEs,1068
+scorebook-0.0.14.dist-info/RECORD,,

scorebook/trismik/__init__.py

@@ -1,10 +0,0 @@
-"""Trismik authentication and API integration.
-
-Note: Trismik evaluation functionality has been moved to scorebook.evaluate module.
-This module now only provides authentication functions.
-"""
-
-# Import shared credential functions
-from .credentials import get_stored_token, get_token, login, logout, whoami
-
-__all__ = ["login", "logout", "whoami", "get_stored_token", "get_token"]