scorebook 0.0.14__py3-none-any.whl → 0.0.15__py3-none-any.whl

{scorebook-0.0.14.dist-info → scorebook-0.0.15.dist-info}/METADATA

@@ -1,43 +1,41 @@
 Metadata-Version: 2.4
 Name: scorebook
-Version: 0.0.14
+Version: 0.0.15
 Summary: A Python project for LLM evaluation.
 License-File: LICENSE
 Author: Euan Campbell
 Author-email: euan@trismik.com
-Requires-Python: >=3.9, <3.14
+Requires-Python: >=3.10, <3.14
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
-Provides-Extra: bedrock
 Provides-Extra: examples
-Provides-Extra: openai
-Provides-Extra: portkey
-Provides-Extra: vertex
+Provides-Extra: metrics
+Provides-Extra: providers
 Requires-Dist: accelerate ; extra == "examples"
-Requires-Dist: boto3 (==1.40.0) ; extra == "bedrock"
+Requires-Dist: bert-score ; extra == "metrics"
+Requires-Dist: boto3 (==1.40.0) ; extra == "providers"
 Requires-Dist: datasets (>=3.6.0)
-Requires-Dist: fsspec[gcs] ; extra == "vertex"
-Requires-Dist: google-cloud-storage ; extra == "vertex"
-Requires-Dist: google-genai ; extra == "vertex"
-Requires-Dist: ipywidgets (>=8.0.0)
-Requires-Dist: notebook (>=7.4.5,<8.0.0)
+Requires-Dist: fsspec[gcs] ; extra == "providers"
+Requires-Dist: google-cloud-storage ; extra == "providers"
+Requires-Dist: google-genai ; extra == "providers"
+Requires-Dist: ipywidgets ; extra == "examples"
+Requires-Dist: jinja2 (>=3.1.6,<4.0.0)
 Requires-Dist: notebook ; extra == "examples"
-Requires-Dist: openai ; extra == "openai"
-Requires-Dist: pandas ; extra == "vertex"
-Requires-Dist: portkey-ai ; extra == "portkey"
-Requires-Dist: python-dotenv ; extra == "bedrock"
-Requires-Dist: python-dotenv ; extra == "openai"
-Requires-Dist: python-dotenv ; extra == "portkey"
-Requires-Dist: python-dotenv ; extra == "vertex"
+Requires-Dist: openai ; extra == "providers"
+Requires-Dist: pandas ; extra == "providers"
+Requires-Dist: portkey-ai ; extra == "providers"
+Requires-Dist: python-dotenv (>=1.0.0)
+Requires-Dist: rouge-score ; extra == "metrics"
+Requires-Dist: sacrebleu ; extra == "metrics"
+Requires-Dist: scikit-learn (>=1.0.0) ; extra == "metrics"
 Requires-Dist: torch ; extra == "examples"
 Requires-Dist: torchaudio ; extra == "examples"
 Requires-Dist: torchvision ; extra == "examples"
 Requires-Dist: transformers ; extra == "examples"
-Requires-Dist: trismik (==1.0.2)
+Requires-Dist: trismik (>=1.0.3)
 Description-Content-Type: text/markdown
 
 <h1 align="center">Scorebook</h1>
@@ -51,6 +49,9 @@ Description-Content-Type: text/markdown
     <img alt="Documentation" src="https://img.shields.io/badge/docs-Scorebook-blue?style=flat">
   </a>
   <img alt="License" src="https://img.shields.io/badge/license-MIT-green">
+  <a target="_blank" href="https://colab.research.google.com/github/trismik/scorebook/blob/main/tutorials/quickstarts/getting_started.ipynb">
+    <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/>
+  </a>
 </p>
 
 Scorebook provides a flexible and extensible framework for evaluating models such as large language models (LLMs). Easily evaluate any model using evaluation datasets from Hugging Face such as MMLU-Pro, HellaSwag, and CommonSenseQA, or with data from any other source. Evaluations calculate scores for any number of specified metrics such as accuracy, precision, and recall, as well as any custom defined metrics, including LLM as a judge (LLMaJ).
@@ -251,9 +252,16 @@ results = evaluate(
 
 ## Metrics
 
-| Metric     | Sync/Async | Aggregate Scores                                 | Item Scores                             |
-|------------|------------|--------------------------------------------------|-----------------------------------------|
-| `Accuracy` | Sync       | `Float`: Percentage of correct outputs           | `Boolean`: Exact match between output and label |
+| Metric       | Sync/Async | Aggregate Scores                                 | Item Scores                             |
+|--------------|------------|--------------------------------------------------|-----------------------------------------|
+| `Accuracy`   | Sync       | `Float`: Percentage of correct outputs           | `Boolean`: Exact match between output and label |
+| `ExactMatch` | Sync       | `Float`: Percentage of exact string matches      | `Boolean`: Exact match with optional case/whitespace normalization |
+| `F1`         | Sync       | `Dict[str, Float]`: F1 scores per averaging method (macro, micro, weighted) | `Boolean`: Exact match between output and label |
+| `Precision`  | Sync       | `Dict[str, Float]`: Precision scores per averaging method (macro, micro, weighted) | `Boolean`: Exact match between output and label |
+| `Recall`     | Sync       | `Dict[str, Float]`: Recall scores per averaging method (macro, micro, weighted) | `Boolean`: Exact match between output and label |
+| `BLEU`       | Sync       | `Float`: Corpus-level BLEU score                 | `Float`: Sentence-level BLEU score |
+| `ROUGE`      | Sync       | `Dict[str, Float]`: Average F1 scores per ROUGE type | `Dict[str, Float]`: F1 scores per ROUGE type |
+| `BertScore`  | Sync       | `Dict[str, Float]`: Average precision, recall, and F1 scores | `Dict[str, Float]`: Precision, recall, and F1 scores per item |
 
 
 ## Tutorials

scorebook-0.0.15.dist-info/RECORD

@@ -0,0 +1,110 @@
+scorebook/__init__.py,sha256=YB3PyPB0-sRicIwiQZ8aRhviu04_FZY8Ne5o-FuNWtA,1236
+scorebook/cli/__init__.py,sha256=E89jR1DljFSHhfjEGSRKLgz0KhxGyRQ9a3vpUOmQL9o,32
+scorebook/cli/auth.py,sha256=VGS5T0CSeS0n_7bntNggrYx-vDwxJJHdYxbKedFAq74,2939
+scorebook/cli/main.py,sha256=cEvShENl6L6feX_sa7FGNTeoz5UtwqzwenmcHaON1hg,1589
+scorebook/dashboard/__init__.py,sha256=36DxO3oXVcZ2I6kizLFCcJkLBpXOU8UIXFT_ZjeFTB4,50
+scorebook/dashboard/create_project.py,sha256=RK90aMN0_XVM-DnawTY_b59yPJaRnpb_GoidCqXB5Vw,2845
+scorebook/dashboard/credentials.py,sha256=CCxafElx_pMLD-c69JvYAcC-9SzZf3tjAnJQUf8q5Us,4796
+scorebook/dashboard/upload_results.py,sha256=sdgOEf0C7QLt7t2QiXvSoceQpAiiPmlG_4SFEEzVPlc,9738
+scorebook/eval_datasets/__init__.py,sha256=wsmFNyuZJdBxjokcKG4NRfuUzPZKuzsKX3aG21zfFV4,39
+scorebook/eval_datasets/eval_dataset.py,sha256=R7upmIhvDTBwyfuFErfc2iICI6M6AkUTMeO0Oi9NFk0,28051
+scorebook/evaluate/__init__.py,sha256=Qqe-l4y3Nu81Fdx83RbtCQESoXC0XukBgOC3DPSWZZA,39
+scorebook/evaluate/_async/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scorebook/evaluate/_async/evaluate_async.py,sha256=Y0bH4nSdwU13A1oUO3gM6oEyM4bCQCpr7fhUWDZZJpY,17198
+scorebook/evaluate/_sync/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scorebook/evaluate/_sync/evaluate.py,sha256=MQqKByraT22EUPfqoq1H4VIyRcPIo0ahv6KZpHgQX1A,16972
+scorebook/evaluate/evaluate_helpers.py,sha256=NnanxLEeHwoZNztGXQJc6u_WqKfDkn1vYmck2BrKF-c,17028
+scorebook/exceptions.py,sha256=3sxCWhFqYgXiWNUAMRR2ggLfqvbDI8e5vLjnT9V7X1M,3649
+scorebook/inference/__init__.py,sha256=gGuZG1rdpxKYC54q0eAS6oTHQbRYhgxlBeAqonqHvRU,60
+scorebook/inference/clients/__init__.py,sha256=VaLW7mi4tywJtR3Q9wr2pPci8NlEQ3bJanZyM5S81Z4,51
+scorebook/inference/clients/bedrock.py,sha256=bsnz0IB6ufjZVPd6syD3yVaOelerB5G_YAmPAVqmBmI,10071
+scorebook/inference/clients/openai.py,sha256=JPPcJdxYwwZNUfXCGTRRzzUUA8I8WiV3bu6-pgS1_UE,9043
+scorebook/inference/clients/portkey.py,sha256=RCuEZB8xNAVeGEt3IJ0esh_wqreZNB3jrDKiWH6miV0,5949
+scorebook/inference/clients/vertex.py,sha256=g6oNnag0qcLOYCtQ4SXAXfnqKtvPAVdigB--I7wU1yM,9871
+scorebook/inference/inference_pipeline.py,sha256=1qSmfI4fBJFS3EcAhRlA-f4-8aI6wDiupSJu-vNXoYI,5571
+scorebook/metrics/README.md,sha256=DTMgT-aYLqlx32vAPEOCvbfFNrwhYXd3z9g1oTjCPPc,4065
+scorebook/metrics/__init__.py,sha256=5m688iVI8k_NUzZJdOL_-IxvwOWU8j4fa6b1yOGPx7w,232
+scorebook/metrics/accuracy.py,sha256=0Qxu0nbzHhQbIdd5b3iOlbTnr4Nwi_wMh7hu4EStw4I,1284
+scorebook/metrics/bertscore.py,sha256=DDjdLZ8sBTROMF1UpGN5gyUYIkEYv7kQZkEpoMMYyuY,1853
+scorebook/metrics/bleu.py,sha256=xUAHIxWisvyq2f0k0zzrf3eEfCdHZ-VSrcmt73z4Xto,3075
+scorebook/metrics/core/__init__.py,sha256=dWfy6rXNvOxxdHSEvB_eJcbbHZ3xDxhGKEXYHAfrXKI,40
+scorebook/metrics/core/metric_base.py,sha256=XDHn-U8oUzhoPukKyYkg5xJLN05fSvnciAYCrCB28Mg,845
+scorebook/metrics/core/metric_registry.py,sha256=jWt8pB4q96po724TzOivjrabuOj6zBP9_uEdYqHxQlU,7242
+scorebook/metrics/exactmatch.py,sha256=xjd9iSlmdJ8Vk-LjfxAGJZyQ75FLjmeT4polG-P3JkY,3128
+scorebook/metrics/f1.py,sha256=R8yQ5mkb4bfPZM6a1X3IL1Q4OnUFD_--qwQdBSlBqaE,3710
+scorebook/metrics/precision.py,sha256=AFBDemPcs03txaihw3BqSoJfPQUFykZ_QR-IFx3du-s,3705
+scorebook/metrics/recall.py,sha256=ek5NV55p_Ux-2mT2xAkvQdVsg3xCJtJ3zT9O-vPEQyA,3658
+scorebook/metrics/rouge.py,sha256=ZDld4mrOmERRuEESgeZITZ-NYO3QzBTYRidULwuPWqA,3228
+scorebook/score/__init__.py,sha256=CqkslUvOw8QfCCbSfwZgGrbmXeSLpZqIVo4ntrctYuY,66
+scorebook/score/_async/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scorebook/score/_async/score_async.py,sha256=SatV9hEUT8MAru2ACSyM03weKX6VTFx7crW59_uX0L8,6155
+scorebook/score/_sync/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scorebook/score/_sync/score.py,sha256=nANQbuyYyIaWnoTQzyGMwPZRMFP6MmyIyHb1GO1mktQ,6101
+scorebook/score/score_helpers.py,sha256=NQt5K-hI4EZErenMyAoR8iQa76KBVTAc-nmUMCbqh8U,8179
+scorebook/settings.py,sha256=qZrNiki6rFXn43udmhjQSmdDKOEaX62WYoEs2Rbggr0,720
+scorebook/types.py,sha256=NJZCV7hB-hKLN-Mmijtm0DoQVMUJVTahPo_n4nfQ5mE,4901
+scorebook/utils/__init__.py,sha256=oBTybVHI5EdHIgzb0TeoAnSLMQdUh20Ww6vcL9542Pk,72
+scorebook/utils/async_utils.py,sha256=2ewk_VOePib8z7DTRl-pZQBGzVI3L3JvnEuYW-DTkRA,1325
+scorebook/utils/common_helpers.py,sha256=lJIqO9XGf1T3S3rdGBTjZJ1BzVPvaU_XTONEfPApnEM,1218
+scorebook/utils/io_helpers.py,sha256=ORO6DwtXOKWJq9v_isuunUrz0viE3xy2qYO4lrgU-TM,1437
+scorebook/utils/jinja_helpers.py,sha256=ksIKHiKdj8N0o7ZJZGasfbSNoAY6K5d9X_KM6mcKYD4,4208
+scorebook/utils/mappers.py,sha256=OcUnPBrnSUxZNhAzJhVmVWUWmqIKFXLTrK-xLi6_SUg,1259
+scorebook/utils/mock_llm/__init__.py,sha256=dK70wNVBKk4hv1o3fceDTBG1_maFbkMvoOtTriPCe78,1293
+scorebook/utils/mock_llm/data/mock_llm_data.json,sha256=b28j7OCR0igpP0rkXDJAR2NWIiuVkOaAkzB-Miv665Y,381567
+scorebook/utils/progress_bars.py,sha256=4k52Y40oXaUo40ZPYGcMM3rFbWF7nsbzLo3DDpkYv9o,3531
+scorebook/utils/render_template.py,sha256=NOaZt-N1WcR5MA7at1XxzD-4sFMFKo9X0k7fKq6oSSM,1654
+scorebook/utils/transform_helpers.py,sha256=UnVLtFvcJrtmBEmLsuA4rrX4iJlNUKxm2DkIOGLl-2o,1030
+tutorials/README.md,sha256=DETPgdCTTZLYMoVZ9DSfad1y7Jq4EMzi6UA9Ijm0GQc,5588
+tutorials/__init__.py,sha256=I1ki4-8iZsO9pbK4_xmRpjzSDYoH0JdYbtwzvgPgeo8,196
+tutorials/examples/1-score/1-scoring_model_accuracy.py,sha256=Uc9DPa31TOR_1egsUhLptUIGXb67FYLtLKpDPtQvcGA,1367
+tutorials/examples/1-score/2-scoring_model_bleu.py,sha256=pE4SMKR32Noa1eU8QUYU1a533WvXzyPi440jsgYs9gk,1991
+tutorials/examples/1-score/3-scoring_model_f1.py,sha256=g_uH_IgVNWtTaQ47MEBztUmGqgUn0y2464xNIoAN_dI,2078
+tutorials/examples/1-score/4-scoring_model_rouge.py,sha256=8HaPzv-Y8oTy88FtxiFR4i0SEd7BndxL_YgzpVSLEP8,2512
+tutorials/examples/1-score/5-scoring_model_exact_match.py,sha256=PcnrF7OfULRQd4veSKwlp5lQ1lMo749cDSdRu88sOVE,3235
+tutorials/examples/1-score/6-scoring_with_bertscore.py,sha256=_LRDrC5K_VmfxH37L7RbvLrQB4DpHbf-8K0Vr0R4cxc,2148
+tutorials/examples/1-score/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tutorials/examples/2-evaluate/1-evaluating_local_models.py,sha256=FSFWQUism2HWHELX3Q-F1jkRJ1feeyx1TZqttSRruPY,3630
+tutorials/examples/2-evaluate/2-evaluating_local_models_with_batching.py,sha256=EKdyTP7svdtxeOdM0E569RZckwZ_GAH04ZMFi5kaVS8,3840
+tutorials/examples/2-evaluate/3-evaluating_cloud_models.py,sha256=C8QFmfkEd3APhh5k9LLgaxa3rlj1iQfQdV5k6jRAgx8,3566
+tutorials/examples/2-evaluate/4-evaluating_cloud_models_with_batching.py,sha256=ckd8NZe9Qyv1FvCbuhAqj3qL1ZKBlSE1fdEy3_jxvXw,6338
+tutorials/examples/2-evaluate/5-hyperparameter_sweeps.py,sha256=p0A_dwoVaGnAgpdRDu1MfC-HUp8UoBanSHvyJBpM97I,3954
+tutorials/examples/2-evaluate/6-inference_pipelines.py,sha256=fy0dmVXV1D7jfLq6Xhhqe3C3UYDBRRtPybvkGctV2E8,4991
+tutorials/examples/3-evaluation_datasets/1-evaluation_datasets_from_files.py,sha256=VihoUKmdaHfvJftyqFTSP0hkK1N1g1tAQZmw5yH438E,3458
+tutorials/examples/3-evaluation_datasets/2-evaluation_datasets_from_huggingface.py,sha256=e9PvbrAoXzIVXtqlg9dZP6Xd8b81nKqzEuTuRXLhutk,3225
+tutorials/examples/3-evaluation_datasets/3-evaluation_datasets_from_huggingface_with_yaml_configs.py,sha256=1odW_2qFNyGES9L0bYWLvP8Cplc8ciIx42SZ_JZ6yis,3911
+tutorials/examples/3-evaluation_datasets/example_datasets/basic_questions.csv,sha256=MvTl1wWx-TX0LgR43BvPCrk6tKUuGItp7_jA7RcWfMw,374
+tutorials/examples/3-evaluation_datasets/example_datasets/basic_questions.json,sha256=au86RNV2JRvw8IHNZK-768lWJdJQnCrod-6N-F-Chmk,921
+tutorials/examples/3-evaluation_datasets/example_yaml_configs/Cais-MMLU.yaml,sha256=F_Aiuw1BJBUNFy_ENiZDiQvcT2FqNprlUb1Y9kouzhE,320
+tutorials/examples/3-evaluation_datasets/example_yaml_configs/TIGER-Lab-MMLU-Pro.yaml,sha256=-D9T78LJ1ihlobHgzrEeJIplimItj6pkzms3lWz50bI,362
+tutorials/examples/4-adaptive_evaluations/1-adaptive_evaluation.py,sha256=2k5U61z3g1kYEwrCW4W27Yt6dYYS-PGL-bVXmwWs5YE,4042
+tutorials/examples/4-adaptive_evaluations/2-adaptive_dataset_splits.py,sha256=KRhzAg6JxhF0ie3YeE2UXY34Lv6zfDuL_536L9i7Qdg,3598
+tutorials/examples/5-upload_results/1-uploading_score_results.py,sha256=VUq9WzymPzm663OEAGpWHh7Hze-tpo_S6pFbxlZsqjw,2914
+tutorials/examples/5-upload_results/2-uploading_evaluate_results.py,sha256=SkDMtw_I4SZO1ooQpvwmq_ptOMzbYSSZlYuDEU2uJb4,3808
+tutorials/examples/5-upload_results/3-uploading_your_results.py,sha256=XdTNTChV78oFAL9fa58YHHsGRNRFzcQ8qJdzCfDBFdI,5562
+tutorials/examples/6-providers/aws/__init__.py,sha256=hgW4VxBUWhsxOLGErdoA5fo9JnXoj1Hz3E1tNLulHSY,63
+tutorials/examples/6-providers/aws/batch_example.py,sha256=3SSx85E36NV2UTpB6N-FimXQRZL-jvVw9-wee0xodyk,7279
+tutorials/examples/6-providers/portkey/__init__.py,sha256=YoLlgr7ULGjAL6BwSJ6hMmLWhI_rlEHpN-sn9QS0fa4,67
+tutorials/examples/6-providers/portkey/batch_example.py,sha256=ea4BQhiiOKwSJSmW6uoxHLMQVk1Ji3k_k_CZ-y9ixJM,4011
+tutorials/examples/6-providers/portkey/messages_example.py,sha256=9wKGZlqksrSuoxgDY2w9IWts4fBzmZmijRdHZzSRAQs,4397
+tutorials/examples/6-providers/vertex/__init__.py,sha256=zVlz0-hpUZDgYFjM-OmxcPsKT1lTE-inZyRAf20aHD8,77
+tutorials/examples/6-providers/vertex/batch_example.py,sha256=rOQKQ4BO8VFkiLYXcCmZByc6zfqM-Iu3LUMvzP_oioU,5433
+tutorials/examples/6-providers/vertex/messages_example.py,sha256=MeH9AiNtpi0BYL_TRGEY-B-d7eS6qGdc3d0usXSWMQY,4989
+tutorials/examples/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tutorials/notebooks/1-scoring.ipynb,sha256=0mWJlJuq2MY0OdS9Ts9BX8MH7G53Tv8rGJT4yEuCAv0,4663
+tutorials/notebooks/2-evaluating.ipynb,sha256=XjCtnypK0CJXDxI18RZocdvr28XWsJykGxs47Ig9SUc,9007
+tutorials/notebooks/3.1-adaptive_evaluation_phi.ipynb,sha256=r7_tAw-pswVxvVt8xA7-ewPHepMfkL1YA39p0u-UJyM,12269
+tutorials/notebooks/3.2-adaptive_evaluation_gpt.ipynb,sha256=6dhU2JFL7Mebu2VslY21WFy8C9lrzmo0gE9DQk8kj9w,9160
+tutorials/notebooks/4-uploading_results.ipynb,sha256=a7DzNglzkBoCkN-dyncixzKNkEFalGaHWsLvhu3Kyd4,10361
+tutorials/quickstarts/adaptive_evaluations/adaptive_evaluation_openai_demo.ipynb,sha256=MZoHegB31jnI6wbXVtC0B2kCeNn_hyUf53m0B8G1LJw,7701
+tutorials/quickstarts/adaptive_evaluations/adaptive_evaluation_qwen_demo.ipynb,sha256=1CIbdr_Y5XcL0iS9GcNs_Al62Mv2hSgdxvoj3lP1zU8,8428
+tutorials/quickstarts/classical_evaluations/classical_evaluation_demo.ipynb,sha256=f7Kjy5AVK_nNxTd3RFtFPmFCEkhr3a17aUvVDvlGyKQ,9295
+tutorials/quickstarts/getting_started.ipynb,sha256=5083CRiTlH-LfGdwjKwBDtn5i2c4FQUGxn0bRgE_DGw,7117
+tutorials/utils/__init__.py,sha256=bI1-auvdIRV53jpT3bWRuKHUhZ92qHVJA_4S6xBB4P0,807
+tutorials/utils/args_parser.py,sha256=SZeqhNaSosw7iT5u0Wkb9m0XqXRx_DAitGXb7QK2nEM,4011
+tutorials/utils/output.py,sha256=4kVdySdxOl8cjSy-as4O71AhJOj7OlGXiOmlww4cZ-E,714
+tutorials/utils/setup.py,sha256=6obslk7PIgUlD4RyAi8SaeoB9ioxM0Vhda65ptovEKQ,3470
+scorebook-0.0.15.dist-info/METADATA,sha256=IOkRQaLVGMQIyfJbkIOqOZxZk90PCANAI3VP3w32PJg,10759
+scorebook-0.0.15.dist-info/WHEEL,sha256=kJCRJT_g0adfAJzTx2GUMmS80rTJIVHRCfG0DQgLq3o,88
+scorebook-0.0.15.dist-info/entry_points.txt,sha256=9gNd3Q0MEozhJ7fog-Q-Z_PrcGMnF-404Jon40MH2_U,53
+scorebook-0.0.15.dist-info/licenses/LICENSE,sha256=JLH1g9FhxHZf6CBCeQ_xAisPtICVObuNGW1bLPiTYEs,1068
+scorebook-0.0.15.dist-info/RECORD,,

{scorebook-0.0.14.dist-info → scorebook-0.0.15.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 2.2.1
+Generator: poetry-core 2.3.1
 Root-Is-Purelib: true
 Tag: py3-none-any

tutorials/README.md

@@ -0,0 +1,147 @@
+# Scorebook Tutorials
+
+This directory contains tutorials, examples, and quickstarts for learning Scorebook - a Python framework for evaluating large language models.
+
+## Directory Structure
+
+```
+tutorials/
+├── quickstarts/           # Quick start notebooks for getting up and running
+├── notebooks/             # Interactive Jupyter notebooks
+├── examples/              # Python script examples organized by topic
+│   ├── 1-score/           # Scoring pre-computed outputs
+│   ├── 2-evaluate/        # Running evaluations
+│   ├── 3-evaluation_datasets/  # Loading datasets
+│   ├── 4-adaptive_evaluations/ # Trismik adaptive testing
+│   ├── 5-upload_results/  # Uploading to Trismik
+│   └── 6-providers/       # Cloud provider integrations
+└── utils/                 # Helper utilities for examples
+```
+
+## Getting Started
+
+### Prerequisites
+
+- Python 3.9+
+- Install Scorebook: `pip install scorebook`
+- For local model examples: `pip install scorebook[examples]` (includes transformers, torch)
+- For cloud examples: `pip install scorebook[openai]` and set `OPENAI_API_KEY`
+- For Trismik features: Set `TRISMIK_API_KEY` environment variable
+
+### Quickstarts
+
+Start here for a rapid introduction:
+
+```bash
+jupyter notebook tutorials/quickstarts/getting_started.ipynb
+```
+
+Available quickstarts:
+- `getting_started.ipynb` - Introduction to Scorebook basics
+- `classical_evaluations/` - Standard evaluation workflows
+- `adaptive_evaluations/` - Trismik's adaptive testing feature
+
+### Notebooks
+
+Interactive tutorials covering core concepts:
+
+```bash
+jupyter notebook tutorials/notebooks/
+```
+
+| Notebook | Description |
+|----------|-------------|
+| `1-scoring.ipynb` | Score pre-computed model outputs |
+| `2-evaluating.ipynb` | Run full evaluation pipelines |
+| `3.1-adaptive_evaluation_phi.ipynb` | Adaptive evaluation with local models |
+| `3.2-adaptive_evaluation_gpt.ipynb` | Adaptive evaluation with OpenAI |
+| `4-uploading_results.ipynb` | Upload results to Trismik dashboard |
+
+## Examples
+
+Python scripts demonstrating specific features. Run examples from the project root:
+
+```bash
+python tutorials/examples/1-score/1-scoring_model_accuracy.py
+```
+### 1-score: Scoring Pre-computed Outputs
+
+Score model predictions that have already been generated.
+
+| Example | Description |
+|---------|-------------|
+| `1-scoring_model_accuracy.py` | Score outputs using accuracy metric |
+| `2-scoring_model_bleu.py` | Score using BLEU metric |
+| `3-scoring_model_f1.py` | Score using F1 metric |
+| `4-scoring_model_rouge.py` | Score using ROUGE metric |
+
+### 2-evaluate: Running Evaluations
+
+End-to-end evaluation workflows with inference.
+
+| Example | Description | Requirements |
+|---------|-------------|--------------|
+| `1-evaluating_local_models.py` | Basic evaluation with local HuggingFace model | - |
+| `2-evaluating_local_models_with_batching.py` | Batch processing for improved throughput | - |
+| `3-evaluating_cloud_models.py` | Evaluate using OpenAI API | OpenAI API key |
+| `4-evaluating_cloud_models_with_batching.py` | OpenAI Batch API for cost savings | OpenAI API key |
+| `5-hyperparameter_sweeps.py` | Test multiple hyperparameter configurations | - |
+| `6-inference_pipelines.py` | Modular preprocessing/inference/postprocessing | - |
+
+### 3-evaluation_datasets: Loading Datasets
+
+Different ways to load evaluation data.
+
+| Example | Description | Requirements |
+|---------|-------------|--------------|
+| `1-evaluation_datasets_from_files.py` | Load from JSON/CSV files | - |
+| `2-evaluation_datasets_from_huggingface.py` | Load from HuggingFace Hub | OpenAI API key |
+| `3-evaluation_datasets_from_huggingface_with_yaml_configs.py` | Use YAML configs for HuggingFace datasets | OpenAI API key |
+
+### 4-adaptive_evaluations: Trismik Adaptive Testing
+
+Efficient evaluation using Item Response Theory (IRT).
+
+| Example | Description | Requirements |
+|---------|-------------|--------------|
+| `1-adaptive_evaluation.py` | Basic adaptive evaluation | Trismik + OpenAI |
+| `2-adaptive_dataset_splits.py` | Adaptive evaluation with dataset splits | Trismik + OpenAI |
+
+### 5-upload_results: Uploading to Trismik
+
+Persist and share results on the Trismik dashboard.
+
+| Example | Description | Requirements |
+|---------|-------------|--------------|
+| `1-uploading_score_results.py` | Upload `score()` results | Trismik API key |
+| `2-uploading_evaluate_results.py` | Upload `evaluate()` results | Trismik API key |
+| `3-uploading_your_results.py` | Upload custom results | Trismik API key |
+
+### 6-providers: Cloud Provider Integrations
+
+Batch processing with different cloud providers.
+
+#### AWS Bedrock (`6-providers/aws/`)
+- `batch_example.py` - Batch inference with Claude models via AWS Bedrock
+
+**Requirements:** AWS CLI configured, S3 bucket, IAM role for Bedrock
+
+#### Google Cloud Vertex AI (`6-providers/vertex/`)
+- `batch_example.py` - Batch inference with Gemini models
+- `messages_example.py` - Real-time inference with Gemini
+
+**Requirements:** Google Cloud SDK, Vertex AI enabled project
+
+#### Portkey (`6-providers/portkey/`)
+- `batch_example.py` - Batch inference via Portkey gateway
+- `messages_example.py` - Real-time inference via Portkey
+
+**Requirements:** Portkey API key, linked provider account
+
+## Additional Resources
+
+- [Scorebook Documentation](https://docs.trismik.com/)
+- [Trismik Platform](https://trismik.com)
+- [API Reference](https://docs.trismik.com/category/reference/)
+- [GitHub Issues](https://github.com/trismik/scorebook/issues)
+- Contact support at support@trismik.com

tutorials/__init__.py

@@ -0,0 +1,5 @@
+"""Scorebook tutorials package.
+
+This package contains tutorial examples and utilities for learning Scorebook.
+The tutorials are optional and not required for using the core Scorebook package.
+"""

tutorials/examples/1-score/1-scoring_model_accuracy.py

@@ -0,0 +1,47 @@
+"""Tutorials - Score - Example 1 - Scoring Models."""
+
+from pathlib import Path
+from pprint import pprint
+from typing import Any
+
+from dotenv import load_dotenv
+
+from tutorials.utils import save_results_to_json, setup_logging
+
+from scorebook import score
+from scorebook.metrics.accuracy import Accuracy
+
+
+def main() -> Any:
+    """Score pre-computed model predictions using Scorebook.
+
+    This example demonstrates how to score generated model predictions.
+    """
+
+    # Prepare a list of items with generated outputs and labels
+    model_predictions = [
+        {"output": "4", "label": "4"},
+        {"output": "Paris", "label": "Paris"},
+        {"output": "George R. R. Martin", "label": "William Shakespeare"},
+    ]
+
+    # Score the predictions against labels using the accuracy metric
+    results = score(
+        items=model_predictions,
+        metrics=Accuracy,
+        upload_results=False,  # Disable uploading for this example
+    )
+
+    print("\nResults:")
+    pprint(results)
+
+    return results
+
+
+if __name__ == "__main__":
+    load_dotenv()
+    log_file = setup_logging(experiment_id="1-scoring_model_accuracy", base_dir=Path(__file__).parent)
+    output_dir = Path(__file__).parent / "results"
+    output_dir.mkdir(exist_ok=True)
+    results_dict = main()
+    save_results_to_json(results_dict, output_dir, "1-scoring_model_accuracy_output.json")

tutorials/examples/1-score/2-scoring_model_bleu.py

@@ -0,0 +1,46 @@
+"""Tutorials - Score - Example 2 - Scoring Models with BLEU."""
+
+from pathlib import Path
+from pprint import pprint
+from typing import Any
+
+from dotenv import load_dotenv
+
+from tutorials.utils import save_results_to_json, setup_logging
+from scorebook import score
+from scorebook.metrics.bleu import BLEU
+
+
+def main() -> Any:
+    """Score pre-computed model predictions using Scorebook.
+
+    This example demonstrates how to score generated model predictions.
+    """
+
+    # Prepare a list of items with generated outputs and labels
+    model_predictions = [
+        {"output": "28-jähriger Koch wurde in San Francisco Mall entdeckt.", "label": "28-jähriger Koch in San Francisco Mall tot aufgefunden"},
+        {"output": "Ein 28-jähriger Koch, der kürzlich nach San Francisco gezogen war, wurde in der Treppe eines lokalen Einkaufszentrums dieser Woche ermordet.", "label": "Ein 28-jähriger Koch, der vor kurzem nach San Francisco gezogen ist, wurde im Treppenhaus eines örtlichen Einkaufzentrums tot aufgefunden."},
+        {"output": 'Der Bruder des Opfers sagt, er könne sich nicht vorstellen, wer ihm schaden wolle, und sagt: "Die Dinge waren endlich gut für ihn."', "label": 'Der Bruder des Opfers sagte aus, dass er sich niemanden vorstellen kann, der ihm schaden wollen würde, "Endlich ging es bei ihm wieder bergauf."'},
+    ]
+
+    # Score the predictions against labels using the accuracy metric
+    results = score(
+        items=model_predictions,
+        metrics=BLEU(compact=False),
+        upload_results=False,  # Disable uploading for this example
+    )
+
+    print("\nResults:")
+    pprint(results)
+
+    return results
+
+
+if __name__ == "__main__":
+    load_dotenv()
+    log_file = setup_logging(experiment_id="2-scoring_model_bleu", base_dir=Path(__file__).parent)
+    output_dir = Path(__file__).parent / "results"
+    output_dir.mkdir(exist_ok=True)
+    results_dict = main()
+    save_results_to_json(results_dict, output_dir, "2-scoring_model_bleu_output.json")

tutorials/examples/1-score/3-scoring_model_f1.py

@@ -0,0 +1,64 @@
+"""Tutorials - Score - Example 3 - F1 Metric Scoring."""
+
+from pathlib import Path
+from pprint import pprint
+from typing import Any
+
+from dotenv import load_dotenv
+
+from tutorials.utils import save_results_to_json, setup_logging
+from scorebook import score
+from scorebook.metrics.f1 import F1
+
+
+def main() -> Any:
+    """Score pre-computed model predictions using F1 metric.
+
+    This example demonstrates how to score NER (Named Entity Recognition)
+    predictions using the F1 metric with different averaging methods.
+    """
+
+    # Sample NER predictions (in CoNLL format with BIO tags)
+    model_predictions = [
+        {"output": "O", "label": "O"},
+        {"output": "B-PER", "label": "B-PER"},
+        {"output": "I-PER", "label": "I-PER"},
+        {"output": "O", "label": "O"},
+        {"output": "B-LOC", "label": "B-LOC"},
+        {"output": "O", "label": "O"},
+        {"output": "B-ORG", "label": "B-LOC"},  # Misclassification
+        {"output": "O", "label": "B-MISC"},     # Missed entity
+        {"output": "B-PER", "label": "B-PER"},
+        {"output": "O", "label": "O"},
+    ]
+
+    print(f"Scoring {len(model_predictions)} NER predictions\n")
+
+    # Score with all averaging methods at once
+    print("All averaging methods:")
+    results_all = score(
+        items=model_predictions,
+        metrics=F1(average="all"),
+        upload_results=False,
+    )
+    pprint(results_all["aggregate_results"])
+
+    # Score with specific combination of methods
+    print("\nMicro and weighted averaging:")
+    results_combo = score(
+        items=model_predictions,
+        metrics=F1(average=["micro", "weighted"]),
+        upload_results=False,
+    )
+    pprint(results_combo["aggregate_results"])
+
+    return results_all
+
+
+if __name__ == "__main__":
+    load_dotenv()
+    log_file = setup_logging(experiment_id="3-scoring_f1_metric", base_dir=Path(__file__).parent)
+    output_dir = Path(__file__).parent / "results"
+    output_dir.mkdir(exist_ok=True)
+    results_dict = main()
+    save_results_to_json(results_dict, output_dir, "3-scoring_f1_metric_output.json")

tutorials/examples/1-score/4-scoring_model_rouge.py

@@ -0,0 +1,64 @@
+"""Tutorials - Score - Example 4 - Scoring Models with ROUGE."""
+
+from pathlib import Path
+from pprint import pprint
+from typing import Any
+
+from dotenv import load_dotenv
+
+from tutorials.utils import save_results_to_json, setup_logging
+
+from scorebook import score
+from scorebook.metrics.rouge import ROUGE
+
+
+def main() -> Any:
+    """Score text generation predictions using ROUGE metric.
+
+    This example demonstrates how to score generated summaries
+    against reference summaries using ROUGE scores.
+    """
+
+    # Prepare a list of items with generated summaries and reference summaries
+    model_predictions = [
+        {
+            "output": "A woman donated her kidney to a stranger. This sparked a chain of six kidney transplants.",
+            "label": "Zully Broussard decided to give a kidney to a stranger. A new computer program helped her donation spur transplants for six kidney patients.",
+        },
+        {
+            "output": "Scientists discovered a new species of frog in the Amazon rainforest. The frog has unique markings that distinguish it from other species.",
+            "label": "A new frog species with distinctive blue and yellow stripes was found in the Amazon. Researchers say this discovery highlights the biodiversity of the region.",
+        },
+        {
+            "output": "The technology company released its quarterly earnings report showing strong growth.",
+            "label": "Tech giant announces record quarterly revenue driven by cloud services and AI products.",
+        },
+    ]
+
+    # Score the predictions against labels using the ROUGE metric
+    results = score(
+        items=model_predictions,
+        metrics=ROUGE(rouge_types=["rouge1", "rougeL"], use_stemmer=True),
+        upload_results=False,  # Disable uploading for this example
+    )
+
+    print("\nResults:")
+    pprint(results)
+
+    # Display individual item scores
+    print("\n\nIndividual ROUGE Scores:")
+    for i, item_score in enumerate(results["item_results"]):
+        print(f"\nItem {i+1}:")
+        print(f"  ROUGE-1 F1: {item_score['rouge1']:.4f}")
+        print(f"  ROUGE-L F1: {item_score['rougeL']:.4f}")
+
+    return results
+
+
+if __name__ == "__main__":
+    load_dotenv()
+    log_file = setup_logging(experiment_id="4-scoring_model_rouge", base_dir=Path(__file__).parent)
+    output_dir = Path(__file__).parent / "results"
+    output_dir.mkdir(exist_ok=True)
+    results_dict = main()
+    save_results_to_json(results_dict, output_dir, "4-scoring_model_rouge_output.json")