scorebook 0.0.8__py3-none-any.whl → 0.0.10__py3-none-any.whl

scorebook/utils/async_utils.py

@@ -1,7 +1,10 @@
 """Async utilities for handling callable objects and coroutines."""
 
 import asyncio
-from typing import Callable
+from contextlib import asynccontextmanager
+from typing import AsyncIterator, Callable, Optional, TypeVar
+
+T = TypeVar("T")
 
 
 def is_awaitable(obj: Callable) -> bool:
@@ -25,3 +28,19 @@ def is_awaitable(obj: Callable) -> bool:
         return True
 
     return False
+
+
+@asynccontextmanager
+async def async_nullcontext(value: Optional[T] = None) -> AsyncIterator[Optional[T]]:
+    """Async version of contextlib.nullcontext for Python 3.9 compatibility.
+
+    contextlib.nullcontext() is sync-only and cannot be used with async with on Python 3.9.
+    This provides an async equivalent that can be used with async context managers.
+
+    Args:
+        value: Optional value to yield from the context manager
+
+    Yields:
+        The provided value
+    """
+    yield value

scorebook/utils/progress_bars.py

@@ -1,7 +1,7 @@
 """Progress bar utilities for evaluation tracking."""
 
 from contextlib import contextmanager
-from typing import Any, Generator, List, Optional
+from typing import Generator, Optional
 
 from tqdm import tqdm
 
@@ -9,20 +9,16 @@ from tqdm import tqdm
 class EvaluationProgressBars:
     """Manages nested progress bars for evaluation tracking."""
 
-    def __init__(
-        self, datasets: List[Any], hyperparam_count: int, parallel: bool, total_eval_runs: int
-    ) -> None:
+    def __init__(self, dataset_count: int, hyperparam_count: int, total_eval_runs: int) -> None:
         """Initialize progress bar manager.
 
         Args:
-            datasets: List of datasets being evaluated
+            dataset_count: Number of datasets being evaluated
             hyperparam_count: Number of hyperparameter configurations per dataset
-            parallel: Whether running in parallel mode
             total_eval_runs: Total number of EvalRunSpecs (dataset_count * hyperparam_count)
         """
-        self.datasets = datasets
+        self.dataset_count = dataset_count
         self.hyperparam_count = hyperparam_count
-        self.parallel = parallel
         self.total_eval_runs = total_eval_runs
 
         self.dataset_pbar: Optional[tqdm] = None
@@ -37,7 +33,7 @@ class EvaluationProgressBars:
         """Start both progress bars."""
         # Top level: Datasets
         self.dataset_pbar = tqdm(
-            total=len(self.datasets),
+            total=self.dataset_count,
             desc="Datasets   ",
             unit="dataset",
             position=0,
@@ -46,35 +42,19 @@ class EvaluationProgressBars:
             bar_format="{desc} {percentage:3.0f}%|{bar:40}| {n_fmt}/{total_fmt}",
         )
 
-        # Bottom level: Hyperparameters/Eval runs
-        if self.parallel:
-            # In parallel mode: show eval runs completed out of total
-            self.hyperparam_pbar = tqdm(
-                total=self.total_eval_runs,
-                desc="Eval Runs  ",
-                unit="run",
-                position=1,
-                leave=False,
-                ncols=80,
-                bar_format="{desc} {percentage:3.0f}%|{bar:40}| {n_fmt}/{total_fmt}",
-            )
-        else:
-            # In sequential mode: show hyperparams per dataset
-            self.hyperparam_pbar = tqdm(
-                total=self.hyperparam_count,
-                desc="Hyperparams",
-                unit="config",
-                position=1,
-                leave=False,
-                ncols=80,
-                bar_format="{desc} {percentage:3.0f}%|{bar:40}| {n_fmt}/{total_fmt}",
-            )
+        # Bottom level: Eval runs
+        self.hyperparam_pbar = tqdm(
+            total=self.total_eval_runs,
+            desc="Eval Runs  ",
+            unit="run",
+            position=1,
+            leave=False,
+            ncols=80,
+            bar_format="{desc} {percentage:3.0f}%|{bar:40}| {n_fmt}/{total_fmt}",
+        )
 
     def on_eval_run_completed(self, dataset_idx: int) -> None:
-        """Update progress when an eval run (EvalRunSpec) completes in parallel mode."""
-        if not self.parallel:
-            return
-
+        """Update progress when an eval run (EvalRunSpec) completes."""
         self.completed_eval_runs += 1
         if self.hyperparam_pbar:
             self.hyperparam_pbar.update(1)
@@ -84,24 +64,6 @@ class EvaluationProgressBars:
             self.completed_hyperparams_per_dataset.get(dataset_idx, 0) + 1
         )
 
-        # Check if this dataset is complete
-        if self.completed_hyperparams_per_dataset[dataset_idx] == self.hyperparam_count:
-            if self.dataset_pbar:
-                self.dataset_pbar.update(1)
-
-    def on_hyperparam_completed(self, dataset_idx: int) -> None:
-        """Update progress when a hyperparameter config completes in sequential mode."""
-        if self.parallel:
-            return
-
-        if self.hyperparam_pbar:
-            self.hyperparam_pbar.update(1)
-
-        # Track completed hyperparams for this dataset
-        self.completed_hyperparams_per_dataset[dataset_idx] = (
-            self.completed_hyperparams_per_dataset.get(dataset_idx, 0) + 1
-        )
-
         # Check if this dataset is complete
         if self.completed_hyperparams_per_dataset[dataset_idx] == self.hyperparam_count:
             # Update dataset progress
@@ -109,7 +71,7 @@ class EvaluationProgressBars:
                 self.dataset_pbar.update(1)
 
             # Reset hyperparameter progress for next dataset (if any)
-            if dataset_idx < len(self.datasets) - 1:
+            if dataset_idx < self.dataset_count - 1:
                 if self.hyperparam_pbar:
                     self.hyperparam_pbar.reset()
 
@@ -125,20 +87,19 @@ class EvaluationProgressBars:
 
 @contextmanager
 def evaluation_progress(
-    datasets: List[Any], hyperparam_count: int, parallel: bool, total_eval_runs: int
+    dataset_count: int, hyperparameter_config_count: int, run_count: int
 ) -> Generator[EvaluationProgressBars, None, None]:
     """Context manager for evaluation progress bars.
 
     Args:
-        datasets: List of datasets being evaluated
-        hyperparam_count: Number of hyperparameter configurations per dataset
-        parallel: Whether running in parallel mode
-        total_eval_runs: Total number of EvalRunSpecs
+        dataset_count: Number of datasets being evaluated
+        hyperparameter_config_count: Number of hyperparameter configurations per dataset
+        run_count: Total number of EvalRunSpecs
 
     Yields:
         EvaluationProgressBars: Progress bar manager instance
     """
-    progress_bars = EvaluationProgressBars(datasets, hyperparam_count, parallel, total_eval_runs)
+    progress_bars = EvaluationProgressBars(dataset_count, hyperparameter_config_count, run_count)
     progress_bars.start_progress_bars()
     try:
         yield progress_bars

{scorebook-0.0.8.dist-info → scorebook-0.0.10.dist-info}/METADATA

@@ -1,18 +1,17 @@
 Metadata-Version: 2.4
 Name: scorebook
-Version: 0.0.8
+Version: 0.0.10
 Summary: A Python project for LLM evaluation.
 License-File: LICENSE
 Author: Euan Campbell
 Author-email: euan@trismik.com
-Requires-Python: >=3.9
+Requires-Python: >=3.9, <3.14
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
-Classifier: Programming Language :: Python :: 3.14
 Provides-Extra: bedrock
 Provides-Extra: examples
 Provides-Extra: openai
@@ -37,7 +36,7 @@ Requires-Dist: torch ; extra == "examples"
 Requires-Dist: torchaudio ; extra == "examples"
 Requires-Dist: torchvision ; extra == "examples"
 Requires-Dist: transformers ; extra == "examples"
-Requires-Dist: trismik (>=0.9.12)
+Requires-Dist: trismik (>=1.0.1,<2.0.0)
 Description-Content-Type: text/markdown
 
 # Scorebook

scorebook-0.0.10.dist-info/RECORD

@@ -0,0 +1,41 @@
+scorebook/__init__.py,sha256=tAe8v8xyiNcl7P4SUIM5dPVMqU8GQ8dKzJ1pfF6B-Ms,629
+scorebook/cli/__init__.py,sha256=E89jR1DljFSHhfjEGSRKLgz0KhxGyRQ9a3vpUOmQL9o,32
+scorebook/cli/auth.py,sha256=T6-5662Jh-HEhZvfUgy82BvxIiRzjZne-4LRp9Gb2JE,2937
+scorebook/cli/main.py,sha256=cEvShENl6L6feX_sa7FGNTeoz5UtwqzwenmcHaON1hg,1589
+scorebook/eval_dataset.py,sha256=LSTyxUkT06iEAVYCnjIDFxFgZzRejwiS5CZA-jvy1ns,15098
+scorebook/evaluate/__init__.py,sha256=m3mCjeLildghT86ZDwY4GxCmaYZmhjbxkuTk0M9S_mc,423
+scorebook/evaluate/_async/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scorebook/evaluate/_async/evaluate_async.py,sha256=vn8rjjveCCF6ItZWngqAP3RhfScHV_LlIomqh-z5-UU,15509
+scorebook/evaluate/_sync/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scorebook/evaluate/_sync/evaluate.py,sha256=4LVdXvCsPmSbkBxphJ9in5l17GL9Zqn66bZm9a8w9nc,15347
+scorebook/evaluate/evaluate_helpers.py,sha256=rAXUroMXfPkWqufMnA97bfscgPik38s3eeepe2RkchA,13026
+scorebook/exceptions.py,sha256=emq2QY-4mW6VXlq1dxunPjt-xZpLQIxo8Ck_gYxz1VE,1827
+scorebook/inference/__init__.py,sha256=u3TmfftO0oMkz8ngwxAKLPfL1so1w2hbK7c5UNlRq-M,345
+scorebook/inference/clients/__init__.py,sha256=QCjbrXYeFd7xK-5ZH7o7bSaKUJqHtGnH5285pezNKyY,242
+scorebook/inference/clients/bedrock.py,sha256=wllq0ysNFQKWJDEqoN-k96Jx43BHCAvfxm14zMRCf90,10074
+scorebook/inference/clients/openai.py,sha256=JPPcJdxYwwZNUfXCGTRRzzUUA8I8WiV3bu6-pgS1_UE,9043
+scorebook/inference/clients/portkey.py,sha256=OHSS-sa2aLxuO6fEfG8MsPlhXc_95_-6j7ImbCkY8KE,5952
+scorebook/inference/clients/vertex.py,sha256=jv_Nbt1NJQ6mMUyEuW_idxhj_3fugBojshtpGP9fMeY,9874
+scorebook/inference/inference_pipeline.py,sha256=SOr1xnglPvFMcJFSpDRLQ6222NJgy_-fVtZLC423TUE,5559
+scorebook/metrics/__init__.py,sha256=be_riJNojebXw2xfkMsHHjl3HFKgk9jQWlLkXJHhheI,782
+scorebook/metrics/accuracy.py,sha256=5KQ4hfOn9M94sB7WsXUelJWJiuKfoCGQEl5q5q9vNfo,1467
+scorebook/metrics/metric_base.py,sha256=I3L0DGcRojFp93UGFnXG1tZ2UK9ilTcXXJG6lj5ddXA,857
+scorebook/metrics/metric_registry.py,sha256=jWwt9P3zvtFLlEYrd60v7LS7X251nZczouE02zcCxWg,3402
+scorebook/metrics/precision.py,sha256=AaYPYYKnY74Nwqp_p3jd2Ewf3VHNOJjoRWf5fhb-tXk,563
+scorebook/settings.py,sha256=CgaumN98QpU7XKMugUG41UAO8oZVuWDco4uooSagFZY,596
+scorebook/trismik/__init__.py,sha256=kWZkEC57LJscRZNLE3sJR1L5w-ltb5mEbQd3_ePtZPQ,380
+scorebook/trismik/credentials.py,sha256=WtJLaNmBMwCi6gT1Bgp4J9x2tq5HDrDI9U074r08TnU,3275
+scorebook/types.py,sha256=zt8sGfbRjXatx1WtttWZDVIoiS-yhh_1lP0K4VHYvAM,5797
+scorebook/utils/__init__.py,sha256=3xdIXJzYEp9k23z4_49VWZtasoZN8tJxVPieE_HOuww,519
+scorebook/utils/async_utils.py,sha256=2ewk_VOePib8z7DTRl-pZQBGzVI3L3JvnEuYW-DTkRA,1325
+scorebook/utils/build_prompt.py,sha256=L_Y84a1ewm3GvwnSSuUXfPO_M0QL1Dl8UgOS_l_zvh4,1617
+scorebook/utils/io_helpers.py,sha256=ksOJ9ILcZqqt-HwRUYy1NMQbS6RuMh8i2ZzUADLMlQ8,913
+scorebook/utils/jinja_helpers.py,sha256=ksIKHiKdj8N0o7ZJZGasfbSNoAY6K5d9X_KM6mcKYD4,4208
+scorebook/utils/mappers.py,sha256=OcUnPBrnSUxZNhAzJhVmVWUWmqIKFXLTrK-xLi6_SUg,1259
+scorebook/utils/progress_bars.py,sha256=uLG_0s_QEHGgjZcVaDJ7wp14Rd3GY5dWu-F4FL8isJg,3783
+scorebook/utils/transform_helpers.py,sha256=UnVLtFvcJrtmBEmLsuA4rrX4iJlNUKxm2DkIOGLl-2o,1030
+scorebook-0.0.10.dist-info/METADATA,sha256=wJXBm9ZzeNYIrhUOz4Uc4D_5_1J8arUnMiOtR5BNeOA,11479
+scorebook-0.0.10.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+scorebook-0.0.10.dist-info/entry_points.txt,sha256=9gNd3Q0MEozhJ7fog-Q-Z_PrcGMnF-404Jon40MH2_U,53
+scorebook-0.0.10.dist-info/licenses/LICENSE,sha256=JLH1g9FhxHZf6CBCeQ_xAisPtICVObuNGW1bLPiTYEs,1068
+scorebook-0.0.10.dist-info/RECORD,,