scorebook 0.0.1__py3-none-any.whl → 0.0.3__py3-none-any.whl

scorebook/utils/logging_utils.py

@@ -0,0 +1 @@
+"""Logging utilities for Scorebook evaluation framework."""

scorebook/utils/progress_bars.py

@@ -9,20 +9,33 @@ from tqdm import tqdm
 class EvaluationProgressBars:
     """Manages nested progress bars for evaluation tracking."""
 
-    def __init__(self, datasets: List[Any], hyperparam_count: int) -> None:
+    def __init__(
+        self, datasets: List[Any], hyperparam_count: int, parallel: bool, total_eval_runs: int
+    ) -> None:
         """Initialize progress bar manager.
 
         Args:
             datasets: List of datasets being evaluated
             hyperparam_count: Number of hyperparameter configurations per dataset
+            parallel: Whether running in parallel mode
+            total_eval_runs: Total number of EvalRunSpecs (dataset_count * hyperparam_count)
         """
         self.datasets = datasets
         self.hyperparam_count = hyperparam_count
+        self.parallel = parallel
+        self.total_eval_runs = total_eval_runs
+
         self.dataset_pbar: Optional[tqdm] = None
         self.hyperparam_pbar: Optional[tqdm] = None
 
-    def start_dataset_progress(self) -> None:
-        """Start the outer progress bar for datasets."""
+        # Track progress per dataset
+        self.current_dataset_idx = 0
+        self.completed_hyperparams_per_dataset: dict[int, int] = {}
+        self.completed_eval_runs = 0
+
+    def start_progress_bars(self) -> None:
+        """Start both progress bars."""
+        # Top level: Datasets
         self.dataset_pbar = tqdm(
             total=len(self.datasets),
             desc="Datasets   ",
@@ -33,57 +46,101 @@ class EvaluationProgressBars:
             bar_format="{desc} {percentage:3.0f}%|{bar:40}| {n_fmt}/{total_fmt}",
         )
 
-    def update_dataset_progress(self) -> None:
-        """Update the dataset progress bar."""
-        if self.dataset_pbar:
-            self.dataset_pbar.update(1)
+        # Bottom level: Hyperparameters/Eval runs
+        if self.parallel:
+            # In parallel mode: show eval runs completed out of total
+            self.hyperparam_pbar = tqdm(
+                total=self.total_eval_runs,
+                desc="Eval Runs  ",
+                unit="run",
+                position=1,
+                leave=False,
+                ncols=80,
+                bar_format="{desc} {percentage:3.0f}%|{bar:40}| {n_fmt}/{total_fmt}",
+            )
+        else:
+            # In sequential mode: show hyperparams per dataset
+            self.hyperparam_pbar = tqdm(
+                total=self.hyperparam_count,
+                desc="Hyperparams",
+                unit="config",
+                position=1,
+                leave=False,
+                ncols=80,
+                bar_format="{desc} {percentage:3.0f}%|{bar:40}| {n_fmt}/{total_fmt}",
+            )
 
-    def close_dataset_progress(self) -> None:
-        """Close the dataset progress bar."""
-        if self.dataset_pbar:
-            self.dataset_pbar.close()
-            self.dataset_pbar = None
+    def on_eval_run_completed(self, dataset_idx: int) -> None:
+        """Update progress when an eval run (EvalRunSpec) completes in parallel mode."""
+        if not self.parallel:
+            return
 
-    @contextmanager
-    def hyperparam_progress_context(self) -> Generator[tqdm, None, None]:
-        """Context manager for hyperparameter progress bar."""
-        self.hyperparam_pbar = tqdm(
-            total=self.hyperparam_count,
-            desc="Hyperparams",
-            unit="config",
-            position=1,
-            leave=False,
-            ncols=80,
-            bar_format="{desc} {percentage:3.0f}%|{bar:40}| {n_fmt}/{total_fmt}",
+        self.completed_eval_runs += 1
+        if self.hyperparam_pbar:
+            self.hyperparam_pbar.update(1)
+
+        # Track how many runs completed for this dataset
+        self.completed_hyperparams_per_dataset[dataset_idx] = (
+            self.completed_hyperparams_per_dataset.get(dataset_idx, 0) + 1
         )
-        try:
-            yield self.hyperparam_pbar
-        finally:
-            self.hyperparam_pbar.close()
-            self.hyperparam_pbar = None
 
-    def update_hyperparam_progress(self) -> None:
-        """Update the hyperparameter progress bar."""
+        # Check if this dataset is complete
+        if self.completed_hyperparams_per_dataset[dataset_idx] == self.hyperparam_count:
+            if self.dataset_pbar:
+                self.dataset_pbar.update(1)
+
+    def on_hyperparam_completed(self, dataset_idx: int) -> None:
+        """Update progress when a hyperparameter config completes in sequential mode."""
+        if self.parallel:
+            return
+
         if self.hyperparam_pbar:
             self.hyperparam_pbar.update(1)
 
+        # Track completed hyperparams for this dataset
+        self.completed_hyperparams_per_dataset[dataset_idx] = (
+            self.completed_hyperparams_per_dataset.get(dataset_idx, 0) + 1
+        )
+
+        # Check if this dataset is complete
+        if self.completed_hyperparams_per_dataset[dataset_idx] == self.hyperparam_count:
+            # Update dataset progress
+            if self.dataset_pbar:
+                self.dataset_pbar.update(1)
+
+            # Reset hyperparameter progress for next dataset (if any)
+            if dataset_idx < len(self.datasets) - 1:
+                if self.hyperparam_pbar:
+                    self.hyperparam_pbar.reset()
+
+    def close_progress_bars(self) -> None:
+        """Close both progress bars."""
+        if self.hyperparam_pbar:
+            self.hyperparam_pbar.close()
+            self.hyperparam_pbar = None
+        if self.dataset_pbar:
+            self.dataset_pbar.close()
+            self.dataset_pbar = None
+
 
 @contextmanager
 def evaluation_progress(
-    datasets: List[Any], hyperparam_count: int
+    datasets: List[Any], hyperparam_count: int, parallel: bool, total_eval_runs: int
 ) -> Generator[EvaluationProgressBars, None, None]:
     """Context manager for evaluation progress bars.
 
     Args:
         datasets: List of datasets being evaluated
         hyperparam_count: Number of hyperparameter configurations per dataset
+        parallel: Whether running in parallel mode
+        total_eval_runs: Total number of EvalRunSpecs
 
     Yields:
         EvaluationProgressBars: Progress bar manager instance
     """
-    progress_bars = EvaluationProgressBars(datasets, hyperparam_count)
-    progress_bars.start_dataset_progress()
+    progress_bars = EvaluationProgressBars(datasets, hyperparam_count, parallel, total_eval_runs)
+    progress_bars.start_progress_bars()
     try:
         yield progress_bars
     finally:
-        progress_bars.close_dataset_progress()
+        progress_bars.close_progress_bars()

{scorebook-0.0.1.dist-info → scorebook-0.0.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: scorebook
-Version: 0.0.1
+Version: 0.0.3
 Summary: A Python project for LLM evaluation.
 Author: Euan Campbell
 Author-email: euan@trismik.com
@@ -11,16 +11,26 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Provides-Extra: bedrock
 Provides-Extra: examples
 Provides-Extra: openai
 Provides-Extra: portkey
+Provides-Extra: vertex
 Requires-Dist: accelerate ; extra == "examples"
+Requires-Dist: boto3 (==1.40.0) ; extra == "bedrock"
 Requires-Dist: datasets (>=3.6.0)
+Requires-Dist: fsspec[gcs] ; extra == "vertex"
+Requires-Dist: google-cloud-storage ; extra == "vertex"
+Requires-Dist: google-genai ; extra == "vertex"
+Requires-Dist: notebook (>=7.4.5,<8.0.0)
 Requires-Dist: notebook ; extra == "examples"
 Requires-Dist: openai ; extra == "openai"
+Requires-Dist: pandas ; extra == "vertex"
 Requires-Dist: portkey-ai ; extra == "portkey"
+Requires-Dist: python-dotenv ; extra == "bedrock"
 Requires-Dist: python-dotenv ; extra == "openai"
 Requires-Dist: python-dotenv ; extra == "portkey"
+Requires-Dist: python-dotenv ; extra == "vertex"
 Requires-Dist: torch ; extra == "examples"
 Requires-Dist: torchaudio ; extra == "examples"
 Requires-Dist: torchvision ; extra == "examples"

scorebook-0.0.3.dist-info/RECORD

@@ -0,0 +1,31 @@
+scorebook/__init__.py,sha256=7ac3KpXU3kKFekq8mZ3cVbF7oQ6Q9E-uqX7ijyte1Q0,406
+scorebook/evaluator.py,sha256=mS3G3PI26nHzqkYX4tqusQZJL5Q1xTxzqshAdwscl0s,14170
+scorebook/exceptions.py,sha256=emq2QY-4mW6VXlq1dxunPjt-xZpLQIxo8Ck_gYxz1VE,1827
+scorebook/inference/__init__.py,sha256=tqSXSyVurc_YRfPypYed8iTH7Fwt7iFCXMxBXnqY-9I,242
+scorebook/inference/bedrock.py,sha256=wllq0ysNFQKWJDEqoN-k96Jx43BHCAvfxm14zMRCf90,10074
+scorebook/inference/openai.py,sha256=FqXua4v4PTYSHrdTm_9fM0Us8Mo2n2LSN94CwRipRw4,7658
+scorebook/inference/portkey.py,sha256=OHSS-sa2aLxuO6fEfG8MsPlhXc_95_-6j7ImbCkY8KE,5952
+scorebook/inference/vertex.py,sha256=jv_Nbt1NJQ6mMUyEuW_idxhj_3fugBojshtpGP9fMeY,9874
+scorebook/metrics/__init__.py,sha256=be_riJNojebXw2xfkMsHHjl3HFKgk9jQWlLkXJHhheI,782
+scorebook/metrics/accuracy.py,sha256=5KQ4hfOn9M94sB7WsXUelJWJiuKfoCGQEl5q5q9vNfo,1467
+scorebook/metrics/metric_base.py,sha256=I3L0DGcRojFp93UGFnXG1tZ2UK9ilTcXXJG6lj5ddXA,857
+scorebook/metrics/metric_registry.py,sha256=jWwt9P3zvtFLlEYrd60v7LS7X251nZczouE02zcCxWg,3402
+scorebook/metrics/precision.py,sha256=AaYPYYKnY74Nwqp_p3jd2Ewf3VHNOJjoRWf5fhb-tXk,563
+scorebook/types/__init__.py,sha256=dXY3Y-GiMipVExzVu7H5pbdFfg4HBMEKxqSTfENywSs,427
+scorebook/types/eval_dataset.py,sha256=dCqOHjGaEb7pGG1VF4aGFn6hngFvlxpxddqsDtM4nTs,13870
+scorebook/types/eval_result.py,sha256=R2zuWrx8p9_4A2W3Gmlp-xGgmelPdg8QB5PoV1hiqRc,4728
+scorebook/types/eval_run_spec.py,sha256=nf7LGa_dG60Qb385W6O6qiu7VlJ03-dpo2X1PgKGcRQ,845
+scorebook/types/inference_pipeline.py,sha256=-HcGGbwM34fGJ_FlXcyqj_pV6DjWIXRGgICiN_63UsU,3251
+scorebook/utils/__init__.py,sha256=l_bfi9lAMz1oyGnuyKuzYasQKt2DJwffqsbfSl4-GIQ,452
+scorebook/utils/async_utils.py,sha256=OeNvMrOT9P4rIyaCf5IbR3ZIFMtEzXgoAArNbINRtMU,728
+scorebook/utils/build_prompt.py,sha256=L_Y84a1ewm3GvwnSSuUXfPO_M0QL1Dl8UgOS_l_zvh4,1617
+scorebook/utils/io_helpers.py,sha256=ksOJ9ILcZqqt-HwRUYy1NMQbS6RuMh8i2ZzUADLMlQ8,913
+scorebook/utils/jinja_helpers.py,sha256=ksIKHiKdj8N0o7ZJZGasfbSNoAY6K5d9X_KM6mcKYD4,4208
+scorebook/utils/logging_utils.py,sha256=M4BXt369mJo037WYpvuWDoe3oGWVdHWaGo4Vbl0WDL0,60
+scorebook/utils/mappers.py,sha256=OcUnPBrnSUxZNhAzJhVmVWUWmqIKFXLTrK-xLi6_SUg,1259
+scorebook/utils/progress_bars.py,sha256=TBz41w3yFujsO9n8vUjeubgOrmdiAMI2P2SSVqTJzAA,5269
+scorebook/utils/transform_helpers.py,sha256=UnVLtFvcJrtmBEmLsuA4rrX4iJlNUKxm2DkIOGLl-2o,1030
+scorebook-0.0.3.dist-info/LICENSE,sha256=JLH1g9FhxHZf6CBCeQ_xAisPtICVObuNGW1bLPiTYEs,1068
+scorebook-0.0.3.dist-info/METADATA,sha256=i0tLm4SNSiPTNEP8QU0ZjsfOqizw4uu3GWPVqdxrcso,11409
+scorebook-0.0.3.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+scorebook-0.0.3.dist-info/RECORD,,

scorebook-0.0.1.dist-info/RECORD

@@ -1,24 +0,0 @@
-scorebook/__init__.py,sha256=cYv8bT3_7o2MTxPVKiv51DcpaPtH_A9qOH5yF_FULZo,336
-scorebook/evaluator.py,sha256=Ce4KerLVPlaF63xng9RKH9M1l-ldo3mdrd3T2dBs_YE,8908
-scorebook/inference/__init__.py,sha256=sU_ZSN9eO7ajZ-QklNpx8_gf3jCdDn69J-SfU0z07-E,333
-scorebook/inference/openai.py,sha256=XD1dbPrEHQJVXOMtqCt9a0yQ-qR381N5XXhCrgz8jio,5826
-scorebook/inference/portkey.py,sha256=OHSS-sa2aLxuO6fEfG8MsPlhXc_95_-6j7ImbCkY8KE,5952
-scorebook/metrics/__init__.py,sha256=be_riJNojebXw2xfkMsHHjl3HFKgk9jQWlLkXJHhheI,782
-scorebook/metrics/accuracy.py,sha256=5KQ4hfOn9M94sB7WsXUelJWJiuKfoCGQEl5q5q9vNfo,1467
-scorebook/metrics/metric_base.py,sha256=I3L0DGcRojFp93UGFnXG1tZ2UK9ilTcXXJG6lj5ddXA,857
-scorebook/metrics/metric_registry.py,sha256=jWwt9P3zvtFLlEYrd60v7LS7X251nZczouE02zcCxWg,3402
-scorebook/metrics/precision.py,sha256=AaYPYYKnY74Nwqp_p3jd2Ewf3VHNOJjoRWf5fhb-tXk,563
-scorebook/types/__init__.py,sha256=xQMOae_fIBbeyeuqoa7SbNwjxAiVinPBbckOcUzo57U,358
-scorebook/types/eval_dataset.py,sha256=TeIeVHQ597NxedxaTEXohZO8gR5iAiDtJbCja_u69EI,11703
-scorebook/types/eval_result.py,sha256=y0vLN6RMgiz1lyai5ltmzDibBHE25-k9bTrQ7U27RZ8,4552
-scorebook/types/inference_pipeline.py,sha256=M3JgchpcVdhRJPzn3mh5ys6iivSt8eBmHIj4F5LcFYU,3167
-scorebook/utils/__init__.py,sha256=DmhS61OZ2nNWkGxDfVrMBwwiH7dmLAbg3MHuNgaXhQg,382
-scorebook/utils/async_utils.py,sha256=OeNvMrOT9P4rIyaCf5IbR3ZIFMtEzXgoAArNbINRtMU,728
-scorebook/utils/io_helpers.py,sha256=ksOJ9ILcZqqt-HwRUYy1NMQbS6RuMh8i2ZzUADLMlQ8,913
-scorebook/utils/mappers.py,sha256=OcUnPBrnSUxZNhAzJhVmVWUWmqIKFXLTrK-xLi6_SUg,1259
-scorebook/utils/progress_bars.py,sha256=BlKqYlXDbik5eUn5nf5f7QnMvnTj8CU_CfXKxCWp3Ww,2909
-scorebook/utils/transform_helpers.py,sha256=UnVLtFvcJrtmBEmLsuA4rrX4iJlNUKxm2DkIOGLl-2o,1030
-scorebook-0.0.1.dist-info/LICENSE,sha256=JLH1g9FhxHZf6CBCeQ_xAisPtICVObuNGW1bLPiTYEs,1068
-scorebook-0.0.1.dist-info/METADATA,sha256=oiwYbuJkRVkoFZkIAQej09LdG5xBLxhKPy2ozWTV-_w,10976
-scorebook-0.0.1.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-scorebook-0.0.1.dist-info/RECORD,,