scorebook 0.0.12__py3-none-any.whl → 0.0.13__py3-none-any.whl

scorebook/eval_datasets/eval_dataset.py

@@ -137,6 +137,24 @@ class EvalDataset:
             raise DatasetNotInitializedError("Dataset is not initialized")
         return list(map(str, self._hf_dataset.column_names))
 
+    @property
+    def split(self) -> Optional[str]:
+        """Return the split name of the underlying HuggingFace dataset, if available.
+
+        Returns:
+            The split name (e.g., "train", "test", "validation") if the dataset was loaded
+            from HuggingFace with a specific split. Returns None if the dataset was created
+            from a list, CSV, JSON, or loaded without a split specification.
+
+        Raises:
+            DatasetNotInitializedError: If the dataset is not initialized.
+        """
+        if self._hf_dataset is None:
+            raise DatasetNotInitializedError("Dataset is not initialized")
+
+        split = self._hf_dataset.split
+        return str(split) if split is not None else None
+
     def shuffle(self) -> None:
         """Randomly shuffle the dataset items."""
         if self._hf_dataset is None:

scorebook/evaluate/_async/evaluate_async.py

@@ -15,6 +15,7 @@ from scorebook.evaluate.evaluate_helpers import (
     make_trismik_inference,
     prepare_datasets,
     prepare_hyperparameter_configs,
+    resolve_adaptive_split,
     validate_parameters,
 )
 from scorebook.exceptions import InferenceError, ScoreBookError
@@ -40,6 +41,7 @@ logger = logging.getLogger(__name__)
 async def evaluate_async(
     inference: Union[Callable, InferencePipeline],
     datasets: Union[str, EvalDataset, List[Union[str, EvalDataset]]],
+    split: Optional[str] = None,
     hyperparameters: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
     metadata: Optional[Dict[str, Any]] = None,
     experiment_id: Optional[str] = None,
@@ -58,6 +60,7 @@ async def evaluate_async(
     Args:
         inference: The inference callable to evaluate
         datasets: Dataset(s) to evaluate on
+        split: Split to use for evaluation (default: "validation")
         hyperparameters: Hyperparameter configuration(s) to evaluate with
         metadata: Optional metadata to attach to the evaluation
         experiment_id: Optional experiment identifier
@@ -82,7 +85,7 @@ async def evaluate_async(
     validate_parameters(locals(), evaluate_async)
 
     # Prepare datasets, hyperparameters, and eval run specs
-    datasets = prepare_datasets(datasets, sample_size)
+    datasets = prepare_datasets(datasets, split, sample_size)
     hyperparameter_configs = prepare_hyperparameter_configs(hyperparameters)
     eval_run_specs = sorted(
         build_eval_run_specs(datasets, hyperparameter_configs, experiment_id, project_id, metadata),
@@ -378,8 +381,20 @@ async def run_adaptive_evaluation(
     Returns:
         Results from the adaptive evaluation
     """
+    # Fetch available splits from Trismik
+    dataset_info = await trismik_client.get_dataset_info(adaptive_run_spec.dataset)
+    available_splits = dataset_info.splits if hasattr(dataset_info, "splits") else []
+
+    # Resolve the split to use (with fallback: user-specified -> validation -> test)
+    resolved_split = resolve_adaptive_split(
+        test_id=adaptive_run_spec.dataset,
+        user_specified_split=adaptive_run_spec.split,
+        available_splits=available_splits,
+    )
+
     trismik_results = await trismik_client.run(
         test_id=adaptive_run_spec.dataset,
+        split=resolved_split,
         project_id=project_id,
         experiment=experiment_id,
         run_metadata=TrismikRunMetadata(

scorebook/evaluate/_sync/evaluate.py

@@ -14,6 +14,7 @@ from scorebook.evaluate.evaluate_helpers import (
     make_trismik_inference,
     prepare_datasets,
     prepare_hyperparameter_configs,
+    resolve_adaptive_split,
     validate_parameters,
 )
 from scorebook.exceptions import InferenceError, ScoreBookError
@@ -39,6 +40,7 @@ logger = logging.getLogger(__name__)
 def evaluate(
     inference: Union[Callable, InferencePipeline],
     datasets: Union[str, EvalDataset, List[Union[str, EvalDataset]]],
+    split: Optional[str] = None,
     hyperparameters: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None,
     metadata: Optional[Dict[str, Any]] = None,
     experiment_id: Optional[str] = None,
@@ -57,6 +59,7 @@ def evaluate(
     Args:
         inference: The inference callable to evaluate
         datasets: Dataset(s) to evaluate on
+        split: Split to use for evaluation (default: "validation")
         hyperparameters: Hyperparameter configuration(s) to evaluate with
         metadata: Optional metadata to attach to the evaluation
         experiment_id: Optional experiment identifier
@@ -81,7 +84,7 @@ def evaluate(
     validate_parameters(locals(), evaluate)
 
     # Prepare datasets, hyperparameters, and eval run specs
-    datasets = prepare_datasets(datasets, sample_size)
+    datasets = prepare_datasets(datasets, split, sample_size)
     hyperparameter_configs = prepare_hyperparameter_configs(hyperparameters)
     eval_run_specs = sorted(
         build_eval_run_specs(datasets, hyperparameter_configs, experiment_id, project_id, metadata),
@@ -377,8 +380,20 @@ def run_adaptive_evaluation(
     Returns:
         Results from the adaptive evaluation
     """
+    # Fetch available splits from Trismik
+    dataset_info = trismik_client.get_dataset_info(adaptive_run_spec.dataset)
+    available_splits = dataset_info.splits if hasattr(dataset_info, "splits") else []
+
+    # Resolve the split to use (with fallback: user-specified -> validation -> test)
+    resolved_split = resolve_adaptive_split(
+        test_id=adaptive_run_spec.dataset,
+        user_specified_split=adaptive_run_spec.split,
+        available_splits=available_splits,
+    )
+
     trismik_results = trismik_client.run(
         test_id=adaptive_run_spec.dataset,
+        split=resolved_split,
         project_id=project_id,
         experiment=experiment_id,
         run_metadata=TrismikRunMetadata(

scorebook/evaluate/evaluate_helpers.py

@@ -91,6 +91,7 @@ def validate_parameters(params: Dict[str, Any], caller: Callable[..., Any]) -> N
 
 def prepare_datasets(
     datasets: Union[str, EvalDataset, List[Union[str, EvalDataset]]],
+    split: Optional[str] = None,
     sample_size: Optional[int] = None,
 ) -> List[Union[EvalDataset, AdaptiveEvalDataset]]:
     """Prepare and separate input datasets into classic and adaptive evaluation datasets."""
@@ -104,6 +105,12 @@ def prepare_datasets(
 
         # Prepare classic datasets
         if isinstance(dataset, EvalDataset):
+            # Warn if dataset split differs from provided split parameter
+            if split is not None and dataset.split is not None and dataset.split != split:
+                logger.warning(
+                    f"Dataset '{dataset.name}' has split '{dataset.split}' but evaluate split "
+                    f"parameter is '{split}'. The dataset split will be used."
+                )
 
             if sample_size is not None:
                 dataset = dataset.sample(sample_size)
@@ -111,8 +118,17 @@ def prepare_datasets(
             datasets_out.append(dataset)
 
         # Prepare adaptive datasets
-        elif isinstance(dataset, str) and dataset.endswith(":adaptive"):
-            datasets_out.append(AdaptiveEvalDataset(dataset))
+        elif isinstance(dataset, str) and ":adaptive" in dataset:
+            # Parse adaptive dataset
+            parts = dataset.split(":")
+            if len(parts) != 2 or parts[1] != "adaptive":
+                raise ParameterValidationError(
+                    f"Invalid adaptive dataset format: '{dataset}'. "
+                    f"Use 'test_id:adaptive' format and specify split via the split parameter."
+                )
+
+            # Use the split parameter for all adaptive datasets
+            datasets_out.append(AdaptiveEvalDataset(name=dataset, split=split))
 
         # TODO: dataset name string registry
         elif isinstance(dataset, str):
@@ -174,6 +190,7 @@ def build_eval_run_specs(
                         hyperparameters_index,
                         experiment_id,
                         project_id,
+                        dataset.split,
                         metadata,
                     )
                 )
@@ -220,6 +237,7 @@ def build_adaptive_eval_run_spec(
     hyperparameter_config_index: int,
     experiment_id: str,
     project_id: str,
+    split: Optional[str] = None,
     metadata: Optional[Dict[str, Any]] = None,
 ) -> AdaptiveEvalRunSpec:
     """Build AdaptiveEvalRunSpec objects for a dataset/hyperparameter combination."""
@@ -231,6 +249,7 @@ def build_adaptive_eval_run_spec(
         hyperparameter_config_index,
         experiment_id,
         project_id,
+        split,
         metadata,
     )
     logger.debug("Built AdaptiveEvalRunSpec: %s", adaptive_eval_run_spec)
@@ -386,3 +405,57 @@ def make_trismik_inference(
         )
 
     return sync_trismik_inference_function
+
+
+def resolve_adaptive_split(
+    test_id: str,
+    user_specified_split: Optional[str],
+    available_splits: List[str],
+) -> str:
+    """Resolve the dataset split to use for adaptive evaluation.
+
+    Resolution order:
+    1. If user specified a split, validate it exists and use it
+    2. If not specified and exactly one split is available, use it
+    3. If not specified and multiple splits are available, raise an error
+    4. If no splits are available, raise an error
+
+    Args:
+        test_id: The test dataset ID (without ":adaptive" suffix)
+        user_specified_split: Optional split name specified by the user
+        available_splits: List of available split names for this dataset
+
+    Returns:
+        The resolved split name to use
+
+    Raises:
+        ScoreBookError: If the specified split doesn't exist, multiple splits exist without
+            user specification, or no splits are available
+    """
+    logger.debug(f"Available splits for {test_id}: {available_splits}")
+
+    # If user specified a split, validate and use it
+    if user_specified_split is not None:
+        if user_specified_split in available_splits:
+            logger.info(f"Using user-specified split '{user_specified_split}' for {test_id}")
+            return user_specified_split
+        else:
+            raise ScoreBookError(
+                f"Specified split '{user_specified_split}' not found for dataset '{test_id}'. "
+                f"Available splits: {available_splits}"
+            )
+
+    # No split specified - check available splits
+    if len(available_splits) == 0:
+        raise ScoreBookError(f"No splits available for dataset '{test_id}'. ")
+    elif len(available_splits) == 1:
+        # Exactly one split - auto-select it
+        selected_split = available_splits[0]
+        logger.info(f"Auto-selecting only available split '{selected_split}' for {test_id}")
+        return selected_split
+    else:
+        # Multiple splits available - user must specify
+        raise ScoreBookError(
+            f"Multiple splits available for dataset '{test_id}': {available_splits}. "
+            f"Please specify which split to use via evaluate's 'split' parameter."
+        )

scorebook/types.py

@@ -17,6 +17,7 @@ class AdaptiveEvalDataset:
     """Represents a dataset configured for adaptive evaluation."""
 
     name: str
+    split: Optional[str] = None
 
 
 @dataclass
@@ -50,6 +51,7 @@ class AdaptiveEvalRunSpec:
     hyperparameters_index: int
     experiment_id: str
     project_id: str
+    split: Optional[str] = None
     metadata: Optional[Dict[str, Any]] = None
 
 

{scorebook-0.0.12.dist-info → scorebook-0.0.13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: scorebook
-Version: 0.0.12
+Version: 0.0.13
 Summary: A Python project for LLM evaluation.
 License-File: LICENSE
 Author: Euan Campbell
@@ -37,7 +37,7 @@ Requires-Dist: torch ; extra == "examples"
 Requires-Dist: torchaudio ; extra == "examples"
 Requires-Dist: torchvision ; extra == "examples"
 Requires-Dist: transformers ; extra == "examples"
-Requires-Dist: trismik (==1.0.1)
+Requires-Dist: trismik (==1.0.2)
 Description-Content-Type: text/markdown
 
 # Scorebook

{scorebook-0.0.12.dist-info → scorebook-0.0.13.dist-info}/RECORD

@@ -3,13 +3,13 @@ scorebook/cli/__init__.py,sha256=E89jR1DljFSHhfjEGSRKLgz0KhxGyRQ9a3vpUOmQL9o,32
 scorebook/cli/auth.py,sha256=T6-5662Jh-HEhZvfUgy82BvxIiRzjZne-4LRp9Gb2JE,2937
 scorebook/cli/main.py,sha256=cEvShENl6L6feX_sa7FGNTeoz5UtwqzwenmcHaON1hg,1589
 scorebook/eval_datasets/__init__.py,sha256=9YPjxjdaMaOrBUzJwvsUlFPl-KdYMgUGTV3WNd7OCU0,128
-scorebook/eval_datasets/eval_dataset.py,sha256=6GgrAaWelU5dK6I-x9zXHCxVSfvo41yyYNPF0ue4zbo,27200
+scorebook/eval_datasets/eval_dataset.py,sha256=rgNLPajIe6RqEq_qEeV6UExT1ZRx4dFX6XA_qnpbRIM,27930
 scorebook/evaluate/__init__.py,sha256=m3mCjeLildghT86ZDwY4GxCmaYZmhjbxkuTk0M9S_mc,423
 scorebook/evaluate/_async/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-scorebook/evaluate/_async/evaluate_async.py,sha256=qZ2y7-uQRT1b4saBoNaPO9fv4G2LhcP_ZyvkSsIEgHg,15629
+scorebook/evaluate/_async/evaluate_async.py,sha256=QAvAA6_upg_vK2M8BV_DVegQ7hxZCO_PaOaM3yj3e6Y,16284
 scorebook/evaluate/_sync/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-scorebook/evaluate/_sync/evaluate.py,sha256=xanhHu-CaY_WarNM1V64W2sHttkM18j42K0MKrdtrvE,15438
-scorebook/evaluate/evaluate_helpers.py,sha256=swbgB1LurWdufeiVIZZ7ildsYO-ptC7uF3x6AVgptkU,13809
+scorebook/evaluate/_sync/evaluate.py,sha256=MQ53VMPLlVBcZm2mOUs3zXeM3hvoWz3fLj_BNvrOLvU,16087
+scorebook/evaluate/evaluate_helpers.py,sha256=j-oTKAfVhyH-kxdM7YjuoolS_SwBb2WbjWhe5jeZiwU,16942
 scorebook/exceptions.py,sha256=3sxCWhFqYgXiWNUAMRR2ggLfqvbDI8e5vLjnT9V7X1M,3649
 scorebook/inference/__init__.py,sha256=u3TmfftO0oMkz8ngwxAKLPfL1so1w2hbK7c5UNlRq-M,345
 scorebook/inference/clients/__init__.py,sha256=QCjbrXYeFd7xK-5ZH7o7bSaKUJqHtGnH5285pezNKyY,242
@@ -33,7 +33,7 @@ scorebook/settings.py,sha256=qZrNiki6rFXn43udmhjQSmdDKOEaX62WYoEs2Rbggr0,720
 scorebook/trismik/__init__.py,sha256=kWZkEC57LJscRZNLE3sJR1L5w-ltb5mEbQd3_ePtZPQ,380
 scorebook/trismik/credentials.py,sha256=WtJLaNmBMwCi6gT1Bgp4J9x2tq5HDrDI9U074r08TnU,3275
 scorebook/trismik/upload_results.py,sha256=jgT9EVFpuv6OmrYgZVi032cbRrcCOyX4ulLDeWPFBWU,9743
-scorebook/types.py,sha256=x5bD2DU-Xafh7pXwmaQQ1i1zoZDsniHJjE-UEfXySAg,4827
+scorebook/types.py,sha256=f6X3lNtpIB6jC12WeLugtSpOgY4KlQgUlhxRG_d2GGE,4891
 scorebook/utils/__init__.py,sha256=crefSaTUWkhFF-w4kotUzcz9_GGZukQDgRit4HxJRHY,805
 scorebook/utils/async_utils.py,sha256=2ewk_VOePib8z7DTRl-pZQBGzVI3L3JvnEuYW-DTkRA,1325
 scorebook/utils/common_helpers.py,sha256=jewPdQH4JqTWcYT31wn1WNucOPLtGbrGdViwwlYRhD4,1216
@@ -43,8 +43,8 @@ scorebook/utils/mappers.py,sha256=OcUnPBrnSUxZNhAzJhVmVWUWmqIKFXLTrK-xLi6_SUg,12
 scorebook/utils/progress_bars.py,sha256=gdT6dJ9LMLYzs7TospP3wQNY9htm_FhVLdX0ueluC6E,31890
 scorebook/utils/render_template.py,sha256=NOaZt-N1WcR5MA7at1XxzD-4sFMFKo9X0k7fKq6oSSM,1654
 scorebook/utils/transform_helpers.py,sha256=UnVLtFvcJrtmBEmLsuA4rrX4iJlNUKxm2DkIOGLl-2o,1030
-scorebook-0.0.12.dist-info/METADATA,sha256=bMjbT1e0GYExB1HcBkAfesaUcXK2-Pck5ox2oCUBXpE,11508
-scorebook-0.0.12.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
-scorebook-0.0.12.dist-info/entry_points.txt,sha256=9gNd3Q0MEozhJ7fog-Q-Z_PrcGMnF-404Jon40MH2_U,53
-scorebook-0.0.12.dist-info/licenses/LICENSE,sha256=JLH1g9FhxHZf6CBCeQ_xAisPtICVObuNGW1bLPiTYEs,1068
-scorebook-0.0.12.dist-info/RECORD,,
+scorebook-0.0.13.dist-info/METADATA,sha256=Z4ZyN6upriTbl-8RciJGFufcwbuY_eNO3O7aSeIAr6U,11508
+scorebook-0.0.13.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+scorebook-0.0.13.dist-info/entry_points.txt,sha256=9gNd3Q0MEozhJ7fog-Q-Z_PrcGMnF-404Jon40MH2_U,53
+scorebook-0.0.13.dist-info/licenses/LICENSE,sha256=JLH1g9FhxHZf6CBCeQ_xAisPtICVObuNGW1bLPiTYEs,1068
+scorebook-0.0.13.dist-info/RECORD,,