scorebook 0.0.9__py3-none-any.whl → 0.0.11__py3-none-any.whl

scorebook/utils/{build_prompt.py → render_template.py}

@@ -7,36 +7,37 @@ and global variables, using strict undefined handling for better error detection
 
 from typing import Any, Dict, Optional
 
-from jinja2 import BaseLoader, Environment, StrictUndefined
+from jinja2 import BaseLoader, StrictUndefined
+from jinja2.sandbox import SandboxedEnvironment
 
 from scorebook.utils.jinja_helpers import default_jinja_filters, default_jinja_globals
 
 
-def build_prompt(
-    prompt_template: str,
-    prompt_args: Dict[str, Any],
+def render_template(
+    template: str,
+    args: Dict[str, Any],
     filters: Optional[Dict[str, Any]] = None,
     globals_dict: Optional[Dict[str, Any]] = None,
 ) -> str:
     """
-    Build a prompt string from a template and arguments.
+    Render a Jinja2 template string with the provided arguments.
 
     Args:
-        prompt_template: Jinja2 template string
-        prompt_args: Dictionary of arguments to pass to the template
+        template: Jinja2 template string
+        args: Dictionary of arguments to pass to the template
         filters: Dictionary of Jinja2 filters. Defaults to default_jinja_filters().
         globals_dict: Dictionary of global functions/variables. Defaults to default_jinja_globals().
 
     Returns:
-        str: Rendered prompt string
+        str: Rendered template string
     """
 
     # Use defaults if not provided
     filters = filters or default_jinja_filters()
     globals_dict = globals_dict or default_jinja_globals()
 
-    # Create a Jinja2 environment with strict undefined handling
-    env = Environment(
+    # Create a sandboxed Jinja2 environment with strict undefined handling
+    env = SandboxedEnvironment(
         loader=BaseLoader(),
         undefined=StrictUndefined,
         trim_blocks=True,
@@ -48,5 +49,5 @@ def build_prompt(
     env.globals.update(globals_dict)
 
     # Render the template
-    template = env.from_string(prompt_template)
-    return str(template.render(**prompt_args))
+    jinja_template = env.from_string(template)
+    return str(jinja_template.render(**args))

{scorebook-0.0.9.dist-info → scorebook-0.0.11.dist-info}/METADATA

@@ -1,18 +1,17 @@
 Metadata-Version: 2.4
 Name: scorebook
-Version: 0.0.9
+Version: 0.0.11
 Summary: A Python project for LLM evaluation.
 License-File: LICENSE
 Author: Euan Campbell
 Author-email: euan@trismik.com
-Requires-Python: >=3.9
+Requires-Python: >=3.9, <3.14
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
-Classifier: Programming Language :: Python :: 3.14
 Provides-Extra: bedrock
 Provides-Extra: examples
 Provides-Extra: openai
@@ -24,6 +23,7 @@ Requires-Dist: datasets (>=3.6.0)
 Requires-Dist: fsspec[gcs] ; extra == "vertex"
 Requires-Dist: google-cloud-storage ; extra == "vertex"
 Requires-Dist: google-genai ; extra == "vertex"
+Requires-Dist: ipywidgets (>=8.0.0)
 Requires-Dist: notebook (>=7.4.5,<8.0.0)
 Requires-Dist: notebook ; extra == "examples"
 Requires-Dist: openai ; extra == "openai"
@@ -37,7 +37,7 @@ Requires-Dist: torch ; extra == "examples"
 Requires-Dist: torchaudio ; extra == "examples"
 Requires-Dist: torchvision ; extra == "examples"
 Requires-Dist: transformers ; extra == "examples"
-Requires-Dist: trismik (>=0.9.12)
+Requires-Dist: trismik (>=1.0.1,<2.0.0)
 Description-Content-Type: text/markdown
 
 # Scorebook

scorebook-0.0.11.dist-info/RECORD

@@ -0,0 +1,42 @@
+scorebook/__init__.py,sha256=uifumovUYn62n_VRFTeieEP3eVP8ycZ3lDp44sVwgeI,639
+scorebook/cli/__init__.py,sha256=E89jR1DljFSHhfjEGSRKLgz0KhxGyRQ9a3vpUOmQL9o,32
+scorebook/cli/auth.py,sha256=T6-5662Jh-HEhZvfUgy82BvxIiRzjZne-4LRp9Gb2JE,2937
+scorebook/cli/main.py,sha256=cEvShENl6L6feX_sa7FGNTeoz5UtwqzwenmcHaON1hg,1589
+scorebook/eval_datasets/__init__.py,sha256=9YPjxjdaMaOrBUzJwvsUlFPl-KdYMgUGTV3WNd7OCU0,128
+scorebook/eval_datasets/eval_dataset.py,sha256=6GgrAaWelU5dK6I-x9zXHCxVSfvo41yyYNPF0ue4zbo,27200
+scorebook/evaluate/__init__.py,sha256=m3mCjeLildghT86ZDwY4GxCmaYZmhjbxkuTk0M9S_mc,423
+scorebook/evaluate/_async/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scorebook/evaluate/_async/evaluate_async.py,sha256=kcUrxfXdleqjn-Ajn06jGAfMvftmwQLzbB2zLu2-Q3U,16983
+scorebook/evaluate/_sync/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+scorebook/evaluate/_sync/evaluate.py,sha256=qBOQZQYfpHSz4i_uRgH7QdZvhw0isOmIYPdqcLIWvwQ,16821
+scorebook/evaluate/evaluate_helpers.py,sha256=sSWWdyH5h1jJsQ_ABzRzgW0-Dy21BQ2PzdCJ0-uROC8,13829
+scorebook/exceptions.py,sha256=i494gw1TnSV-vS3RWGGZ6FYdlldjcY1qQuXO1htukuQ,3551
+scorebook/inference/__init__.py,sha256=u3TmfftO0oMkz8ngwxAKLPfL1so1w2hbK7c5UNlRq-M,345
+scorebook/inference/clients/__init__.py,sha256=QCjbrXYeFd7xK-5ZH7o7bSaKUJqHtGnH5285pezNKyY,242
+scorebook/inference/clients/bedrock.py,sha256=bsnz0IB6ufjZVPd6syD3yVaOelerB5G_YAmPAVqmBmI,10071
+scorebook/inference/clients/openai.py,sha256=JPPcJdxYwwZNUfXCGTRRzzUUA8I8WiV3bu6-pgS1_UE,9043
+scorebook/inference/clients/portkey.py,sha256=RCuEZB8xNAVeGEt3IJ0esh_wqreZNB3jrDKiWH6miV0,5949
+scorebook/inference/clients/vertex.py,sha256=g6oNnag0qcLOYCtQ4SXAXfnqKtvPAVdigB--I7wU1yM,9871
+scorebook/inference/inference_pipeline.py,sha256=SOr1xnglPvFMcJFSpDRLQ6222NJgy_-fVtZLC423TUE,5559
+scorebook/metrics/__init__.py,sha256=be_riJNojebXw2xfkMsHHjl3HFKgk9jQWlLkXJHhheI,782
+scorebook/metrics/accuracy.py,sha256=5KQ4hfOn9M94sB7WsXUelJWJiuKfoCGQEl5q5q9vNfo,1467
+scorebook/metrics/metric_base.py,sha256=I3L0DGcRojFp93UGFnXG1tZ2UK9ilTcXXJG6lj5ddXA,857
+scorebook/metrics/metric_registry.py,sha256=jWwt9P3zvtFLlEYrd60v7LS7X251nZczouE02zcCxWg,3402
+scorebook/metrics/precision.py,sha256=AaYPYYKnY74Nwqp_p3jd2Ewf3VHNOJjoRWf5fhb-tXk,563
+scorebook/settings.py,sha256=qZrNiki6rFXn43udmhjQSmdDKOEaX62WYoEs2Rbggr0,720
+scorebook/trismik/__init__.py,sha256=kWZkEC57LJscRZNLE3sJR1L5w-ltb5mEbQd3_ePtZPQ,380
+scorebook/trismik/credentials.py,sha256=WtJLaNmBMwCi6gT1Bgp4J9x2tq5HDrDI9U074r08TnU,3275
+scorebook/types.py,sha256=8weXPYzUw9m4fy2O_TleqThg1OI4z_lQyx78brJigJU,5955
+scorebook/utils/__init__.py,sha256=IhL8bss1s-QkguwYp7T3jamk5GmsJRFjf9MynAN1W5g,544
+scorebook/utils/async_utils.py,sha256=2ewk_VOePib8z7DTRl-pZQBGzVI3L3JvnEuYW-DTkRA,1325
+scorebook/utils/io_helpers.py,sha256=ORO6DwtXOKWJq9v_isuunUrz0viE3xy2qYO4lrgU-TM,1437
+scorebook/utils/jinja_helpers.py,sha256=ksIKHiKdj8N0o7ZJZGasfbSNoAY6K5d9X_KM6mcKYD4,4208
+scorebook/utils/mappers.py,sha256=OcUnPBrnSUxZNhAzJhVmVWUWmqIKFXLTrK-xLi6_SUg,1259
+scorebook/utils/progress_bars.py,sha256=bkKZBSUt5kJi_ZyRhEE0UWrgB2NZpua7jxZ8xqsNyuk,29792
+scorebook/utils/render_template.py,sha256=NOaZt-N1WcR5MA7at1XxzD-4sFMFKo9X0k7fKq6oSSM,1654
+scorebook/utils/transform_helpers.py,sha256=UnVLtFvcJrtmBEmLsuA4rrX4iJlNUKxm2DkIOGLl-2o,1030
+scorebook-0.0.11.dist-info/METADATA,sha256=FTV_4abv4NK1z9KbKx4sL8Qto74pMcHhCKZnskGWolo,11515
+scorebook-0.0.11.dist-info/WHEEL,sha256=zp0Cn7JsFoX2ATtOhtaFYIiE2rmFAD4OcMhtUki8W3U,88
+scorebook-0.0.11.dist-info/entry_points.txt,sha256=9gNd3Q0MEozhJ7fog-Q-Z_PrcGMnF-404Jon40MH2_U,53
+scorebook-0.0.11.dist-info/licenses/LICENSE,sha256=JLH1g9FhxHZf6CBCeQ_xAisPtICVObuNGW1bLPiTYEs,1068
+scorebook-0.0.11.dist-info/RECORD,,

scorebook/eval_dataset.py

@@ -1,404 +0,0 @@
-"""Eval Dataset implementation for scorebook."""
-
-import csv
-import json
-import random
-from typing import Any, Dict, Iterator, List, Optional, Type, Union
-
-import yaml
-from datasets import Dataset as HuggingFaceDataset
-from datasets import DatasetDict as HuggingFaceDatasetDict
-from datasets import load_dataset
-
-from scorebook.metrics import MetricBase, MetricRegistry
-from scorebook.utils import validate_path
-
-
-class EvalDataset:
-    """Eval Dataset implementation for scorebook."""
-
-    def __init__(
-        self,
-        name: str,
-        label: str,
-        metrics: Union[str, Type[MetricBase], List[Union[str, Type[MetricBase]]]],
-        hf_dataset: HuggingFaceDataset,
-        prompt_template: Optional[str] = None,
-    ):
-        """
-        Create a new scorebook evaluation dataset instance.
-
-        :param name: The name of the evaluation dataset.
-        :param label: The label field of the dataset.
-        :param metrics: The specified metrics associated with the dataset.
-        :param hf_dataset: The dataset as a hugging face dataset object.
-        :param prompt_template: Optional prompt template for building prompts from dataset items.
-        """
-        self.name: str = name
-        self.label: str = label
-        self.metrics: List[MetricBase] = self._resolve_metrics(metrics)
-        self._hf_dataset: Optional[HuggingFaceDataset] = hf_dataset
-        self.prompt_template: Optional[str] = prompt_template
-
-    def __len__(self) -> int:
-        """Return the number of items in the dataset."""
-        if self._hf_dataset is None:
-            raise ValueError("Dataset is not initialized")
-        return len(self._hf_dataset)
-
-    def __getitem__(self, key: Union[int, str]) -> Union[Dict[str, Any], List[Any]]:
-        """
-        Allow item access by index (int) or by column name (str).
-
-        - eval_dataset[i] returns the i-th example (dict).
-        - eval_dataset["feature"] returns a list of values for that feature.
-        """
-        if self._hf_dataset is None:
-            raise ValueError("Dataset is not initialized")
-        if isinstance(key, int):
-            return dict(self._hf_dataset[key])  # Ensure we return a Dict[str, Any]
-        elif isinstance(key, str):
-            return list(self._hf_dataset[key])  # Ensure we return a List[Any]
-        else:
-            raise TypeError(f"Invalid key type: {type(key)}. Must be int or str.")
-
-    def __str__(self) -> str:
-        """Return a formatted string summary of the evaluation dataset."""
-        if self._hf_dataset is None:
-            return f"EvalDataset(name='{self.name}', status='uninitialized')"
-
-        num_rows = len(self._hf_dataset)
-        fields = ", ".join(self.column_names)
-        metrics = ", ".join([metric.name for metric in self.metrics])
-
-        return (
-            f"EvalDataset(\n"
-            f"  name='{self.name}',\n"
-            f"  rows={num_rows},\n"
-            f"  label='{self.label}',\n"
-            f"  fields=[{fields}],\n"
-            f"  metrics=[{metrics}]\n"
-            f")"
-        )
-
-    def __iter__(self) -> Iterator[Dict[str, Any]]:
-        """Return an iterator over all examples in the dataset."""
-        if self._hf_dataset is None:
-            raise ValueError("Dataset is not initialized")
-        return iter(self._hf_dataset)
-
-    def shuffle(self) -> None:
-        """Randomly shuffle the dataset items."""
-        if self._hf_dataset is None:
-            raise ValueError("Dataset is not initialized")
-        self._hf_dataset.shuffle()
-
-    @property
-    def items(self) -> List[Any]:
-        """Return a list of all examples in the dataset."""
-        if self._hf_dataset is None:
-            raise ValueError("Dataset is not initialized")
-        return list(self._hf_dataset)
-
-    @property
-    def column_names(self) -> List[str]:
-        """Return a list of column/feature names available in the dataset."""
-        if self._hf_dataset is None:
-            raise ValueError("Dataset is not initialized")
-        return list(map(str, self._hf_dataset.column_names))
-
-    @classmethod
-    def from_list(
-        cls,
-        name: str,
-        label: str,
-        metrics: Union[str, Type[MetricBase], List[Union[str, Type[MetricBase]]]],
-        data: List[Dict[str, Any]],
-    ) -> "EvalDataset":
-        """Instantiate an EvalDataset from a list of dictionaries.
-
-        Args:
-            name: The name of the evaluation dataset.
-            label: The field used as the evaluation label (ground truth).
-            metrics: The specified metrics associated with the dataset.
-            data: List of dictionaries containing the dataset examples.
-
-        Returns:
-            A scorebook EvalDataset wrapping a Hugging Face dataset.
-        """
-        return cls(
-            name=name, label=label, metrics=metrics, hf_dataset=HuggingFaceDataset.from_list(data)
-        )
-
-    @classmethod
-    def from_csv(
-        cls,
-        file_path: str,
-        label: str,
-        metrics: Union[str, Type[MetricBase], List[Union[str, Type[MetricBase]]]],
-        name: Optional[str] = None,
-        encoding: str = "utf-8",
-        newline: str = "",
-        **reader_kwargs: Any,
-    ) -> "EvalDataset":
-        """Instantiate a scorebook dataset from a CSV file.
-
-        Args:
-            file_path: Path to the CSV file.
-            label: The field used as the evaluation label (ground truth).
-            metrics: The specified metrics associated with the dataset.
-            name: Optional name for the eval dataset, if not provided, the path is used
-            encoding: Encoding of the CSV file.
-            newline: Newline character of the CSV file.
-            reader_kwargs: Dict of kwargs passed to `csv.DictReader`.
-
-        Returns:
-            A scorebook EvalDataset.
-
-        Raises:
-            FileNotFoundError: If the file does not exist at the given path.
-            ValueError: If the CSV file cannot be parsed or is empty.
-        """
-        reader_kwargs = reader_kwargs or {}
-        path = validate_path(file_path, expected_suffix=".csv")
-
-        try:
-            with open(path, encoding=encoding, newline=newline) as csvfile:
-                reader = csv.DictReader(csvfile, **reader_kwargs)
-                data = [row for row in reader]
-        except csv.Error as e:
-            raise ValueError(f"Failed to parse CSV file {file_path}: {e}") from e
-
-        if not data:
-            raise ValueError(f"CSV file {file_path} is empty or contains only headers.")
-
-        name = name if name else path.stem
-        return cls(
-            name=name,
-            label=label,
-            metrics=metrics,
-            hf_dataset=HuggingFaceDataset.from_list(data),
-        )
-
-    @classmethod
-    def from_json(
-        cls,
-        file_path: str,
-        label: str,
-        metrics: Union[str, Type[MetricBase], List[Union[str, Type[MetricBase]]]],
-        name: Optional[str] = None,
-        split: Optional[str] = None,
-    ) -> "EvalDataset":
-        """Instantiate an EvalDataset from a JSON file.
-
-        The JSON file must follow one of two supported formats:
-
-        1. **Flat format** – a list of dictionaries:
-            [
-                {"input": "What is 2+2?", "label": "4"},
-                {"input": "Capital of France?", "label": "Paris"}
-            ]
-
-        2. **Split format** – a dictionary of named splits:
-            {
-                "train": [{"input": ..., "label": ...}],
-                "test": [{"input": ..., "label": ...}]
-            }
-
-        Args:
-            file_path: Path to the JSON file on disk.
-            label: The field used as the evaluation label (ground truth).
-            metrics: The specified metrics associated with the dataset.
-            name: Optional name for the eval dataset, if not provided, the path is used
-            split: If the JSON uses a split structure, this is the split name to load.
-
-        Returns:
-            A scorebook EvalDataset wrapping a Hugging Face dataset.
-
-        Raises:
-            FileNotFoundError: If the file does not exist.
-            ValueError: If the JSON is invalid or the structure is unsupported.
-        """
-        path = validate_path(file_path, expected_suffix=".json")
-
-        try:
-            with path.open("r", encoding="utf-8") as f:
-                data = json.load(f)
-        except json.JSONDecodeError as e:
-            raise ValueError(f"Invalid JSON in {file_path}: {e}") from e
-
-        if isinstance(data, dict):
-            if split is None:
-                raise ValueError(f"Split name must be provided for split-style JSON: {file_path}")
-            split_data = data.get(split)
-            if split_data is None:
-                raise ValueError(f"Split '{split}' not found in JSON file: {file_path}")
-            if not isinstance(split_data, list):
-                raise ValueError(f"Split '{split}' is not a list of examples in: {file_path}")
-            hf_dataset = HuggingFaceDataset.from_list(split_data)
-        elif isinstance(data, list):
-            hf_dataset = HuggingFaceDataset.from_list(data)
-        else:
-            raise ValueError(f"Unsupported JSON structure in {file_path}. Expected list or dict.")
-
-        name = name if name else path.stem
-        return cls(name=name, label=label, metrics=metrics, hf_dataset=hf_dataset)
-
-    @classmethod
-    def from_huggingface(
-        cls,
-        path: str,
-        label: str,
-        metrics: Union[str, Type[MetricBase], List[Union[str, Type[MetricBase]]]],
-        split: Optional[str] = None,
-        name: Optional[str] = None,
-    ) -> "EvalDataset":
-        """Instantiate an EvalDataset from a dataset available on Hugging Face Hub.
-
-        If a specific split is provided (e.g., "train" or "test"), it will be loaded directly.
-        If no split is specified, the method attempts to load the full dataset. If the dataset
-        is split into multiple subsets (i.e., a DatasetDict), it defaults to loading the "test"
-        split.
-
-        Args:
-            path: The path of the dataset on the Hugging Face Hub.
-            label: The field used as the evaluation label (ground truth).
-            metrics: The specified metrics associated with the dataset.
-            split: Optional name of the split to load.
-            name: Optional dataset configuration name.
-
-        Returns:
-            An EvalDataset wrapping the selected Hugging Face dataset.
-
-        Raises:
-            ValueError: If the dataset cannot be loaded, or the expected split is missing.
-        """
-        try:
-            kwargs = {}
-            if split is not None:
-                kwargs["split"] = split
-            if name is not None:
-                kwargs["name"] = name
-            ds = load_dataset(path, **kwargs)
-        except Exception as e:
-            raise ValueError(f"Failed to load dataset '{path}' from Hugging Face: {e}") from e
-
-        if isinstance(ds, HuggingFaceDataset):
-            hf_dataset = ds
-        elif isinstance(ds, HuggingFaceDatasetDict):
-            if "test" in ds:
-                hf_dataset = ds["test"]
-            else:
-                raise ValueError(
-                    f"Split not specified and no 'test' split found in dataset '{path}'."
-                )
-        else:
-            raise ValueError(f"Unexpected dataset type for '{path}': {type(ds)}")
-
-        return cls(name=path, label=label, metrics=metrics, hf_dataset=hf_dataset)
-
-    @classmethod
-    def from_yaml(cls, file_path: str) -> "EvalDataset":
-        """Instantiate an EvalDataset from a YAML file.
-
-        The YAML file should contain configuration for loading a dataset, including:
-        - name: Name of the dataset or Hugging Face dataset path
-        - label: The field used as the evaluation label
-        - metrics: List of metrics to evaluate
-        - split: Optional split name to load
-        - template: Optional prompt template
-
-        Returns:
-            An EvalDataset instance configured according to the YAML file.
-
-        Raises:
-            ValueError: If the YAML file is invalid or missing required fields.
-        """
-        path = validate_path(file_path, expected_suffix=".yaml")
-
-        try:
-            with path.open("r", encoding="utf-8") as f:
-                config = yaml.safe_load(f)
-        except yaml.YAMLError as e:
-            raise ValueError(f"Invalid YAML in {file_path}: {e}") from e
-
-        # Validate required fields
-        required_fields = ["name", "label", "metrics"]
-        missing_fields = [field for field in required_fields if field not in config]
-        if missing_fields:
-            raise ValueError(f"Missing required fields in YAML config: {', '.join(missing_fields)}")
-
-        # Load the dataset from Hugging Face
-        dataset = cls.from_huggingface(
-            path=config["name"],
-            label=config["label"],
-            metrics=config["metrics"],
-            split=config.get("split"),  # Optional field
-            name=config.get("config"),  # Optional field
-        )
-
-        # Add template if provided
-        if "template" in config:
-            dataset.prompt_template = config["template"]
-
-        return dataset
-
-    @staticmethod
-    def _resolve_metrics(
-        metrics: Union[
-            str, Type[MetricBase], MetricBase, List[Union[str, Type[MetricBase], MetricBase]]
-        ]
-    ) -> List[MetricBase]:
-        """
-        Convert metric names/classes into a list of MetricBase instances using MetricRegistry.
-
-        Used to normalize metrics to a metric type.
-        """
-        if not isinstance(metrics, list):
-            metrics = [metrics]
-
-        resolved: List[MetricBase] = []
-        for m in metrics:
-            if isinstance(m, MetricBase):
-                resolved.append(m)  # Already an instance
-            else:
-                resolved.append(MetricRegistry.get(m))  # Use registry for str or class
-
-        return resolved
-
-    def sample(self, sample_size: int) -> "EvalDataset":
-        """Create a new dataset with randomly sampled items from this dataset.
-
-        Args:
-            sample_size: The number of items to sample from the dataset
-
-        Returns:
-            A new EvalDataset with randomly sampled items
-
-        Raises:
-            ValueError: If sample_size is larger than the dataset size
-        """
-        dataset_size = len(self.items)
-
-        if sample_size > dataset_size:
-            raise ValueError(
-                f"Sample size {sample_size} is larger than dataset size {dataset_size} "
-                f"for dataset '{self.name}'"
-            )
-
-        # Create randomly sampled items
-        sampled_items = random.sample(self.items, sample_size)
-
-        # Create a new EvalDataset instance with sampled items using from_list
-        sampled_dataset = self.from_list(
-            name=self.name,
-            label=self.label,
-            metrics=self.metrics,
-            data=sampled_items,
-        )
-
-        # Preserve the prompt template if it exists
-        if self.prompt_template is not None:
-            sampled_dataset.prompt_template = self.prompt_template
-
-        return sampled_dataset