ScandEval 16.11.0__py3-none-any.whl → 16.13.0__py3-none-any.whl

scandeval/string_utils.py

@@ -0,0 +1,157 @@
+"""Utility functions related to string manipulation or structuring."""
+
+import collections.abc as c
+import logging
+import re
+import typing as t
+
+import demjson3
+import numpy as np
+
+from .exceptions import InvalidBenchmark, InvalidModel
+from .logging_utils import log
+
+if t.TYPE_CHECKING:
+    from .data_models import ModelIdComponents
+
+
+def scramble(text: str) -> str:
+    """Scramble a string in a bijective manner.
+
+    Args:
+        text:
+            The string to scramble.
+
+    Returns:
+        The scrambled string.
+    """
+    rng = np.random.default_rng(seed=4242)
+    permutation = rng.permutation(x=len(text))
+    scrambled = "".join(text[i] for i in permutation)
+    return scrambled
+
+
+def unscramble(scrambled_text: str) -> str:
+    """Unscramble a string in a bijective manner.
+
+    Args:
+        scrambled_text:
+            The scrambled string to unscramble.
+
+    Returns:
+        The unscrambled string.
+    """
+    rng = np.random.default_rng(seed=4242)
+    permutation = rng.permutation(x=len(scrambled_text))
+    inverse_permutation = np.argsort(permutation)
+    unscrambled = "".join(scrambled_text[i] for i in inverse_permutation)
+    return unscrambled
+
+
+def extract_json_dict_from_string(s: str) -> dict | None:
+    """Extract a JSON dictionary from a string.
+
+    Args:
+        s:
+            The string to extract the JSON dictionary from.
+
+    Returns:
+        The extracted JSON dictionary, or None if no JSON dictionary could be found.
+    """
+    json_regex = r"\{[^{}]*?\}"
+    if (json_match := re.search(pattern=json_regex, string=s, flags=re.DOTALL)) is None:
+        log(
+            "The model output does not contain any JSON dictionary, so cannot parse "
+            f"it. Skipping. Here is the output: {s!r}",
+            level=logging.DEBUG,
+        )
+        return None
+    json_string = json_match.group()
+    try:
+        json_output = demjson3.decode(txt=json_string)
+    except demjson3.JSONDecodeError:
+        log(
+            "The model output is not valid JSON, so cannot parse it. Skipping. "
+            f"Here is the output: {json_string!r}",
+            level=logging.DEBUG,
+        )
+        return None
+    if not isinstance(json_output, dict):
+        log(
+            "The model output is not a JSON dictionary, so cannot parse "
+            f"it. Skipping. Here is the output: {json_string!r}",
+            level=logging.DEBUG,
+        )
+        return None
+    elif not all(isinstance(key, str) for key in json_output.keys()):
+        log(
+            "The model output is not a JSON dictionary with string keys, "
+            "so cannot parse it. Skipping. Here is the output: "
+            f"{json_string!r}",
+            level=logging.DEBUG,
+        )
+        return None
+    return json_output
+
+
+def extract_multiple_choice_labels(
+    prompt: str, candidate_labels: c.Sequence[str]
+) -> c.Sequence[str]:
+    """Extract multiple choice labels from a prompt.
+
+    Args:
+        prompt:
+            The prompt to extract the labels from.
+        candidate_labels:
+            The candidate labels to look for in the prompt.
+
+    Returns:
+        The extracted labels.
+    """
+    sample_candidate_labels: list[str] = list()
+    for candidate_label in candidate_labels:
+        candidate_label_match = re.search(
+            pattern=rf"\b{candidate_label}\. ", string=prompt, flags=re.IGNORECASE
+        )
+        if candidate_label_match is not None:
+            sample_candidate_labels.append(candidate_label)
+    if not sample_candidate_labels:
+        raise InvalidBenchmark(
+            "Could not extract any candidate labels from the prompt. Please ensure "
+            "that the candidate labels are present in the prompt, each followed by a "
+            "dot and a space (e.g., 'a. '). The candidate labels are: "
+            f"{', '.join(candidate_labels)}. Here is the prompt: {prompt!r}"
+        )
+    return sample_candidate_labels
+
+
+def split_model_id(model_id: str) -> "ModelIdComponents":
+    """Split a model ID into its components.
+
+    Args:
+        model_id:
+            The model ID to split.
+
+    Returns:
+        The split model ID.
+
+    Raises:
+        If the model ID is not valid.
+    """
+    # Importing here to avoid circular imports
+    from .data_models import ModelIdComponents
+
+    # Attempt to extract the model ID, revision, and param using regex
+    model_id_match = re.match(pattern=r"^[^@#]+", string=model_id)
+    revision_match = re.search(pattern=r"@([^@#]+)", string=model_id)
+    param_match = re.search(pattern=r"#([^@#]+)", string=model_id)
+
+    # If we cannot extract the model ID, raise an error
+    if model_id_match is None:
+        raise InvalidModel(f"The model ID {model_id!r} is not valid.")
+    model_id = model_id_match.group()
+
+    # Extract the revision and param and return the result
+    revision = revision_match.group(1) if revision_match is not None else "main"
+    param = param_match.group(1) if param_match is not None else None
+    return ModelIdComponents(model_id=model_id, revision=revision, param=param)

scandeval/task_group_utils/sequence_classification.py

@@ -10,12 +10,9 @@ import numpy as np
 
 from ..enums import TaskGroup
 from ..exceptions import InvalidBenchmark
+from ..string_utils import extract_multiple_choice_labels
 from ..types import Predictions
-from ..utils import (
-    extract_multiple_choice_labels,
-    log_once,
-    raise_if_model_output_contains_nan_values,
-)
+from ..utils import log_once, raise_if_model_output_contains_nan_values
 
 if t.TYPE_CHECKING:
     from datasets.arrow_dataset import Dataset

scandeval/task_group_utils/token_classification.py

@@ -9,10 +9,8 @@ import numpy as np
 
 from ..exceptions import InvalidBenchmark
 from ..logging_utils import log
-from ..utils import (
-    extract_json_dict_from_string,
-    raise_if_model_output_contains_nan_values,
-)
+from ..string_utils import extract_json_dict_from_string
+from ..utils import raise_if_model_output_contains_nan_values
 
 if t.TYPE_CHECKING:
     from datasets.arrow_dataset import Dataset

scandeval/tasks.py

@@ -153,6 +153,28 @@ EUROPEAN_VALUES = Task(
 )
 
 
+MCSTEREO = Task(
+    name="multiple-choice-stereotype-bias",
+    task_group=TaskGroup.MULTIPLE_CHOICE_CLASSIFICATION,
+    template_dict=MULTIPLE_CHOICE_TEMPLATES,
+    metrics=[
+        m.bias_adjusted_accuracy_ambig_metric,
+        m.bias_ambig_metric,
+        m.accuracy_ambig_metric,
+    ],
+    default_num_few_shot_examples=0,
+    default_max_generated_tokens=NUM_GENERATION_TOKENS_FOR_CLASSIFICATION,
+    default_labels=["a", "b", "c"],
+    default_allowed_model_types=[ModelType.GENERATIVE],
+    default_allowed_generative_types=[
+        GenerativeType.INSTRUCTION_TUNED,
+        GenerativeType.REASONING,
+    ],
+    requires_zero_shot=True,
+    uses_logprobs=True,
+)
+
+
 SPEED = Task(
     name="speed",
     task_group=TaskGroup.SPEED,

scandeval/tokenisation_utils.py

@@ -6,6 +6,7 @@ import re
 import typing as t
 
 import torch
+from transformers import BatchEncoding
 
 from .constants import BOS_TOKENS, EOS_TOKENS, PAD_TOKENS
 from .enums import GenerativeType
@@ -340,7 +341,17 @@ def get_end_of_chat_token_ids(
         if "does not have a chat template" in str(e):
             return None
         raise e
-    assert isinstance(token_ids, list)
+
+    assert isinstance(token_ids, (BatchEncoding, list)), (
+        f"Expected token_ids to be a BatchEncoding or list, but got {type(token_ids)}.",
+    )
+
+    if isinstance(token_ids, BatchEncoding):
+        token_ids = token_ids.input_ids
+
+    assert isinstance(token_ids, list), (
+        f"Expected token_ids to be a list, but got {type(token_ids)}.",
+    )
 
     for idx, token in enumerate(tokeniser.convert_ids_to_tokens(token_ids)):
         if "X" in token:

scandeval/utils.py

@@ -1,22 +1,14 @@
 """Utility functions to be used in other scripts."""
 
-import asyncio
-import collections.abc as c
 import gc
-import importlib
-import importlib.metadata
-import importlib.util
 import logging
 import os
 import random
-import re
 import socket
 import sys
 import typing as t
 from pathlib import Path
-from types import ModuleType, TracebackType
 
-import demjson3
 import huggingface_hub as hf_hub
 import numpy as np
 import torch
@@ -24,48 +16,14 @@ from huggingface_hub.errors import LocalTokenNotFoundError
 from requests.exceptions import RequestException
 
 from .caching_utils import cache_arguments
-from .constants import T
-from .exceptions import InvalidBenchmark, InvalidModel, NaNValueInModelOutput
-from .logging_utils import log, log_once
+from .constants import LOCAL_MODELS_REQUIRED_FILES
+from .exceptions import InvalidModel, NaNValueInModelOutput
+from .logging_utils import log_once
 
 if t.TYPE_CHECKING:
-    from .data_models import ModelIdComponents
     from .types import Predictions
 
 
-def create_model_cache_dir(cache_dir: str, model_id: str) -> str:
-    """Create cache directory for a model.
-
-    Args:
-        cache_dir:
-            The cache directory.
-        model_id:
-            The model ID.
-
-    Returns:
-        The path to the cache directory.
-    """
-    # If the model ID is a path, we just use that as the cache dir
-    if Path(model_id).is_dir():
-        log_once(
-            f"Since the model {model_id!r} is a local model, we will use the model "
-            "directory directly as the model cache directory.",
-            level=logging.DEBUG,
-        )
-        return model_id
-
-    # Otherwise, we create a cache dir based on the model ID
-    model_cache_dir = Path(
-        cache_dir, "model_cache", model_id.replace("/", "--")
-    ).as_posix()
-    log_once(
-        f"Using the model cache directory {model_cache_dir!r} for the model "
-        f"{model_id!r}.",
-        level=logging.DEBUG,
-    )
-    return model_cache_dir
-
-
 def resolve_model_path(download_dir: str) -> str:
     """Resolve the path to the directory containing the model config files and weights.
 
@@ -107,16 +65,16 @@ def resolve_model_path(download_dir: str) -> str:
             f"at {model_path}"
         )
 
-    # Check that found_files contains at least a 'config.json'
-    config_file = next(
-        (file for file in found_files if file.name == "config.json"), None
+    # Check that found_files contains at least one of the required files
+    found_required_file = next(
+        (file for file in found_files if file.name in LOCAL_MODELS_REQUIRED_FILES), None
     )
-    if config_file is None:
+    if found_required_file is None:
         raise InvalidModel(
-            f"Missing required file 'config.json' for {model_id_path.strip('models--')}"
-            f"at {model_path}"
+            f"At least one of the files {LOCAL_MODELS_REQUIRED_FILES} must be present "
+            f"for {model_id_path.strip('models--')} at {model_path}"
         )
-    model_path = config_file.parent
+    model_path = found_required_file.parent
 
     # As a precaution we also check that all of the files are in the same directory
     # if not we create a new dir with symlinks to all of the files from all snapshots
@@ -164,47 +122,6 @@ def enforce_reproducibility(seed: int = 4242) -> np.random.Generator:
     return rng
 
 
-def get_class_by_name(
-    class_name: str | c.Sequence[str], module_name: str
-) -> t.Type | None:
-    """Get a class by its name.
-
-    Args:
-        class_name:
-            The name of the class, written in kebab-case. The corresponding class name
-            must be the same, but written in PascalCase, and lying in a module with the
-            same name, but written in snake_case. If a list of strings is passed, the
-            first class that is found is returned.
-        module_name:
-            The name of the module where the class is located.
-
-    Returns:
-        The class. If the class is not found, None is returned.
-    """
-    if isinstance(class_name, str):
-        class_name = [class_name]
-
-    error_messages = list()
-    for name in class_name:
-        try:
-            module = importlib.import_module(name=module_name)
-            class_: t.Type = getattr(module, name)
-            return class_
-        except (ModuleNotFoundError, AttributeError) as e:
-            error_messages.append(str(e))
-
-    if error_messages:
-        errors = "\n- " + "\n- ".join(error_messages)
-        log(
-            f"Could not find the class with the name(s) {', '.join(class_name)}. The "
-            f"following error messages were raised: {errors}",
-            level=logging.DEBUG,
-        )
-
-    # If the class could not be found, return None
-    return None
-
-
 def get_min_cuda_compute_capability() -> float | None:
     """Gets the lowest cuda capability.
 
@@ -222,7 +139,7 @@ def get_min_cuda_compute_capability() -> float | None:
 
 @cache_arguments(disable_condition=lambda: hasattr(sys, "_called_from_test"))
 def internet_connection_available() -> bool:
-    """Checks if internet connection is available by pinging google.com.
+    """Checks if internet connection is available.
 
     Returns:
         Whether or not internet connection is available.
@@ -265,141 +182,6 @@ def raise_if_model_output_contains_nan_values(model_output: "Predictions") -> No
                 raise NaNValueInModelOutput()
 
 
-def scramble(text: str) -> str:
-    """Scramble a string in a bijective manner.
-
-    Args:
-        text:
-            The string to scramble.
-
-    Returns:
-        The scrambled string.
-    """
-    rng = np.random.default_rng(seed=4242)
-    permutation = rng.permutation(x=len(text))
-    scrambled = "".join(text[i] for i in permutation)
-    return scrambled
-
-
-def unscramble(scrambled_text: str) -> str:
-    """Unscramble a string in a bijective manner.
-
-    Args:
-        scrambled_text:
-            The scrambled string to unscramble.
-
-    Returns:
-        The unscrambled string.
-    """
-    rng = np.random.default_rng(seed=4242)
-    permutation = rng.permutation(x=len(scrambled_text))
-    inverse_permutation = np.argsort(permutation)
-    unscrambled = "".join(scrambled_text[i] for i in inverse_permutation)
-    return unscrambled
-
-
-def get_package_version(package_name: str) -> str | None:
-    """Get the version of a package.
-
-    Args:
-        package_name:
-            The name of the package.
-
-    Returns:
-        The version of the package, or None if the package is not installed.
-    """
-    try:
-        return importlib.metadata.version(package_name)
-    except importlib.metadata.PackageNotFoundError:
-        return None
-
-
-def safe_run(coroutine: t.Coroutine[t.Any, t.Any, T]) -> T:
-    """Run a coroutine, ensuring that the event loop is always closed when we're done.
-
-    Args:
-        coroutine:
-            The coroutine to run.
-
-    Returns:
-        The result of the coroutine.
-    """
-    try:
-        loop = asyncio.get_event_loop()
-    except RuntimeError:  # If the current event loop is closed
-        loop = asyncio.new_event_loop()
-        asyncio.set_event_loop(loop)
-    response = loop.run_until_complete(coroutine)
-    return response
-
-
-async def add_semaphore_and_catch_exception(
-    coroutine: t.Coroutine[t.Any, t.Any, T], semaphore: asyncio.Semaphore
-) -> T | Exception:
-    """Run a coroutine with a semaphore.
-
-    Args:
-        coroutine:
-            The coroutine to run.
-        semaphore:
-            The semaphore to use.
-
-    Returns:
-        The result of the coroutine.
-    """
-    async with semaphore:
-        try:
-            return await coroutine
-        except Exception as exc:
-            return exc
-
-
-def extract_json_dict_from_string(s: str) -> dict | None:
-    """Extract a JSON dictionary from a string.
-
-    Args:
-        s:
-            The string to extract the JSON dictionary from.
-
-    Returns:
-        The extracted JSON dictionary, or None if no JSON dictionary could be found.
-    """
-    json_regex = r"\{[^{}]*?\}"
-    if (json_match := re.search(pattern=json_regex, string=s, flags=re.DOTALL)) is None:
-        log(
-            "The model output does not contain any JSON dictionary, so cannot parse "
-            f"it. Skipping. Here is the output: {s!r}",
-            level=logging.DEBUG,
-        )
-        return None
-    json_string = json_match.group()
-    try:
-        json_output = demjson3.decode(txt=json_string)
-    except demjson3.JSONDecodeError:
-        log(
-            "The model output is not valid JSON, so cannot parse it. Skipping. "
-            f"Here is the output: {json_string!r}",
-            level=logging.DEBUG,
-        )
-        return None
-    if not isinstance(json_output, dict):
-        log(
-            "The model output is not a JSON dictionary, so cannot parse "
-            f"it. Skipping. Here is the output: {json_string!r}",
-            level=logging.DEBUG,
-        )
-        return None
-    elif not all(isinstance(key, str) for key in json_output.keys()):
-        log(
-            "The model output is not a JSON dictionary with string keys, "
-            "so cannot parse it. Skipping. Here is the output: "
-            f"{json_string!r}",
-            level=logging.DEBUG,
-        )
-        return None
-    return json_output
-
-
 @cache_arguments()
 def get_hf_token(api_key: str | None) -> str | bool:
     """Get the Hugging Face token.
@@ -419,10 +201,9 @@ def get_hf_token(api_key: str | None) -> str | bool:
             level=logging.DEBUG,
         )
         return api_key
-    elif (token := os.getenv("HUGGINGFACE_API_KEY")) is not None:
+    elif (token := os.getenv("HF_TOKEN")) is not None:
         log_once(
-            "Using the Hugging Face API key from the environment variable "
-            "`HUGGINGFACE_API_KEY`.",
+            "Using the Hugging Face API key from the environment variable `HF_TOKEN`.",
             level=logging.DEBUG,
         )
         return token
@@ -448,154 +229,3 @@ def get_hf_token(api_key: str | None) -> str | bool:
             level=logging.DEBUG,
         )
         return False
-
-
-def extract_multiple_choice_labels(
-    prompt: str, candidate_labels: c.Sequence[str]
-) -> c.Sequence[str]:
-    """Extract multiple choice labels from a prompt.
-
-    Args:
-        prompt:
-            The prompt to extract the labels from.
-        candidate_labels:
-            The candidate labels to look for in the prompt.
-
-    Returns:
-        The extracted labels.
-    """
-    sample_candidate_labels: list[str] = list()
-    for candidate_label in candidate_labels:
-        candidate_label_match = re.search(
-            pattern=rf"\b{candidate_label}\. ", string=prompt, flags=re.IGNORECASE
-        )
-        if candidate_label_match is not None:
-            sample_candidate_labels.append(candidate_label)
-    if not sample_candidate_labels:
-        raise InvalidBenchmark(
-            "Could not extract any candidate labels from the prompt. Please ensure "
-            "that the candidate labels are present in the prompt, each followed by a "
-            "dot and a space (e.g., 'a. '). The candidate labels are: "
-            f"{', '.join(candidate_labels)}. Here is the prompt: {prompt!r}"
-        )
-    return sample_candidate_labels
-
-
-def split_model_id(model_id: str) -> "ModelIdComponents":
-    """Split a model ID into its components.
-
-    Args:
-        model_id:
-            The model ID to split.
-
-    Returns:
-        The split model ID.
-
-    Raises:
-        If the model ID is not valid.
-    """
-    # Importing here to avoid circular imports
-    from .data_models import ModelIdComponents
-
-    # Attempt to extract the model ID, revision, and param using regex
-    model_id_match = re.match(pattern=r"^[^@#]+", string=model_id)
-    revision_match = re.search(pattern=r"@([^@#]+)", string=model_id)
-    param_match = re.search(pattern=r"#([^@#]+)", string=model_id)
-
-    # If we cannot extract the model ID, raise an error
-    if model_id_match is None:
-        raise InvalidModel(f"The model ID {model_id!r} is not valid.")
-    model_id = model_id_match.group()
-
-    # Extract the revision and param and return the result
-    revision = revision_match.group(1) if revision_match is not None else "main"
-    param = param_match.group(1) if param_match is not None else None
-    return ModelIdComponents(model_id=model_id, revision=revision, param=param)
-
-
-def load_custom_datasets_module(custom_datasets_file: Path) -> ModuleType | None:
-    """Load the custom datasets module if it exists.
-
-    Args:
-        custom_datasets_file:
-            The path to the custom datasets module.
-
-    Raises:
-        RuntimeError:
-            If the custom datasets module cannot be loaded.
-    """
-    if custom_datasets_file.exists():
-        spec = importlib.util.spec_from_file_location(
-            name="custom_datasets_module", location=str(custom_datasets_file.resolve())
-        )
-        if spec is None:
-            log_once(
-                "Could not load the spec for the custom datasets file from "
-                f"{custom_datasets_file.resolve()}.",
-                level=logging.ERROR,
-            )
-            return None
-        module = importlib.util.module_from_spec(spec=spec)
-        if spec.loader is None:
-            log_once(
-                "Could not load the module for the custom datasets file from "
-                f"{custom_datasets_file.resolve()}.",
-                level=logging.ERROR,
-            )
-            return None
-        spec.loader.exec_module(module)
-        return module
-    return None
-
-
-class attention_backend:
-    """Context manager to temporarily set the attention backend.
-
-    This sets the `VLLM_ATTENTION_BACKEND` environment variable to the desired value
-    for the duration of the context manager, and restores the previous value afterwards.
-    """
-
-    def __init__(self, value: str | None) -> None:
-        """Initialise the context manager.
-
-        Args:
-            value:
-                The name of the attention backend to set. If None then no change is
-                made. Also, if the user has already set the `VLLM_ATTENTION_BACKEND` env
-                var, then no change is made.
-        """
-        user_has_set_backend = (
-            os.environ.get("USER_HAS_SET_VLLM_ATTENTION_BACKEND", "0") == "1"
-        )
-        self.value = None if user_has_set_backend else value
-        self.previous_value: str | None = None
-
-    def __enter__(self) -> None:
-        """Enter the context manager."""
-        if self.value is None:
-            return
-        self.previous_value = os.getenv("VLLM_ATTENTION_BACKEND")
-        os.environ["VLLM_ATTENTION_BACKEND"] = self.value
-
-    def __exit__(
-        self,
-        exc_type: t.Type[BaseException] | None,
-        exc_value: BaseException | None,
-        exc_tb: TracebackType | None,
-    ) -> None:
-        """Exit the context manager.
-
-        Args:
-            exc_type:
-                The type of the exception.
-            exc_value:
-                The value of the exception.
-            exc_tb:
-                The traceback of the exception.
-        """
-        if self.value is None:
-            return
-        if self.previous_value is None:
-            os.environ.pop("VLLM_ATTENTION_BACKEND", None)
-        else:
-            os.environ["VLLM_ATTENTION_BACKEND"] = self.previous_value