eval-framework 0.2.3__tar.gz → 0.2.4__tar.gz

{eval_framework-0.2.3 → eval_framework-0.2.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.2.3
+Version: 0.2.4
 Summary: Evalulation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License
@@ -232,8 +232,9 @@ Requires-Dist: llm-sandbox[docker]>=0.1.8,<0.2
 Requires-Dist: jsonlines>=4,<5
 Requires-Dist: lxml>=6,<7
 Requires-Dist: python-iso639>=2025.2.18
-Requires-Dist: wandb>=0.21.1,<1
+Requires-Dist: wandb>=0.23.0,<1
 Requires-Dist: boto3>=1.40.54,<2
+Requires-Dist: numpy>=1.26.4
 Requires-Dist: accelerate ; extra == 'accelerate'
 Requires-Dist: eval-framework[determined,api,openai,transformers,accelerate,vllm,comet,optional,mistral] ; extra == 'all'
 Requires-Dist: aleph-alpha-client>=10,<11 ; extra == 'api'
@@ -243,8 +244,9 @@ Requires-Dist: tensorboard==2.19.0 ; extra == 'determined'
 Requires-Dist: mistral-common>=1.7,<2 ; extra == 'mistral'
 Requires-Dist: huggingface-hub>=0.33.2,<0.34 ; extra == 'mistral'
 Requires-Dist: eval-framework[vllm] ; extra == 'mistral'
-Requires-Dist: openai>=1.62,<2 ; extra == 'openai'
+Requires-Dist: openai>=1.62,<2.8 ; extra == 'openai'
 Requires-Dist: tiktoken>=0.9,<0.10 ; extra == 'openai'
+Requires-Dist: transformers>=4.45.2,<5 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'optional'
 Requires-Dist: jinja2>=3.1.6,<4 ; extra == 'optional'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'transformers'
@@ -354,22 +356,25 @@ For more detailed CLI usage instructions, see the [CLI Usage Guide](docs/cli_usa
 
 ### Core Capabilities
 
-| **Reasoning** | **Knowledge** | **Coding** | **Long Context** |
-|---------------|---------------|------------|------------------|
-| MMLU (57 subjects) | TriviaQA | HumanEval | InfiniteBench |
-| SQuAD v1/v2 | MBPP |
-| ARC | Natural Questions | CodeT5 | ZeroSCROLLS |
-| HellaSwag | QuAC | Programming | QuALITY |
-| Winogrande | COPA | Debugging  |
+Subset of core capabilities benchmarks coverd by `eval-framework`:
+
+| **Reasoning** | **Knowledge** | **Math** | **Coding** | **Structured outputs** | **Long Context** |
+|---------------|---------------|----------|------------|------------------------|------------------|
+| COPA | ARC | AIME | BigCodeBench | IFEval | InfiniteBench |
+| Hellaswag | MMLU | GSM8K | HumanEval | StructEval | QUALITY |
+| Winogrande | Openbook QA| MATH-500 | MBPP | | ZeroSCROLLS |
+
 
 ### Languages & Domains
 
-| **Multilingual** | **Specialized** | **Safety & Bias** | **Efficiency** |
+Subset of language-specific and domain-specific benchmarks coverd by `eval-framework`:
+
+| **Multilingual** | **Specialized** | **Safety & Bias** | **Efficiency Metrics** |
 |------------------|-----------------|-------------------|----------------|
-| WMT Translation | Legal (CaseHold) | TruthfulQA | Token counting |
-| FLORES-200 | Winogender | Latency metrics |
-| Multilingual MMLU | Medical (MedQA) | Stereotype detection | Memory usage |
-| German/Finnish tasks | Scientific (SciQ) | Harmful content | Cost analysis |
+| WMT Translation | MMLU | TruthfulQA | Compression ratios |
+| FLORES-200 | Legal (CaseHold) | Winogender | Runtime |
+| Multilingual MMLU | Scientific (SciQ) | | |
+| German/Finnish tasks |  | | |
 
 ### Completion
 

{eval_framework-0.2.3 → eval_framework-0.2.4}/README.md

@@ -86,22 +86,25 @@ For more detailed CLI usage instructions, see the [CLI Usage Guide](docs/cli_usa
 
 ### Core Capabilities
 
-| **Reasoning** | **Knowledge** | **Coding** | **Long Context** |
-|---------------|---------------|------------|------------------|
-| MMLU (57 subjects) | TriviaQA | HumanEval | InfiniteBench |
-| SQuAD v1/v2 | MBPP |
-| ARC | Natural Questions | CodeT5 | ZeroSCROLLS |
-| HellaSwag | QuAC | Programming | QuALITY |
-| Winogrande | COPA | Debugging  |
+Subset of core capabilities benchmarks coverd by `eval-framework`:
+
+| **Reasoning** | **Knowledge** | **Math** | **Coding** | **Structured outputs** | **Long Context** |
+|---------------|---------------|----------|------------|------------------------|------------------|
+| COPA | ARC | AIME | BigCodeBench | IFEval | InfiniteBench |
+| Hellaswag | MMLU | GSM8K | HumanEval | StructEval | QUALITY |
+| Winogrande | Openbook QA| MATH-500 | MBPP | | ZeroSCROLLS |
+
 
 ### Languages & Domains
 
-| **Multilingual** | **Specialized** | **Safety & Bias** | **Efficiency** |
+Subset of language-specific and domain-specific benchmarks coverd by `eval-framework`:
+
+| **Multilingual** | **Specialized** | **Safety & Bias** | **Efficiency Metrics** |
 |------------------|-----------------|-------------------|----------------|
-| WMT Translation | Legal (CaseHold) | TruthfulQA | Token counting |
-| FLORES-200 | Winogender | Latency metrics |
-| Multilingual MMLU | Medical (MedQA) | Stereotype detection | Memory usage |
-| German/Finnish tasks | Scientific (SciQ) | Harmful content | Cost analysis |
+| WMT Translation | MMLU | TruthfulQA | Compression ratios |
+| FLORES-200 | Legal (CaseHold) | Winogender | Runtime |
+| Multilingual MMLU | Scientific (SciQ) | | |
+| German/Finnish tasks |  | | |
 
 ### Completion
 

{eval_framework-0.2.3 → eval_framework-0.2.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.2.3"
+version = "0.2.4"
 description = "Evalulation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -39,8 +39,9 @@ dependencies = [
   "jsonlines>=4,<5",
   "lxml>=6,<7",
   "python-iso639>=2025.2.18",
-  "wandb>=0.21.1,<1",
+  "wandb>=0.23.0,<1",
   "boto3>=1.40.54,<2",
+  "numpy>=1.26.4",
 ]
 
 [project.optional-dependencies]
@@ -51,8 +52,9 @@ determined = [
 ]
 api = ["aleph-alpha-client>=10,<11"]
 openai = [
-  "openai>=1.62,<2",
-  "tiktoken>=0.9,<0.10"
+  "openai>=1.62,<2.8",
+  "tiktoken>=0.9,<0.10",
+  "transformers>=4.45.2,<5",
 ]
 transformers = [
   "transformers>=4.45.2,<5",
@@ -149,6 +151,7 @@ select = [
     "UP", # Auto-upgrading of new Python features
     "I", # Sort imports
 ]
+
 [tool.ruff.lint.isort]
 # https://github.com/astral-sh/ruff-pre-commit/issues/121
 # https://github.com/astral-sh/ruff/issues/10519
@@ -163,7 +166,7 @@ known-third-party = ["wandb"]
 plugins = "pydantic.mypy"
 disallow_untyped_defs = true
 ignore_missing_imports = true
-files = ["src", "tests", "utils"]
+files = ["src", "utils"]
 
 [tool.pytest.ini_options]
 testpaths = ["./tests"]

{eval_framework-0.2.3 → eval_framework-0.2.4}/src/eval_framework/evaluation_generator.py

@@ -20,6 +20,7 @@ from eval_framework.tasks.base import ResponseType
 from eval_framework.tasks.eval_config import EvalConfig
 from eval_framework.tasks.registry import get_task
 from eval_framework.utils.constants import RED, RESET
+from eval_framework.utils.tqdm_handler import get_disable_bar_flag, safe_tqdm_write
 
 logger = logging.getLogger(__name__)
 
@@ -71,8 +72,8 @@ class EvaluationGenerator:
                 metric = metric_class()
 
             logger.info(f"Starting calculation of {metric.NAME}")
-            tqdm.write(f"INFO: Calculating {metric.NAME}")
-            for response in tqdm(responses, desc=f"Calculating {metric.NAME}"):
+            safe_tqdm_write(f"INFO: Calculating {metric.NAME}")
+            for response in tqdm(responses, desc=f"Calculating {metric.NAME}", disable=get_disable_bar_flag()):
                 if f"{response.subject}_{response.id}_{metric.__class__.__name__}" in subject_result_id_existing:
                     continue
 
@@ -109,7 +110,7 @@ class EvaluationGenerator:
                         self.result_processor.save_metrics_result(result)
 
             logger.info(f"Completed calculation of {metric.NAME}")
-            tqdm.write(f"INFO: Completed {metric.NAME}")
+            safe_tqdm_write(f"INFO: Completed {metric.NAME}")
 
         if not self.save_intermediate_results:
             self.result_processor.save_metrics_results(results)
@@ -224,7 +225,6 @@ class EvaluationGenerator:
         aggregated_results = self._aggregate_results(metrics_results)
 
         wandb.log(aggregated_results)
-
         self.result_processor.save_aggregated_results(aggregated_results)
         logger.info(aggregated_results)
         logger.info(f"{RED}[ Evaluation completed and results saved! ]{RESET}")

{eval_framework-0.2.3 → eval_framework-0.2.4}/src/eval_framework/llm/aleph_alpha.py

@@ -50,6 +50,7 @@ class AlephAlphaAPIModel(BaseLLM):
         self,
         formatter: BaseFormatter | None = None,
         checkpoint_name: str | None = None,
+        temperature: float | None = None,
         # Please see README.md for tips if adapting the following parameters.
         max_retries: int = 100,
         max_async_concurrent_requests: int = 32,
@@ -65,6 +66,7 @@ class AlephAlphaAPIModel(BaseLLM):
         else:
             self._formatter = formatter
         self._llm_name = checkpoint_name or self.LLM_NAME
+        self._temperature = temperature if temperature is not None else 0.0
         self.max_async_concurrent_requests = max_async_concurrent_requests
         self.max_retries = max_retries
         self.request_timeout_seconds = request_timeout_seconds
@@ -249,13 +251,7 @@ class AlephAlphaAPIModel(BaseLLM):
         max_tokens: int | None = None,
         temperature: float | None = None,
     ) -> list[RawCompletion]:
-        if temperature is None:
-            effective_temperature = 0.0  # Current default, TODO: refactor to use model's default
-            logger.info(
-                f"Using default temperature value: {effective_temperature} as no custom temperature value was provided"
-            )
-        else:
-            effective_temperature = temperature
+        effective_temperature = temperature if temperature is not None else self._temperature
 
         requests = []
 

eval_framework-0.2.4/src/eval_framework/llm/openai.py

@@ -0,0 +1,400 @@
+import concurrent.futures
+import logging
+import math
+import os
+import traceback
+from collections.abc import Callable, Sequence
+from functools import partial
+
+import tiktoken
+from openai import OpenAI
+from openai.types.chat import ChatCompletionAssistantMessageParam, ChatCompletionUserMessageParam
+from tokenizers import Tokenizer
+from transformers import AutoTokenizer
+
+from eval_framework.llm.base import BaseLLM
+from eval_framework.shared.types import ConcatCompression, Error, RawCompletion, RawLoglikelihood
+from eval_framework.tasks.base import Sample
+from template_formatting.formatter import BaseFormatter, ConcatFormatter, HFFormatter, Message
+
+logger = logging.getLogger(__name__)
+
+
+class OpenAIModel(BaseLLM):
+    """
+    LLM wrapper for OpenAI API providing text/chat completions and log-probability evaluation output.
+    """
+
+    LLM_NAME: str | None = None
+    DEFAULT_FORMATTER: Callable[[], BaseFormatter] | None = None
+    BYTES_PER_TOKEN: float = 4.0  # rule of thumb according to https://platform.openai.com/tokenizer
+
+    def __init__(
+        self,
+        model_name: str | None = None,
+        formatter: BaseFormatter | None = None,
+        temperature: float | None = None,
+        api_key: str | None = os.getenv("OPENAI_API_KEY", ""),
+        organization: str | None = None,
+        base_url: str | None = None,
+        bytes_per_token: float | None = None,
+    ) -> None:
+        """
+        Initialize the OpenAIModel.
+
+        Args:
+            model_name: OpenAI model name (e.g., "gpt-4o", "gpt-3.5-turbo"). If None, uses LLM_NAME class attribute.
+            formatter: Optional message formatter.
+            temperature: Sampling temperature used when not passed to generate methods (from 0.0 to 2.0).
+            api_key: OpenAI API key (defaults to OPENAI_API_KEY env variable).
+            organization: Optional OpenAI organization ID.
+            base_url: Optional API base URL for Azure or alternate endpoints.
+            bytes_per_token: Optional custom bytes per token scalar for non-standard models.
+        """
+        assert model_name is not None or self.LLM_NAME is not None, "A model name must be specified."
+        self._model_name = model_name if model_name else self.LLM_NAME
+        logger.info(f"Instantiating OpenAIModel with name: {self._model_name}")
+
+        self._formatter = formatter or (self.DEFAULT_FORMATTER() if self.DEFAULT_FORMATTER is not None else None)
+        self._temperature = temperature if temperature is not None else 0.0
+        assert 0.0 <= self._temperature <= 2.0, "Temperature must be between 0.0 and 2.0"
+
+        self._client = OpenAI(
+            api_key=api_key,
+            organization=organization,
+            base_url=base_url,
+        )
+
+        # Initialize tokenizer for the model
+        self._encoder = self._get_encoder()
+
+        # set bytes_per_token_scalar for non-standard models
+        if bytes_per_token is not None and bytes_per_token <= 0:
+            raise ValueError("bytes_per_token must be positive")
+        self.bytes_per_token_scalar = (
+            4.0 / bytes_per_token if bytes_per_token is not None else 4.0 / self.BYTES_PER_TOKEN
+        )
+
+    def _get_encoder(self) -> tiktoken.Encoding:
+        assert self._model_name is not None
+        return tiktoken.encoding_for_model(self._model_name)
+
+    def _count_tokens(self, text: str) -> int:
+        """
+        Count tokens for the given text using the encoder.
+
+        Args:
+            text: Input string.
+
+        Returns:
+            Number of tokens.
+        """
+        return len(self._encoder.encode(text))
+
+    def generate_from_messages(
+        self,
+        messages: list[Sequence[Message]],
+        stop_sequences: list[str] | None = None,
+        max_tokens: int | None = None,
+        temperature: float | None = None,
+    ) -> list[RawCompletion]:
+        """
+        Generate completions for a list of message sequences concurrently.
+
+        Uses text completion API when a formatter is configured, otherwise uses chat completion API.
+
+        Args:
+            messages: Sequence of messages.
+            stop_sequences: Optional list of stop sequences.
+            max_tokens: Optional maximum number of tokens to generate.
+            temperature: Sampling temperature.
+
+        Returns:
+            List of RawCompletion objects containing prompts and completions.
+        """
+
+        effective_temperature = temperature if temperature is not None else self._temperature
+        assert 0.0 <= effective_temperature <= 2.0, "Temperature must be between 0.0 and 2.0"
+
+        def _process_one(single_messages: Sequence[Message]) -> RawCompletion:
+            # Adjust max tokens based on bytes_per_token_scalar so that non-standard models generate full responses
+            scaled_max_tokens = math.ceil(max_tokens * self.bytes_per_token_scalar) if max_tokens is not None else None
+
+            if self._formatter is not None:
+                # Use formatter and text completion API
+                prompt = self._formatter.format(single_messages, output_mode="string")
+                # documentation: https://platform.openai.com/docs/api-reference/completions/create
+                assert self._model_name is not None
+                response = self._client.completions.create(
+                    model=self._model_name,
+                    prompt=prompt,
+                    temperature=effective_temperature,
+                    max_tokens=scaled_max_tokens,
+                    stop=stop_sequences,
+                )
+                completion = response.choices[0].text
+                return RawCompletion(
+                    prompt=prompt,
+                    prompt_sequence_positions=self._count_tokens(prompt),
+                    concat_compression=ConcatCompression.calculate(
+                        single_messages, count_tokens=self._count_tokens, completion=completion
+                    ),
+                    completion=completion,
+                    completion_sequence_positions=self._count_tokens(completion),
+                )
+
+            else:
+                # Use chat completion API
+                chat_messages = [
+                    (
+                        ChatCompletionUserMessageParam(role="user", content=m.content)
+                        if m.role is not None and m.role.value.lower() == "user"
+                        else ChatCompletionAssistantMessageParam(role="assistant", content=m.content)
+                    )
+                    for m in single_messages
+                ]
+                assert self._model_name is not None
+                chat_response = self._client.chat.completions.create(
+                    model=self._model_name,
+                    messages=chat_messages,
+                    temperature=effective_temperature,
+                    max_tokens=scaled_max_tokens,
+                    stop=stop_sequences,
+                )
+                prompt = "\n".join([f"{m.get('role', '')}: {m.get('content', '')}" for m in chat_messages])
+                prompt_tokens = getattr(chat_response.usage, "prompt_tokens", None)
+                completion = chat_response.choices[0].message.content or ""
+                return RawCompletion(
+                    prompt=prompt,
+                    prompt_sequence_positions=prompt_tokens,
+                    concat_compression=ConcatCompression.calculate(
+                        single_messages, count_tokens=self._count_tokens, completion=completion
+                    ),
+                    completion=completion,
+                    completion_sequence_positions=self._count_tokens(completion),
+                )
+
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            results = list(executor.map(_process_one, messages))
+        return results
+
+    def logprobs(self, samples: list[Sample]) -> list[RawLoglikelihood]:
+        """
+        Compute total log-probabilities for possible completions given each sample's prompt.
+
+        Args:
+            samples: List of Sample objects, each with prompt messages and possible completions.
+
+        Returns:
+            List of RawLoglikelihood objects mapping each prompt and completion to its log-probability.
+
+        Note:
+            Uses the OpenAI completions API with echo=True; chat logprobs are not supported.
+        """
+        assert self._model_name in ["babbage-002", "davinci-002"], (
+            "Log-probs for prompt tokens are only supported for a limited set of models."
+        )
+        # apparently OpenAI stopped providing logprobs of prompt tokens, see discussion in:
+        # https://github.com/EleutherAI/lm-evaluation-harness/issues/1196
+
+        assert self._formatter is not None, "Log-probs require a formatter to create text prompts."
+        results: list[RawLoglikelihood] = []
+        for sample in samples:
+            prompt = self._formatter.format(sample.messages, output_mode="string") if sample.messages else ""
+            choices_log_probs: dict[str, float] = {}
+            choices_sequence_positions: dict[str, int] = {}
+            prompt_sequence_positions: int | None = self._count_tokens(prompt)
+            error: Error | None = None
+
+            for choice in sample.possible_completions or []:
+                if error is not None:
+                    continue
+
+                # Tokenize prompt and completion
+                prompt_tokens = self._encoder.encode(prompt)
+                completion_tokens = self._encoder.encode(choice)
+                full_text = prompt + choice
+
+                try:
+                    response = self._client.completions.create(
+                        model=self._model_name,
+                        prompt=full_text,
+                        echo=True,
+                        max_tokens=0,
+                        logprobs=1,
+                        temperature=0,
+                    )
+
+                    choice_obj = response.choices[0]
+                    if not hasattr(choice_obj, "logprobs") or choice_obj.logprobs is None:
+                        raise ValueError("Logprobs not returned in response.")
+
+                    all_tokens = getattr(choice_obj.logprobs, "tokens", None)
+                    all_logprobs = getattr(choice_obj.logprobs, "token_logprobs", None)
+
+                    if all_tokens is None or all_logprobs is None:
+                        raise ValueError("Logprobs response missing expected 'tokens' or 'token_logprobs' fields.")
+
+                    if len(all_tokens) != len(prompt_tokens) + len(completion_tokens):
+                        raise ValueError(
+                            f"Token count mismatch: tokens in response ({len(all_tokens)}) != prompt+completion "
+                            f"({len(prompt_tokens) + len(completion_tokens)})"
+                        )
+
+                    # Sum logprobs for the completion portion
+                    choices_log_probs[choice] = sum(all_logprobs[len(prompt_tokens) :])
+                    choices_sequence_positions[choice] = len(completion_tokens)
+
+                except Exception as e:
+                    error = Error(error_class=e.__class__.__name__, message=str(e), traceback=traceback.format_exc())
+                    prompt_sequence_positions = None
+                    choices_log_probs = {}
+                    choices_sequence_positions = {}
+
+            results.append(
+                RawLoglikelihood(
+                    prompt=prompt,
+                    prompt_sequence_positions=prompt_sequence_positions,
+                    loglikelihoods=choices_log_probs,
+                    loglikelihoods_sequence_positions=choices_sequence_positions,
+                    raw_loglikelihood_error=error,
+                )
+            )
+        return results
+
+    def __del__(self) -> None:
+        if hasattr(self, "_client"):
+            self._client.close()
+
+
+class OpenAIEmbeddingModel(BaseLLM):
+    def __init__(
+        self,
+        model_name: str = "text-embedding-3-large",
+        formatter: BaseFormatter | None = None,
+        api_key: str | None = None,
+        organization: str | None = None,
+        base_url: str | None = None,
+    ) -> None:
+        """Initialize OpenAI API client.
+        Args:
+            model_name: Name of the OpenAI model to use (e.g., "text-embedding-3-large")
+            formatter: Optional message formatter
+            api_key: OpenAI API key (defaults to OPENAI_API_KEY env variable)
+            organization: Optional organization ID
+            base_url: Optional API base URL for Azure or other endpoints
+        """
+        if formatter is not None:
+            raise ValueError("Formatter is not supported for embedding model.")
+        self._model_name = model_name
+        logger.info(f"Using {model_name} as embedding model")
+        self._client = OpenAI(
+            api_key=api_key or os.getenv("OPENAI_API_KEY", ""),
+            organization=organization,
+            base_url=base_url,
+        )
+
+    def generate_from_messages(
+        self,
+        messages: list[Sequence[Message]],
+        stop_sequences: list[str] | None = None,
+        max_tokens: int | None = None,
+        temperature: float | None = None,
+    ) -> list[RawCompletion]:
+        raise NotImplementedError(
+            "Embedding model does not support generate_from_messages. Use generate_embeddings instead."
+        )
+
+    def generate_embeddings(
+        self,
+        messages: list[Sequence[Message]],
+    ) -> list[list[float]]:
+        embeddings = []
+        for single_messages in messages:
+            prompt = "".join([m.content for m in single_messages])
+            response = self._client.embeddings.create(model=self._model_name, input=[prompt])
+            embedding = response.data[0].embedding
+            embeddings.append(embedding)
+        return embeddings
+
+    def logprobs(self, samples: list[Sample]) -> list[RawLoglikelihood]:
+        raise NotImplementedError("Embedding model cannot return logprobs.")
+
+    def __del__(self) -> None:
+        if hasattr(self, "_client"):
+            self._client.close()
+            try:
+                self._client.close()
+            except Exception:
+                pass
+
+
+class DeepseekModel(OpenAIModel):
+    """
+    General Deepseek model wrapper using OpenAI-compatible API for deepseek-chat and deepseek-reasoner models.
+
+    Using the deepseek API: https://api-docs.deepseek.com/quick_start/pricing
+    """
+
+    def __init__(
+        self,
+        model_name: str | None = None,
+        formatter: BaseFormatter | None = None,
+        temperature: float | None = None,
+        api_key: str | None = None,
+        organization: str | None = None,
+        base_url: str | None = None,
+        tokenizer_name: str | None = None,
+    ) -> None:
+        super().__init__(
+            model_name=model_name,
+            formatter=formatter,
+            temperature=temperature,
+            api_key=os.getenv("DEEPSEEK_API_KEY", ""),
+            organization=organization,
+            base_url="https://api.deepseek.com/beta",
+        )
+        self._tokenizer_name = tokenizer_name if tokenizer_name is not None else "deepseek-ai/DeepSeek-V3.2-Exp"
+
+    def _get_encoder(self) -> Tokenizer:
+        return AutoTokenizer.from_pretrained(self._tokenizer_name)
+
+    def _count_tokens(self, text: str) -> int:
+        return len(self._encoder.encode(text))
+
+
+### Model Aliases ###
+
+
+class OpenAI_gpt_4o_mini(OpenAIModel):
+    LLM_NAME = "gpt-4o-mini-2024-07-18"
+
+
+class OpenAI_gpt_4o_mini_with_ConcatFormatter(OpenAIModel):
+    LLM_NAME = "gpt-4o-mini-2024-07-18"
+    DEFAULT_FORMATTER = ConcatFormatter
+
+
+class OpenAI_davinci_002(OpenAIModel):
+    LLM_NAME = "davinci-002"
+    DEFAULT_FORMATTER = ConcatFormatter
+
+
+class Deepseek_reasoner(DeepseekModel):
+    LLM_NAME = "deepseek-reasoner"  # DeepSeek-V3.2-Exp (Thinking Mode)
+    # multi-round conversations for reasoning model documented here:
+    # https://api-docs.deepseek.com/guides/reasoning_model#api-example
+    # does not support completion API
+
+
+class Deepseek_chat(DeepseekModel):
+    LLM_NAME = "deepseek-chat"  # DeepSeek-V3.2-Exp (Non-thinking Mode)
+
+
+class Deepseek_chat_with_formatter(DeepseekModel):
+    LLM_NAME = "deepseek-chat"  # DeepSeek-V3.2-Exp (Non-thinking Mode)
+    DEFAULT_FORMATTER = partial(HFFormatter, "deepseek-ai/DeepSeek-V3.2-Exp")
+    """
+        <｜begin▁of▁sentence｜><｜User｜>Question: What color is the night sky?
+        <｜Assistant｜></think>Answer:
+    """

{eval_framework-0.2.3 → eval_framework-0.2.4}/src/eval_framework/main.py

@@ -29,12 +29,13 @@ def main(
     trial_id: int | None = None,
     *args: Any,
     resource_cleanup: bool = False,
+    verbosity: int = 1,
 ) -> list[Result]:
     """Runs the entire evaluation process: responses generation and evaluation."""
     # Set up centralized logging early
     output_dir = generate_output_dir(llm.name, config)
-    print(f"Output directory for evaluation: {output_dir}")
-    setup_logging(output_dir=output_dir, log_level=logging.INFO, log_filename="evaluation.log")
+    setup_logging(output_dir=output_dir, log_level=verbosity, log_filename="evaluation.log")
+    logger.info(f"Output directory for evaluation: {output_dir}")
 
     logger.info(f"{RED}[ Running full evaluation process ------- ]{RESET}")
     logger.info(f"Evaluating {llm.name} on {config.task_name}")

eval_framework-0.2.4/src/eval_framework/metrics/completion/aidanbench.py

@@ -0,0 +1,28 @@
+import logging
+
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import Completion
+
+logger = logging.getLogger(__name__)
+
+
+class AidanBenchMetric(BaseMetric[Completion]):
+    NAME = "AidanBench"
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        # subtract 2 to not count 1) initial instruction and 2) the latest model response, which caused the stop
+        # i.e. was not (unique && coherent)
+        num_unique_responses = len(response.messages) - 2 if response.messages is not None else 0
+        if num_unique_responses < 0:
+            logger.warning(
+                "Number of unique responses calculated as negative, setting to 0."
+                "Likely something went wrong during answer generation."
+            )
+            num_unique_responses = 0
+        return [
+            MetricResult(
+                metric_name=f"{self.NAME}/num_responses",
+                value=num_unique_responses,
+                higher_is_better=True,
+            )
+        ]