eval-framework 0.10.4__tar.gz → 0.10.6__tar.gz

{eval_framework-0.10.4 → eval_framework-0.10.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.10.4
+Version: 0.10.6
 Summary: Evaluation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License

{eval_framework-0.10.4 → eval_framework-0.10.6}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.10.4"
+version = "0.10.6"
 description = "Evaluation Framework"
 readme = "README.md"
 requires-python = ">=3.12,<3.15"
@@ -78,12 +78,12 @@ eval_framework = "eval_framework.run:run"
 
 [dependency-groups]
 dev = [
-    "mypy>=2.3.0,<3",
+    "mypy>=2.3.1,<3",
     "pytest>=9.1.1,<10",
     "pytest-mock>=3.15.1",
     "pytest-xdist>=3.8.0,<4",
     "pytest-sugar>1.1,<2",
-    "types-pyyaml>=6.0.12.20260724,<7",
+    "types-pyyaml>=6.0.12.20260815,<7",
     "types-python-dateutil>=2.9.0.20260807,<3",
     "types-requests>=2.33.0.20260712,<3",
     "plotly>=6.9.0,<7",

{eval_framework-0.10.4 → eval_framework-0.10.6}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.10.4"
+version = "0.10.6"
 description = "Evaluation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -84,12 +84,12 @@ eval_framework = "eval_framework.run:run"
 
 [dependency-groups]
 dev = [
-  "mypy>=2.3.0,<3",
+  "mypy>=2.3.1,<3",
   "pytest>=9.1.1,<10",
   "pytest-mock>=3.15.1",
   "pytest-xdist>=3.8.0,<4",
   "pytest-sugar>1.1,<2",
-  "types-pyyaml>=6.0.12.20260724,<7",
+  "types-pyyaml>=6.0.12.20260815,<7",
   "types-python-dateutil>=2.9.0.20260807,<3",
   "types-requests>=2.33.0.20260712,<3",
   "plotly>=6.9.0,<7",

{eval_framework-0.10.4 → eval_framework-0.10.6}/src/eval_framework/evaluation_generator.py

@@ -7,16 +7,9 @@ import wandb
 from tqdm import tqdm
 
 from eval_framework.metrics.base import BaseMetric
-from eval_framework.metrics.efficiency.bytes_per_sequence_position import (
-    BytesCompletion,
-    BytesLoglikelihood,
-    SequencePositionsCompletion,
-    SequencePositionsLoglikelihood,
-)
 from eval_framework.metrics.llm.base import BaseLLMJudgeMetric
 from eval_framework.result_processors.base import Result, ResultProcessor
 from eval_framework.shared.types import Completion, Loglikelihood
-from eval_framework.tasks.base import ResponseType
 from eval_framework.tasks.eval_config import EvalConfig
 from eval_framework.tasks.registry import registry
 from eval_framework.utils.constants import RED, RESET
@@ -37,16 +30,7 @@ class EvaluationGenerator:
         self.save_intermediate_results = config.save_intermediate_results
 
         eval_ = registry()[config.task_name]
-        response_type = eval_.response_type()
-        task_metrics = eval_.metrics()
-
-        if response_type == ResponseType.COMPLETION:
-            self.metrics = task_metrics + [BytesCompletion, SequencePositionsCompletion]
-        elif response_type == ResponseType.LOGLIKELIHOODS:
-            self.metrics = task_metrics + [BytesLoglikelihood, SequencePositionsLoglikelihood]
-        else:
-            raise NotImplementedError
-
+        self.metrics = eval_.metrics()
         self.task_name = eval_.display_name()
 
     def _run_metric_calculators(self, responses: list[Completion | Loglikelihood]) -> list[Result]:

{eval_framework-0.10.4 → eval_framework-0.10.6}/src/eval_framework/llm/aleph_alpha.py

@@ -165,33 +165,33 @@ class AlephAlphaAPIModel(BaseLLM):
         if isinstance(response, Error):
             return RawCompletion(
                 prompt=prompt,
-                prompt_sequence_positions=None,
+                prompt_num_tokens=None,
                 completion="",
-                completion_sequence_positions=0,
+                completion_num_tokens=0,
                 raw_completion_error=response,
             )
 
         assert len(response.completions) == 1
         completion = response.completions[0].completion or ""
-        prompt_sequence_positions: int | None = None
-        completion_sequence_positions: int | None = None
+        prompt_num_tokens: int | None = None
+        completion_num_tokens: int | None = None
 
         # Support workaround in api-worker-transformer's scaling generator to return the correct number of tokens.
         # These are part of the completion string; those in CompletionResponse are invalid in this case.
         m = re.match(r"\uf8c9(\d+),(\d+)\uf8c9(.*)", completion, re.DOTALL)
         if m is not None:
             num_input_tokens, num_completion_tokens, completion = m.groups()
-            prompt_sequence_positions = int(num_input_tokens)
-            completion_sequence_positions = int(num_completion_tokens)
+            prompt_num_tokens = int(num_input_tokens)
+            completion_num_tokens = int(num_completion_tokens)
         else:
-            prompt_sequence_positions = response.num_tokens_prompt_total if response else None
-            completion_sequence_positions = response.num_tokens_generated if response else None
+            prompt_num_tokens = response.num_tokens_prompt_total if response else None
+            completion_num_tokens = response.num_tokens_generated if response else None
 
         return RawCompletion(
             prompt=prompt,
-            prompt_sequence_positions=prompt_sequence_positions,
+            prompt_num_tokens=prompt_num_tokens,
             completion=completion,
-            completion_sequence_positions=completion_sequence_positions,
+            completion_num_tokens=completion_num_tokens,
         )
 
     def generate_from_messages(
@@ -254,8 +254,8 @@ class AlephAlphaAPIModel(BaseLLM):
         results: list[RawLoglikelihood] = []
         for sample_idx, (sample, prompt) in enumerate(zip(samples, prompts, strict=True)):
             choices_log_probs: dict[str, float] = {}
-            choices_sequence_positions: dict[str, int] = {}
-            prompt_sequence_positions: int | None = 0
+            choices_num_tokens: dict[str, int] = {}
+            prompt_num_tokens: int | None = 0
             number_of_initial_choices_tokens: int | None = None
             error: Error | None = None
 
@@ -267,9 +267,9 @@ class AlephAlphaAPIModel(BaseLLM):
 
                 if isinstance(response, Error):
                     error = response
-                    prompt_sequence_positions = None
+                    prompt_num_tokens = None
                     choices_log_probs = {}
-                    choices_sequence_positions = {}
+                    choices_num_tokens = {}
                 else:
                     try:
                         logprob, choice_token_count = self._extract_choice_logprob_from_completion(
@@ -278,7 +278,7 @@ class AlephAlphaAPIModel(BaseLLM):
                             response=response,
                         )
                         choices_log_probs[choice] = logprob
-                        choices_sequence_positions[choice] = choice_token_count
+                        choices_num_tokens[choice] = choice_token_count
                         if number_of_initial_choices_tokens is None:
                             number_of_initial_choices_tokens = choice_token_count
 
@@ -294,16 +294,16 @@ class AlephAlphaAPIModel(BaseLLM):
                             message=str(exc),
                             traceback=traceback.format_exc(),
                         )
-                        prompt_sequence_positions = None
+                        prompt_num_tokens = None
                         choices_log_probs = {}
-                        choices_sequence_positions = {}
+                        choices_num_tokens = {}
 
             results.append(
                 RawLoglikelihood(
                     prompt=prompt,
-                    prompt_sequence_positions=prompt_sequence_positions,
+                    prompt_num_tokens=prompt_num_tokens,
                     loglikelihoods=choices_log_probs,
-                    loglikelihoods_sequence_positions=choices_sequence_positions,
+                    loglikelihoods_num_tokens=choices_num_tokens,
                     raw_loglikelihood_error=error,
                 )
             )

{eval_framework-0.10.4 → eval_framework-0.10.6}/src/eval_framework/llm/huggingface.py

@@ -196,9 +196,9 @@ class BaseHFLLM(BaseLLM):
                 raw_completions.append(
                     RawCompletion(
                         prompt=prompt,
-                        prompt_sequence_positions=prompt_token_count,
+                        prompt_num_tokens=prompt_token_count,
                         completion="",
-                        completion_sequence_positions=0,
+                        completion_num_tokens=0,
                         raw_completion_error=Error(
                             error_class=PromptTooLongException.__name__,
                             message="Prompt exceeded context size.",
@@ -225,12 +225,12 @@ class BaseHFLLM(BaseLLM):
             raw_completions.append(
                 RawCompletion(
                     prompt=prompt,
-                    prompt_sequence_positions=prompt_token_count,
+                    prompt_num_tokens=prompt_token_count,
                     concat_compression=ConcatCompression.calculate(
                         single_messages, count_tokens=self.count_tokens, completion=completion
                     ),
                     completion=completion,
-                    completion_sequence_positions=completion_token_count,
+                    completion_num_tokens=completion_token_count,
                 )
             )
         return raw_completions
@@ -251,7 +251,7 @@ class BaseHFLLM(BaseLLM):
             # format
             prompt = self._formatter.format(sample.messages, output_mode="string")
             choices_log_probs: dict[str, float] = {}
-            choices_log_probs_sequence_positions: dict[str, float] = {}
+            choices_log_probs_num_tokens: dict[str, float] = {}
             error: Error | None = None
 
             for choice in sample.possible_completions or []:
@@ -266,7 +266,7 @@ class BaseHFLLM(BaseLLM):
                     if raise_errors():
                         raise PromptTooLongException("Prompt exceeded context size.")
                     choices_log_probs = {}
-                    choices_log_probs_sequence_positions = {}
+                    choices_log_probs_num_tokens = {}
                     error = Error(
                         error_class=PromptTooLongException.__name__,
                         message="Prompt and choice exceeded context size.",
@@ -278,17 +278,17 @@ class BaseHFLLM(BaseLLM):
                     sum_log_probs = self._model_log_probs(prompt_and_choice, num_choice_tokens)
 
                 choices_log_probs.update({choice: sum_log_probs})
-                choices_log_probs_sequence_positions.update({choice: num_choice_tokens})
+                choices_log_probs_num_tokens.update({choice: num_choice_tokens})
 
             results.append(
                 RawLoglikelihood(
                     prompt=prompt,
-                    prompt_sequence_positions=len(self.tokenizer.encode(prompt, add_special_tokens=False)),
+                    prompt_num_tokens=len(self.tokenizer.encode(prompt, add_special_tokens=False)),
                     concat_compression=ConcatCompression.calculate(
                         sample.messages, count_tokens=self.count_tokens, choices=sample.possible_completions
                     ),
                     loglikelihoods=choices_log_probs,
-                    loglikelihoods_sequence_positions=choices_log_probs_sequence_positions,
+                    loglikelihoods_num_tokens=choices_log_probs_num_tokens,
                     raw_loglikelihood_error=error,
                 )
             )

{eval_framework-0.10.4 → eval_framework-0.10.6}/src/eval_framework/llm/openai.py

@@ -186,7 +186,7 @@ class OpenAIModel(BaseLLM):
                 completion_tokens = getattr(usage, "completion_tokens", None) if usage is not None else None
                 return RawCompletion(
                     prompt=prompt,
-                    prompt_sequence_positions=(
+                    prompt_num_tokens=(
                         prompt_tokens
                         if prompt_tokens is not None
                         else (self._count_tokens(prompt) if self._encoder is not None else None)
@@ -201,7 +201,7 @@ class OpenAIModel(BaseLLM):
                         else None
                     ),
                     completion=completion,
-                    completion_sequence_positions=(
+                    completion_num_tokens=(
                         completion_tokens
                         if completion_tokens is not None
                         else (self._count_tokens(completion) if self._encoder is not None else None)
@@ -223,10 +223,12 @@ class OpenAIModel(BaseLLM):
                 prompt = "\n".join([f"{m.get('role', '')}: {m.get('content', '')}" for m in chat_messages])
                 prompt_tokens = getattr(chat_response.usage, "prompt_tokens", None)
                 completion_tokens = getattr(chat_response.usage, "completion_tokens", None)
+                completion_details = getattr(chat_response.usage, "completion_tokens_details", None)
+                reasoning_tokens = getattr(completion_details, "reasoning_tokens", None)
                 completion = chat_response.choices[0].message.content or ""
                 return RawCompletion(
                     prompt=prompt,
-                    prompt_sequence_positions=prompt_tokens,
+                    prompt_num_tokens=prompt_tokens,
                     concat_compression=(
                         ConcatCompression.calculate(
                             single_messages,
@@ -237,11 +239,12 @@ class OpenAIModel(BaseLLM):
                         else None
                     ),
                     completion=completion,
-                    completion_sequence_positions=(
+                    completion_num_tokens=(
                         completion_tokens
                         if completion_tokens is not None
                         else (self._count_tokens(completion) if self._encoder is not None else None)
                     ),
+                    reasoning_num_tokens=reasoning_tokens,
                 )
 
         with concurrent.futures.ThreadPoolExecutor() as executor:
@@ -276,8 +279,8 @@ class OpenAIModel(BaseLLM):
         for sample in samples:
             prompt = self._formatter.format(sample.messages, output_mode="string") if sample.messages else ""
             choices_log_probs: dict[str, float] = {}
-            choices_sequence_positions: dict[str, int] = {}
-            prompt_sequence_positions: int | None = self._count_tokens(prompt)
+            choices_num_tokens: dict[str, int] = {}
+            prompt_num_tokens: int | None = self._count_tokens(prompt)
             error: Error | None = None
 
             for choice in sample.possible_completions or []:
@@ -317,20 +320,20 @@ class OpenAIModel(BaseLLM):
 
                     # Sum logprobs for the completion portion
                     choices_log_probs[choice] = sum(all_logprobs[len(prompt_tokens) :])
-                    choices_sequence_positions[choice] = len(completion_tokens)
+                    choices_num_tokens[choice] = len(completion_tokens)
 
                 except Exception as e:
                     error = Error(error_class=e.__class__.__name__, message=str(e), traceback=traceback.format_exc())
-                    prompt_sequence_positions = None
+                    prompt_num_tokens = None
                     choices_log_probs = {}
-                    choices_sequence_positions = {}
+                    choices_num_tokens = {}
 
             results.append(
                 RawLoglikelihood(
                     prompt=prompt,
-                    prompt_sequence_positions=prompt_sequence_positions,
+                    prompt_num_tokens=prompt_num_tokens,
                     loglikelihoods=choices_log_probs,
-                    loglikelihoods_sequence_positions=choices_sequence_positions,
+                    loglikelihoods_num_tokens=choices_num_tokens,
                     raw_loglikelihood_error=error,
                 )
             )

eval_framework-0.10.6/src/eval_framework/metrics/efficiency/token_counters.py

@@ -0,0 +1,38 @@
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import Completion
+
+
+class TokenCounts(BaseMetric[Completion]):
+    """Number of tokens the model generated for the completion, and how many of
+    those were spent on reasoning (thinking).
+
+    Reads the token counts backends already attach to the response, so no extra
+    tokenisation is performed. Each value independently falls back to None when
+    the backend did not report that particular count or when the sample errored.
+    Reasoning counts are only exposed by some backends (e.g. OpenAI reasoning
+    models via `usage.completion_tokens_details.reasoning_tokens`, or vLLM
+    started with `--reasoning-parser`); non-reasoning models and backends that
+    do not surface a per-response count return None for that key.
+    """
+
+    NAME = "TokenCounts"
+    KEYS = ["Completion", "Reasoning"]
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error:
+            values: dict[str, float | None] = {"Completion": None, "Reasoning": None}
+        else:
+            values = {
+                "Completion": response.raw_completion_num_tokens,
+                "Reasoning": response.raw_completion_reasoning_num_tokens,
+            }
+
+        return [
+            MetricResult(
+                metric_name=f"{self.NAME}/{key}",
+                value=value,
+                higher_is_better=False,
+                error=response.error,
+            )
+            for key, value in values.items()
+        ]

{eval_framework-0.10.4 → eval_framework-0.10.6}/src/eval_framework/response_generator.py

@@ -110,9 +110,9 @@ class ResponseGenerator:
             raw_loglikelihoods = [
                 RawLoglikelihood(
                     prompt="",
-                    prompt_sequence_positions=0,
+                    prompt_num_tokens=0,
                     loglikelihoods={},
-                    loglikelihoods_sequence_positions={},
+                    loglikelihoods_num_tokens={},
                     raw_loglikelihood_error=Error(
                         error_class=e.__class__.__name__,
                         message=str(e),
@@ -132,10 +132,10 @@ class ResponseGenerator:
                     subject=sample.subject,
                     ground_truth=sample.ground_truth,
                     prompt=raw_loglikelihood.prompt,
-                    prompt_sequence_positions=raw_loglikelihood.prompt_sequence_positions,
+                    prompt_num_tokens=raw_loglikelihood.prompt_num_tokens,
                     concat_compression=raw_loglikelihood.concat_compression,
                     loglikelihoods=raw_loglikelihood.loglikelihoods,
-                    loglikelihoods_sequence_positions=raw_loglikelihood.loglikelihoods_sequence_positions,
+                    loglikelihoods_num_tokens=raw_loglikelihood.loglikelihoods_num_tokens,
                     error=raw_loglikelihood.raw_loglikelihood_error,
                 )
             )

{eval_framework-0.10.4 → eval_framework-0.10.6}/src/eval_framework/shared/types.py

@@ -78,21 +78,25 @@ class PromptTooLongException(Exception):
 class BaseCompletion(BaseModel):
     model_config = ConfigDict(extra="forbid")
     prompt: Annotated[str, "prompt as passed to the llm"]
-    prompt_sequence_positions: Annotated[
+    prompt_num_tokens: Annotated[
         int | None,
-        "number of sequence positions that the prompt occupies in the llm architecture (e.g. token count) "
-        "or None if the info is not available",
+        "number of tokens in the prompt, or None if the backend did not report it",
     ]
     completion: Annotated[str, "completion as generated by the llm"]
     concat_compression: Annotated[ConcatCompression | None, "Compression info for the concat formatter."] = None
 
 
 class RawCompletion(BaseCompletion):
-    completion_sequence_positions: Annotated[
+    completion_num_tokens: Annotated[
         int | None,
-        "number of sequence positions that the completion occupies in the llm architecture "
-        "(e.g. token count) or None if the info is not available",
+        "number of tokens the model generated for the completion, or None if the backend did not report it",
     ]
+    reasoning_num_tokens: Annotated[
+        int | None,
+        "portion of completion_num_tokens that the model spent on reasoning (thinking), "
+        "or None if the backend did not report it. Non-reasoning models and backends that "
+        "do not surface a reasoning-token count leave this None.",
+    ] = None
     raw_completion_error: Error | None = None
 
 
@@ -102,11 +106,15 @@ class Completion(BaseCompletion):
     ground_truth: str | None | list[str]
     messages: list[Message] | None  # needed for LLM as a judge
     raw_completion: Annotated[str, "raw completion as generated by the llm"]
-    raw_completion_sequence_positions: Annotated[
+    raw_completion_num_tokens: Annotated[
         int | None,
-        "number of sequence positions that the completion occupies in the llm architecture or None "
-        "if the info is not available",
+        "number of tokens the model generated for the raw completion, or None if the backend did not report it",
     ]
+    raw_completion_reasoning_num_tokens: Annotated[
+        int | None,
+        "portion of raw_completion_num_tokens that the model spent on reasoning, or None if the "
+        "backend did not report it",
+    ] = None
     context: list[BaseMetricContext] | BaseMetricContext | None = None
     error: Error | None = None
 
@@ -183,9 +191,9 @@ class Completion(BaseCompletion):
 class BaseLoglikelihood(BaseModel):
     model_config = ConfigDict(extra="forbid")
     prompt: str
-    prompt_sequence_positions: int | None
+    prompt_num_tokens: int | None
     loglikelihoods: dict[str, float]
-    loglikelihoods_sequence_positions: dict[str, int]  # Is empty if the model does not provide sequence positions
+    loglikelihoods_num_tokens: dict[str, int]  # Is empty if the model does not report per-choice token counts
     concat_compression: Annotated[ConcatCompression | None, "Compression info for the concat formatter"] = None
 
 

{eval_framework-0.10.4 → eval_framework-0.10.6}/src/eval_framework/tasks/base.py

@@ -2,6 +2,7 @@ import logging
 import os
 import random
 import traceback
+import typing
 from collections.abc import Iterable, Sequence
 from enum import Enum
 from pathlib import Path
@@ -10,6 +11,13 @@ from typing import TYPE_CHECKING, Any, Self, TypeVar
 import iso639
 from datasets import DatasetDict, DownloadConfig, load_dataset
 
+from eval_framework.metrics.efficiency.bytes_per_sequence_position import (
+    BytesCompletion,
+    BytesLoglikelihood,
+    SequencePositionsCompletion,
+    SequencePositionsLoglikelihood,
+)
+from eval_framework.metrics.efficiency.token_counters import TokenCounts
 from eval_framework.shared.types import BaseMetricContext, Completion, Error, RawCompletion
 from eval_framework.tasks.dataset_revisions import pinned_revision
 from eval_framework.tasks.markdown_doc import markdown_doc as render_markdown_doc
@@ -340,7 +348,7 @@ class BaseTask[SubjectType](Task):
             "sample_split": self.SAMPLE_SPLIT,
             "fewshot_split": self.FEWSHOT_SPLIT,
             "response_type": self.get_response_type().value,
-            "metrics": [m.NAME for m in self.get_metrics()],
+            "metrics": [m.NAME for m in self._get_task_specific_metrics()],
             "subjects": [str(s) for s in self.SUBJECTS],
         }
         if hasattr(self, "TASK_STYLER"):
@@ -377,9 +385,9 @@ class BaseTask[SubjectType](Task):
             raw_completions = [
                 RawCompletion(
                     prompt="",
-                    prompt_sequence_positions=0,
+                    prompt_num_tokens=0,
                     completion="",
-                    completion_sequence_positions=0,
+                    completion_num_tokens=0,
                     raw_completion_error=Error(
                         error_class=e.__class__.__name__, message=str(e), traceback=traceback.format_exc()
                     ),
@@ -414,12 +422,13 @@ class BaseTask[SubjectType](Task):
                     subject=sample.subject,
                     ground_truth=sample.ground_truth,
                     prompt=raw_completion.prompt,
-                    prompt_sequence_positions=raw_completion.prompt_sequence_positions,
+                    prompt_num_tokens=raw_completion.prompt_num_tokens,
                     concat_compression=raw_completion.concat_compression,
                     messages=messages,
                     completion=completion,
                     raw_completion=raw_completion.completion,
-                    raw_completion_sequence_positions=raw_completion.completion_sequence_positions,
+                    raw_completion_num_tokens=raw_completion.completion_num_tokens,
+                    raw_completion_reasoning_num_tokens=raw_completion.reasoning_num_tokens,
                     context=sample.context,
                     error=raw_completion.raw_completion_error or error,
                 )
@@ -433,12 +442,38 @@ class BaseTask[SubjectType](Task):
             return cls.TASK_STYLER.response_type
         return cls.RESPONSE_TYPE
 
+    @classmethod
+    def _get_task_specific_metrics(cls) -> list[type["BaseMetric"]]:
+        if hasattr(cls, "TASK_STYLER"):
+            task_metrics = cls.TASK_STYLER.metrics
+        else:
+            task_metrics = cls.METRICS
+        return task_metrics
+
+    @classmethod
+    def _get_response_type_specific_metrics(cls) -> list[type["BaseMetric"]]:
+        metrics: list[type[BaseMetric]]
+        match cls.get_response_type():
+            case ResponseType.COMPLETION:
+                metrics = [
+                    BytesCompletion,
+                    SequencePositionsCompletion,
+                    TokenCounts,
+                ]
+            case ResponseType.LOGLIKELIHOODS:
+                metrics = [BytesLoglikelihood, SequencePositionsLoglikelihood]
+            case _:
+                typing.assert_never(cls.get_response_type())
+
+        return metrics
+
     @classmethod
     def get_metrics(cls) -> list[type["BaseMetric"]]:
         """Return the metrics of the task (or the styler if it exists)."""
-        if hasattr(cls, "TASK_STYLER"):
-            return cls.TASK_STYLER.metrics
-        return cls.METRICS
+        task_metrics = cls._get_task_specific_metrics()
+        response_type_metrics = cls._get_response_type_specific_metrics()
+
+        return task_metrics + response_type_metrics
 
     @classproperty
     def RESPONSE_TYPE(cls) -> ResponseType:

{eval_framework-0.10.4 → eval_framework-0.10.6}/src/eval_framework/tasks/benchmarks/humaneval.py

@@ -114,7 +114,7 @@ class HumanEvalBPB_V2(HumanEvalBPB):
     NAME = "Human Eval BPB V2"
 
     def _get_ground_truth(self, item: dict[str, Any]) -> str | None:
-        return item["canonical_solution"]
+        return item["canonical_solution"].rstrip() + "\n```"
 
     def _get_instruction_text(self, item: dict[str, Any]) -> str:
         return "```python\n" + item["prompt"].rstrip() + "\n"
@@ -141,10 +141,10 @@ class HumanEval_OLMES(HumanEval):
         self.max_tokens = 1024
 
     def _get_instruction_text(self, item: dict[str, Any]) -> str:
-        return "```python\n" + item["prompt"].rstrip()
+        return "```python\n" + item["prompt"]
 
     def _get_fewshot_target_text(self, item: dict[str, Any]) -> str:
-        return "\n" + item["canonical_solution"] + "```"
+        return item["canonical_solution"] + "```"
 
 
 class HumanEvalInstruct(HumanEval):
@@ -168,11 +168,11 @@ class HumanEvalInstruct(HumanEval):
         return self.CUE_PREFIX + item["prompt"].lstrip()
 
 
-class HumanEval_NL(HumanEval_OLMES):
-    NAME = "Human Eval Newline OLMES"
+class HumanEval_OLMES_V2(HumanEval_OLMES):
+    NAME = "Human Eval Olmes v2"
 
     def _get_instruction_text(self, item: dict[str, Any]) -> str:
         return "```python\n" + item["prompt"].rstrip() + "\n"
 
     def _get_fewshot_target_text(self, item: dict[str, Any]) -> str:
-        return item["canonical_solution"].lstrip("\n") + "```"
+        return item["canonical_solution"].rstrip() + "\n```"