eval-framework 0.10.0__tar.gz → 0.10.6__tar.gz

{eval_framework-0.10.0 → eval_framework-0.10.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.10.0
+Version: 0.10.6
 Summary: Evaluation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License
@@ -218,7 +218,7 @@ Requires-Dist: xmltodict>=1.0.4,<1.1
 Requires-Dist: pydantic>=2.13.4,<3
 Requires-Dist: datasets>=5.0.1,<6
 Requires-Dist: pycountry>=26.2.16,<27
-Requires-Dist: nltk>=3.10.2,<4
+Requires-Dist: nltk>=3.10.3,<4
 Requires-Dist: python-dotenv>=1.2.2,<2
 Requires-Dist: lingua-language-detector>=2.2.0,<3
 Requires-Dist: google-crc32c>=1.8.0,<2
@@ -231,8 +231,8 @@ Requires-Dist: llm-sandbox[docker]==0.3.44
 Requires-Dist: jsonlines>=4,<5
 Requires-Dist: lxml>=6.1.1,<7
 Requires-Dist: python-iso639>=2026.7.23
-Requires-Dist: wandb>=0.28.1,<1
-Requires-Dist: boto3>=1.43.19,<2
+Requires-Dist: wandb>=0.28.2,<1
+Requires-Dist: boto3>=1.43.72,<2
 Requires-Dist: numpy>=2.5.2
 Requires-Dist: antlr4-python3-runtime==4.11.0
 Requires-Dist: scipy>=1.18.0,<2
@@ -241,7 +241,7 @@ Requires-Dist: eval-framework[determined,api,openai,transformers,accelerate,opti
 Requires-Dist: aleph-alpha-client>=11.5.1 ; extra == 'api'
 Requires-Dist: determined>=0.38.1,<0.39 ; extra == 'determined'
 Requires-Dist: tensorboard==2.21.0 ; extra == 'determined'
-Requires-Dist: openai>=2.53.0,<3 ; extra == 'openai'
+Requires-Dist: openai>=3.1.0,<4 ; extra == 'openai'
 Requires-Dist: tiktoken>=0.13.0,<1 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'optional'
@@ -315,8 +315,6 @@ There are optional extras available to unlock specific features of the library:
 
 As a short hand, the `all` extra installs all of the above.
 
-For local vLLM inference, see the sibling [`local-vllm-server/`](./local-vllm-server/README.md) package.
-
 We use `uv` to better resolve dependencies when downloading the extras. You can install uv with:
 ```bash
 curl -LsSf https://astral.sh/uv/install.sh | sh

{eval_framework-0.10.0 → eval_framework-0.10.6}/README.md

@@ -53,8 +53,6 @@ There are optional extras available to unlock specific features of the library:
 
 As a short hand, the `all` extra installs all of the above.
 
-For local vLLM inference, see the sibling [`local-vllm-server/`](./local-vllm-server/README.md) package.
-
 We use `uv` to better resolve dependencies when downloading the extras. You can install uv with:
 ```bash
 curl -LsSf https://astral.sh/uv/install.sh | sh

eval_framework-0.10.6/pyproject.toml

@@ -0,0 +1,162 @@
+[project]
+name = "eval-framework"
+version = "0.10.6"
+description = "Evaluation Framework"
+readme = "README.md"
+requires-python = ">=3.12,<3.15"
+classifiers = [
+    "Operating System :: OS Independent",
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+    "Programming Language :: Python :: 3 :: Only",
+    "Topic :: Software Development :: Libraries",
+    "Typing :: Typed",
+]
+dependencies = [
+    "pyyaml>=6.0.3,<7",
+    "xmltodict>=1.0.4,<1.1",
+    "pydantic>=2.13.4,<3",
+    "datasets>=5.0.1,<6",
+    "pycountry>=26.2.16,<27",
+    "nltk>=3.10.3,<4",
+    "python-dotenv>=1.2.2,<2",
+    "lingua-language-detector>=2.2.0,<3",
+    "google-crc32c>=1.8.0,<2",
+    "langdetect>=1.0.9,<2",
+    "jsonschema>=4.26.0,<5",
+    "mysql-connector-python>=26.7.0,<27",
+    "psycopg2-binary>=2.9.12,<3",
+    "sympy>=1.14.0,<2",
+    "llm-sandbox[docker]==0.3.44",
+    "jsonlines>=4,<5",
+    "lxml>=6.1.1,<7",
+    "python-iso639>=2026.7.23",
+    "wandb>=0.28.2,<1",
+    "boto3>=1.43.72,<2",
+    "numpy>=2.5.2",
+    "antlr4-python3-runtime==4.11.0",
+    "scipy>=1.18.0,<2",
+]
+
+[project.license]
+file = "LICENSE"
+
+[[project.authors]]
+name = "Aleph Alpha Research"
+
+[project.optional-dependencies]
+determined = [
+    "determined>=0.38.1,<0.39",
+    "tensorboard==2.21.0",
+]
+api = ["aleph-alpha-client>=11.5.1"]
+openai = [
+    "openai>=3.1.0,<4",
+    "tiktoken>=0.13.0,<1",
+    "transformers>=4.45.2,<5",
+]
+transformers = [
+    "transformers>=4.45.2,<5",
+    "torch>=2.13.0,<3",
+    "accelerate>=1.14.0,<2",
+]
+accelerate = ["accelerate"]
+optional = [
+    "transformers>=4.45.2,<5",
+    "jinja2>=3.1.6,<4",
+]
+all = ["eval_framework[determined,api,openai,transformers,accelerate,optional]"]
+
+[project.urls]
+repository = "https://github.com/Aleph-Alpha-Research/eval-framework"
+
+[project.scripts]
+eval_framework = "eval_framework.run:run"
+
+[dependency-groups]
+dev = [
+    "mypy>=2.3.1,<3",
+    "pytest>=9.1.1,<10",
+    "pytest-mock>=3.15.1",
+    "pytest-xdist>=3.8.0,<4",
+    "pytest-sugar>1.1,<2",
+    "types-pyyaml>=6.0.12.20260815,<7",
+    "types-python-dateutil>=2.9.0.20260807,<3",
+    "types-requests>=2.33.0.20260712,<3",
+    "plotly>=6.9.0,<7",
+    "ruff>=0.16.3",
+    "pip-licenses>=5.5.5",
+]
+flash-attn = [
+    "flash-attn>=2.8.3.post1,<2.9",
+    "torch",
+]
+
+[build-system]
+requires = ["uv_build>=0.12.5,<0.12.6"]
+build-backend = "uv_build"
+
+[tool.uv]
+override-dependencies = ["requests>=2.32,<3"]
+
+[tool.uv.build-backend]
+module-name = [
+    "eval_framework",
+    "template_formatting",
+]
+
+[[tool.uv.extra-build-dependencies.flash-attn]]
+requirement = "torch"
+match-runtime = true
+
+[tool.ruff]
+line-length = 120
+
+[tool.ruff.lint]
+select = [
+    "E",
+    "F",
+    "UP",
+    "I",
+]
+
+[tool.ruff.lint.isort]
+known-third-party = ["wandb"]
+
+[tool.ruff.lint.extend-per-file-ignores]
+"__init__.py" = ["F401"]
+"tests/tests_eval_framework/tasks/benchmarks/test_csqa_ellamind.py" = ["E501"]
+"tests/tests_eval_framework/tasks/benchmarks/test_gpqa_ellamind.py" = ["E501"]
+"tests/tests_eval_framework/tasks/benchmarks/test_gsm8k_ellamind.py" = ["E501"]
+"tests/tests_eval_framework/tasks/benchmarks/test_hendrycks_math_ellamind.py" = ["E501"]
+"tests/tests_eval_framework/tasks/benchmarks/test_hle_ellamind.py" = ["E501"]
+"tests/tests_eval_framework/tasks/benchmarks/test_mbpp_ellamind.py" = ["E501"]
+"tests/tests_eval_framework/tasks/benchmarks/test_piqa_ellamind.py" = ["E501"]
+"tests/tests_eval_framework/tasks/benchmarks/test_siqa_ellamind.py" = ["E501"]
+"tests/tests_eval_framework/tasks/benchmarks/test_winogrande_ellamind.py" = ["E501"]
+
+[tool.mypy]
+plugins = "pydantic.mypy"
+disallow_untyped_defs = true
+ignore_missing_imports = true
+files = [
+    "src",
+    "utils",
+]
+
+[tool.pytest.ini_options]
+testpaths = ["./tests"]
+minversion = 8.0
+addopts = "-p 'no:legacypath' --doctest-modules -n auto"
+markers = [
+    "gpu: needs a GPU runner, otherwise test can not be run",
+    "cpu_slow: runs for a long time (on CPU)",
+    "slow_download: smoke tests that download large datasets (>15s); excluded from CI, run manually with -m slow_download",
+    "external_api: needs external services for execution",
+    "formatter_hash: formatter consistency tests using hash comparisons",
+]
+filterwarnings = ["ignore::DeprecationWarning:datasets.utils._dill:"]
+env = ["WANDB_MODE = disabled"]

eval_framework-0.10.0/pyproject.toml → eval_framework-0.10.6/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.10.0"
+version = "0.10.6"
 description = "Evaluation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -25,7 +25,7 @@ dependencies = [
   "pydantic>=2.13.4,<3",
   "datasets>=5.0.1,<6",
   "pycountry>=26.2.16,<27",
-  "nltk>=3.10.2,<4",
+  "nltk>=3.10.3,<4",
   "python-dotenv>=1.2.2,<2",
   "lingua-language-detector>=2.2.0,<3",
   "google-crc32c>=1.8.0,<2",
@@ -38,8 +38,8 @@ dependencies = [
   "jsonlines>=4,<5",
   "lxml>=6.1.1,<7",
   "python-iso639>=2026.7.23",
-  "wandb>=0.28.1,<1",
-  "boto3>=1.43.19,<2",
+  "wandb>=0.28.2,<1",
+  "boto3>=1.43.72,<2",
   "numpy>=2.5.2",
   # is a dependency of sympy, but not explicitly listed in the requirements.txt
   # https://github.com/sympy/sympy/blob/0204fa34e8f6f6f8ccb4de01209be9a2345c9d6e/doc/src/contributing/dependencies.md?plain=1#L125
@@ -55,7 +55,7 @@ determined = [
 ]
 api = ["aleph-alpha-client>=11.5.1"]
 openai = [
-  "openai>=2.53.0,<3",
+  "openai>=3.1.0,<4",
   "tiktoken>=0.13.0,<1",
   "transformers>=4.45.2,<5",
 ]
@@ -84,16 +84,16 @@ eval_framework = "eval_framework.run:run"
 
 [dependency-groups]
 dev = [
-  "mypy>=2.3.0,<3",
+  "mypy>=2.3.1,<3",
   "pytest>=9.1.1,<10",
   "pytest-mock>=3.15.1",
   "pytest-xdist>=3.8.0,<4",
   "pytest-sugar>1.1,<2",
-  "types-pyyaml>=6.0.12.20260724,<7",
+  "types-pyyaml>=6.0.12.20260815,<7",
   "types-python-dateutil>=2.9.0.20260807,<3",
   "types-requests>=2.33.0.20260712,<3",
   "plotly>=6.9.0,<7",
-  "ruff>=0.16.2",
+  "ruff>=0.16.3",
   "pip-licenses>=5.5.5",
 ]
 flash-attn = [
@@ -102,7 +102,7 @@ flash-attn = [
 ]
 
 [build-system]
-requires = ["uv_build>=0.11.32,<0.11.33"]
+requires = ["uv_build>=0.12.5,<0.12.6"]
 build-backend = "uv_build"
 
 [tool.uv.build-backend]

{eval_framework-0.10.0 → eval_framework-0.10.6}/src/eval_framework/evaluation_generator.py

@@ -7,16 +7,9 @@ import wandb
 from tqdm import tqdm
 
 from eval_framework.metrics.base import BaseMetric
-from eval_framework.metrics.efficiency.bytes_per_sequence_position import (
-    BytesCompletion,
-    BytesLoglikelihood,
-    SequencePositionsCompletion,
-    SequencePositionsLoglikelihood,
-)
 from eval_framework.metrics.llm.base import BaseLLMJudgeMetric
 from eval_framework.result_processors.base import Result, ResultProcessor
 from eval_framework.shared.types import Completion, Loglikelihood
-from eval_framework.tasks.base import ResponseType
 from eval_framework.tasks.eval_config import EvalConfig
 from eval_framework.tasks.registry import registry
 from eval_framework.utils.constants import RED, RESET
@@ -37,16 +30,7 @@ class EvaluationGenerator:
         self.save_intermediate_results = config.save_intermediate_results
 
         eval_ = registry()[config.task_name]
-        response_type = eval_.response_type()
-        task_metrics = eval_.metrics()
-
-        if response_type == ResponseType.COMPLETION:
-            self.metrics = task_metrics + [BytesCompletion, SequencePositionsCompletion]
-        elif response_type == ResponseType.LOGLIKELIHOODS:
-            self.metrics = task_metrics + [BytesLoglikelihood, SequencePositionsLoglikelihood]
-        else:
-            raise NotImplementedError
-
+        self.metrics = eval_.metrics()
         self.task_name = eval_.display_name()
 
     def _run_metric_calculators(self, responses: list[Completion | Loglikelihood]) -> list[Result]:

{eval_framework-0.10.0 → eval_framework-0.10.6}/src/eval_framework/llm/aleph_alpha.py

@@ -165,33 +165,33 @@ class AlephAlphaAPIModel(BaseLLM):
         if isinstance(response, Error):
             return RawCompletion(
                 prompt=prompt,
-                prompt_sequence_positions=None,
+                prompt_num_tokens=None,
                 completion="",
-                completion_sequence_positions=0,
+                completion_num_tokens=0,
                 raw_completion_error=response,
             )
 
         assert len(response.completions) == 1
         completion = response.completions[0].completion or ""
-        prompt_sequence_positions: int | None = None
-        completion_sequence_positions: int | None = None
+        prompt_num_tokens: int | None = None
+        completion_num_tokens: int | None = None
 
         # Support workaround in api-worker-transformer's scaling generator to return the correct number of tokens.
         # These are part of the completion string; those in CompletionResponse are invalid in this case.
         m = re.match(r"\uf8c9(\d+),(\d+)\uf8c9(.*)", completion, re.DOTALL)
         if m is not None:
             num_input_tokens, num_completion_tokens, completion = m.groups()
-            prompt_sequence_positions = int(num_input_tokens)
-            completion_sequence_positions = int(num_completion_tokens)
+            prompt_num_tokens = int(num_input_tokens)
+            completion_num_tokens = int(num_completion_tokens)
         else:
-            prompt_sequence_positions = response.num_tokens_prompt_total if response else None
-            completion_sequence_positions = response.num_tokens_generated if response else None
+            prompt_num_tokens = response.num_tokens_prompt_total if response else None
+            completion_num_tokens = response.num_tokens_generated if response else None
 
         return RawCompletion(
             prompt=prompt,
-            prompt_sequence_positions=prompt_sequence_positions,
+            prompt_num_tokens=prompt_num_tokens,
             completion=completion,
-            completion_sequence_positions=completion_sequence_positions,
+            completion_num_tokens=completion_num_tokens,
         )
 
     def generate_from_messages(
@@ -254,8 +254,8 @@ class AlephAlphaAPIModel(BaseLLM):
         results: list[RawLoglikelihood] = []
         for sample_idx, (sample, prompt) in enumerate(zip(samples, prompts, strict=True)):
             choices_log_probs: dict[str, float] = {}
-            choices_sequence_positions: dict[str, int] = {}
-            prompt_sequence_positions: int | None = 0
+            choices_num_tokens: dict[str, int] = {}
+            prompt_num_tokens: int | None = 0
             number_of_initial_choices_tokens: int | None = None
             error: Error | None = None
 
@@ -267,9 +267,9 @@ class AlephAlphaAPIModel(BaseLLM):
 
                 if isinstance(response, Error):
                     error = response
-                    prompt_sequence_positions = None
+                    prompt_num_tokens = None
                     choices_log_probs = {}
-                    choices_sequence_positions = {}
+                    choices_num_tokens = {}
                 else:
                     try:
                         logprob, choice_token_count = self._extract_choice_logprob_from_completion(
@@ -278,7 +278,7 @@ class AlephAlphaAPIModel(BaseLLM):
                             response=response,
                         )
                         choices_log_probs[choice] = logprob
-                        choices_sequence_positions[choice] = choice_token_count
+                        choices_num_tokens[choice] = choice_token_count
                         if number_of_initial_choices_tokens is None:
                             number_of_initial_choices_tokens = choice_token_count
 
@@ -294,16 +294,16 @@ class AlephAlphaAPIModel(BaseLLM):
                             message=str(exc),
                             traceback=traceback.format_exc(),
                         )
-                        prompt_sequence_positions = None
+                        prompt_num_tokens = None
                         choices_log_probs = {}
-                        choices_sequence_positions = {}
+                        choices_num_tokens = {}
 
             results.append(
                 RawLoglikelihood(
                     prompt=prompt,
-                    prompt_sequence_positions=prompt_sequence_positions,
+                    prompt_num_tokens=prompt_num_tokens,
                     loglikelihoods=choices_log_probs,
-                    loglikelihoods_sequence_positions=choices_sequence_positions,
+                    loglikelihoods_num_tokens=choices_num_tokens,
                     raw_loglikelihood_error=error,
                 )
             )

{eval_framework-0.10.0 → eval_framework-0.10.6}/src/eval_framework/llm/huggingface.py

@@ -196,9 +196,9 @@ class BaseHFLLM(BaseLLM):
                 raw_completions.append(
                     RawCompletion(
                         prompt=prompt,
-                        prompt_sequence_positions=prompt_token_count,
+                        prompt_num_tokens=prompt_token_count,
                         completion="",
-                        completion_sequence_positions=0,
+                        completion_num_tokens=0,
                         raw_completion_error=Error(
                             error_class=PromptTooLongException.__name__,
                             message="Prompt exceeded context size.",
@@ -225,12 +225,12 @@ class BaseHFLLM(BaseLLM):
             raw_completions.append(
                 RawCompletion(
                     prompt=prompt,
-                    prompt_sequence_positions=prompt_token_count,
+                    prompt_num_tokens=prompt_token_count,
                     concat_compression=ConcatCompression.calculate(
                         single_messages, count_tokens=self.count_tokens, completion=completion
                     ),
                     completion=completion,
-                    completion_sequence_positions=completion_token_count,
+                    completion_num_tokens=completion_token_count,
                 )
             )
         return raw_completions
@@ -251,7 +251,7 @@ class BaseHFLLM(BaseLLM):
             # format
             prompt = self._formatter.format(sample.messages, output_mode="string")
             choices_log_probs: dict[str, float] = {}
-            choices_log_probs_sequence_positions: dict[str, float] = {}
+            choices_log_probs_num_tokens: dict[str, float] = {}
             error: Error | None = None
 
             for choice in sample.possible_completions or []:
@@ -266,7 +266,7 @@ class BaseHFLLM(BaseLLM):
                     if raise_errors():
                         raise PromptTooLongException("Prompt exceeded context size.")
                     choices_log_probs = {}
-                    choices_log_probs_sequence_positions = {}
+                    choices_log_probs_num_tokens = {}
                     error = Error(
                         error_class=PromptTooLongException.__name__,
                         message="Prompt and choice exceeded context size.",
@@ -278,17 +278,17 @@ class BaseHFLLM(BaseLLM):
                     sum_log_probs = self._model_log_probs(prompt_and_choice, num_choice_tokens)
 
                 choices_log_probs.update({choice: sum_log_probs})
-                choices_log_probs_sequence_positions.update({choice: num_choice_tokens})
+                choices_log_probs_num_tokens.update({choice: num_choice_tokens})
 
             results.append(
                 RawLoglikelihood(
                     prompt=prompt,
-                    prompt_sequence_positions=len(self.tokenizer.encode(prompt, add_special_tokens=False)),
+                    prompt_num_tokens=len(self.tokenizer.encode(prompt, add_special_tokens=False)),
                     concat_compression=ConcatCompression.calculate(
                         sample.messages, count_tokens=self.count_tokens, choices=sample.possible_completions
                     ),
                     loglikelihoods=choices_log_probs,
-                    loglikelihoods_sequence_positions=choices_log_probs_sequence_positions,
+                    loglikelihoods_num_tokens=choices_log_probs_num_tokens,
                     raw_loglikelihood_error=error,
                 )
             )

{eval_framework-0.10.0 → eval_framework-0.10.6}/src/eval_framework/llm/openai.py

@@ -186,7 +186,7 @@ class OpenAIModel(BaseLLM):
                 completion_tokens = getattr(usage, "completion_tokens", None) if usage is not None else None
                 return RawCompletion(
                     prompt=prompt,
-                    prompt_sequence_positions=(
+                    prompt_num_tokens=(
                         prompt_tokens
                         if prompt_tokens is not None
                         else (self._count_tokens(prompt) if self._encoder is not None else None)
@@ -201,7 +201,7 @@ class OpenAIModel(BaseLLM):
                         else None
                     ),
                     completion=completion,
-                    completion_sequence_positions=(
+                    completion_num_tokens=(
                         completion_tokens
                         if completion_tokens is not None
                         else (self._count_tokens(completion) if self._encoder is not None else None)
@@ -223,10 +223,12 @@ class OpenAIModel(BaseLLM):
                 prompt = "\n".join([f"{m.get('role', '')}: {m.get('content', '')}" for m in chat_messages])
                 prompt_tokens = getattr(chat_response.usage, "prompt_tokens", None)
                 completion_tokens = getattr(chat_response.usage, "completion_tokens", None)
+                completion_details = getattr(chat_response.usage, "completion_tokens_details", None)
+                reasoning_tokens = getattr(completion_details, "reasoning_tokens", None)
                 completion = chat_response.choices[0].message.content or ""
                 return RawCompletion(
                     prompt=prompt,
-                    prompt_sequence_positions=prompt_tokens,
+                    prompt_num_tokens=prompt_tokens,
                     concat_compression=(
                         ConcatCompression.calculate(
                             single_messages,
@@ -237,11 +239,12 @@ class OpenAIModel(BaseLLM):
                         else None
                     ),
                     completion=completion,
-                    completion_sequence_positions=(
+                    completion_num_tokens=(
                         completion_tokens
                         if completion_tokens is not None
                         else (self._count_tokens(completion) if self._encoder is not None else None)
                     ),
+                    reasoning_num_tokens=reasoning_tokens,
                 )
 
         with concurrent.futures.ThreadPoolExecutor() as executor:
@@ -276,8 +279,8 @@ class OpenAIModel(BaseLLM):
         for sample in samples:
             prompt = self._formatter.format(sample.messages, output_mode="string") if sample.messages else ""
             choices_log_probs: dict[str, float] = {}
-            choices_sequence_positions: dict[str, int] = {}
-            prompt_sequence_positions: int | None = self._count_tokens(prompt)
+            choices_num_tokens: dict[str, int] = {}
+            prompt_num_tokens: int | None = self._count_tokens(prompt)
             error: Error | None = None
 
             for choice in sample.possible_completions or []:
@@ -317,20 +320,20 @@ class OpenAIModel(BaseLLM):
 
                     # Sum logprobs for the completion portion
                     choices_log_probs[choice] = sum(all_logprobs[len(prompt_tokens) :])
-                    choices_sequence_positions[choice] = len(completion_tokens)
+                    choices_num_tokens[choice] = len(completion_tokens)
 
                 except Exception as e:
                     error = Error(error_class=e.__class__.__name__, message=str(e), traceback=traceback.format_exc())
-                    prompt_sequence_positions = None
+                    prompt_num_tokens = None
                     choices_log_probs = {}
-                    choices_sequence_positions = {}
+                    choices_num_tokens = {}
 
             results.append(
                 RawLoglikelihood(
                     prompt=prompt,
-                    prompt_sequence_positions=prompt_sequence_positions,
+                    prompt_num_tokens=prompt_num_tokens,
                     loglikelihoods=choices_log_probs,
-                    loglikelihoods_sequence_positions=choices_sequence_positions,
+                    loglikelihoods_num_tokens=choices_num_tokens,
                     raw_loglikelihood_error=error,
                 )
             )

eval_framework-0.10.6/src/eval_framework/metrics/efficiency/token_counters.py

@@ -0,0 +1,38 @@
+from eval_framework.metrics.base import BaseMetric, MetricResult
+from eval_framework.shared.types import Completion
+
+
+class TokenCounts(BaseMetric[Completion]):
+    """Number of tokens the model generated for the completion, and how many of
+    those were spent on reasoning (thinking).
+
+    Reads the token counts backends already attach to the response, so no extra
+    tokenisation is performed. Each value independently falls back to None when
+    the backend did not report that particular count or when the sample errored.
+    Reasoning counts are only exposed by some backends (e.g. OpenAI reasoning
+    models via `usage.completion_tokens_details.reasoning_tokens`, or vLLM
+    started with `--reasoning-parser`); non-reasoning models and backends that
+    do not surface a per-response count return None for that key.
+    """
+
+    NAME = "TokenCounts"
+    KEYS = ["Completion", "Reasoning"]
+
+    def calculate(self, response: Completion) -> list[MetricResult]:
+        if response.error:
+            values: dict[str, float | None] = {"Completion": None, "Reasoning": None}
+        else:
+            values = {
+                "Completion": response.raw_completion_num_tokens,
+                "Reasoning": response.raw_completion_reasoning_num_tokens,
+            }
+
+        return [
+            MetricResult(
+                metric_name=f"{self.NAME}/{key}",
+                value=value,
+                higher_is_better=False,
+                error=response.error,
+            )
+            for key, value in values.items()
+        ]

{eval_framework-0.10.0 → eval_framework-0.10.6}/src/eval_framework/response_generator.py

@@ -110,9 +110,9 @@ class ResponseGenerator:
             raw_loglikelihoods = [
                 RawLoglikelihood(
                     prompt="",
-                    prompt_sequence_positions=0,
+                    prompt_num_tokens=0,
                     loglikelihoods={},
-                    loglikelihoods_sequence_positions={},
+                    loglikelihoods_num_tokens={},
                     raw_loglikelihood_error=Error(
                         error_class=e.__class__.__name__,
                         message=str(e),
@@ -132,10 +132,10 @@ class ResponseGenerator:
                     subject=sample.subject,
                     ground_truth=sample.ground_truth,
                     prompt=raw_loglikelihood.prompt,
-                    prompt_sequence_positions=raw_loglikelihood.prompt_sequence_positions,
+                    prompt_num_tokens=raw_loglikelihood.prompt_num_tokens,
                     concat_compression=raw_loglikelihood.concat_compression,
                     loglikelihoods=raw_loglikelihood.loglikelihoods,
-                    loglikelihoods_sequence_positions=raw_loglikelihood.loglikelihoods_sequence_positions,
+                    loglikelihoods_num_tokens=raw_loglikelihood.loglikelihoods_num_tokens,
                     error=raw_loglikelihood.raw_loglikelihood_error,
                 )
             )