eval-framework 0.11.6__tar.gz → 0.11.8__tar.gz

{eval_framework-0.11.6 → eval_framework-0.11.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eval-framework
-Version: 0.11.6
+Version: 0.11.8
 Summary: Evaluation Framework
 Author: Aleph Alpha Research
 License:                                  Apache License
@@ -229,10 +229,10 @@ Requires-Dist: psycopg2-binary>=2.9.12,<3
 Requires-Dist: sympy>=1.14.0,<2
 Requires-Dist: llm-sandbox[docker]==0.3.44
 Requires-Dist: jsonlines>=4,<5
-Requires-Dist: lxml>=6.1.2,<7
+Requires-Dist: lxml>=6.1.3,<7
 Requires-Dist: python-iso639>=2026.7.23
 Requires-Dist: wandb>=0.29.0,<1
-Requires-Dist: boto3>=1.43.85,<2
+Requires-Dist: boto3>=1.43.87,<2
 Requires-Dist: numpy>=2.5.2
 Requires-Dist: antlr4-python3-runtime==4.11.0
 Requires-Dist: scipy>=1.18.1,<2
@@ -241,13 +241,13 @@ Requires-Dist: eval-framework[determined,api,openai,transformers,accelerate,opti
 Requires-Dist: aleph-alpha-client>=11.5.1 ; extra == 'api'
 Requires-Dist: determined>=0.38.1,<0.39 ; extra == 'determined'
 Requires-Dist: tensorboard==2.21.0 ; extra == 'determined'
-Requires-Dist: openai>=3.6.0,<4 ; extra == 'openai'
+Requires-Dist: openai>=3.7.0,<4 ; extra == 'openai'
 Requires-Dist: tiktoken>=0.14.0,<1 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'openai'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'optional'
 Requires-Dist: jinja2>=3.1.6,<4 ; extra == 'optional'
 Requires-Dist: transformers>=4.45.2,<5 ; extra == 'transformers'
-Requires-Dist: torch>=2.13.0,<3 ; extra == 'transformers'
+Requires-Dist: torch>=2.14.0,<3 ; extra == 'transformers'
 Requires-Dist: accelerate>=1.14.0,<2 ; extra == 'transformers'
 Requires-Python: >=3.12, <3.15
 Project-URL: repository, https://github.com/Aleph-Alpha-Research/eval-framework

{eval_framework-0.11.6 → eval_framework-0.11.8}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.11.6"
+version = "0.11.8"
 description = "Evaluation Framework"
 readme = "README.md"
 requires-python = ">=3.12,<3.15"
@@ -32,10 +32,10 @@ dependencies = [
     "sympy>=1.14.0,<2",
     "llm-sandbox[docker]==0.3.44",
     "jsonlines>=4,<5",
-    "lxml>=6.1.2,<7",
+    "lxml>=6.1.3,<7",
     "python-iso639>=2026.7.23",
     "wandb>=0.29.0,<1",
-    "boto3>=1.43.85,<2",
+    "boto3>=1.43.87,<2",
     "numpy>=2.5.2",
     "antlr4-python3-runtime==4.11.0",
     "scipy>=1.18.1,<2",
@@ -54,13 +54,13 @@ determined = [
 ]
 api = ["aleph-alpha-client>=11.5.1"]
 openai = [
-    "openai>=3.6.0,<4",
+    "openai>=3.7.0,<4",
     "tiktoken>=0.14.0,<1",
     "transformers>=4.45.2,<5",
 ]
 transformers = [
     "transformers>=4.45.2,<5",
-    "torch>=2.13.0,<3",
+    "torch>=2.14.0,<3",
     "accelerate>=1.14.0,<2",
 ]
 accelerate = ["accelerate"]
@@ -96,7 +96,7 @@ flash-attn = [
 ]
 
 [build-system]
-requires = ["uv_build>=0.12.8,<0.12.9"]
+requires = ["uv_build>=0.12.9,<0.12.10"]
 build-backend = "uv_build"
 
 [tool.uv]

{eval_framework-0.11.6 → eval_framework-0.11.8}/pyproject.toml.orig

@@ -1,6 +1,6 @@
 [project]
 name = "eval-framework"
-version = "0.11.6"
+version = "0.11.8"
 description = "Evaluation Framework"
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -36,10 +36,10 @@ dependencies = [
   "sympy>=1.14.0,<2",
   "llm-sandbox[docker]==0.3.44",
   "jsonlines>=4,<5",
-  "lxml>=6.1.2,<7",
+  "lxml>=6.1.3,<7",
   "python-iso639>=2026.7.23",
   "wandb>=0.29.0,<1",
-  "boto3>=1.43.85,<2",
+  "boto3>=1.43.87,<2",
   "numpy>=2.5.2",
   # is a dependency of sympy, but not explicitly listed in the requirements.txt
   # https://github.com/sympy/sympy/blob/0204fa34e8f6f6f8ccb4de01209be9a2345c9d6e/doc/src/contributing/dependencies.md?plain=1#L125
@@ -55,13 +55,13 @@ determined = [
 ]
 api = ["aleph-alpha-client>=11.5.1"]
 openai = [
-  "openai>=3.6.0,<4",
+  "openai>=3.7.0,<4",
   "tiktoken>=0.14.0,<1",
   "transformers>=4.45.2,<5",
 ]
 transformers = [
   "transformers>=4.45.2,<5",
-  "torch>=2.13.0,<3",
+  "torch>=2.14.0,<3",
   "accelerate>=1.14.0,<2",
 ]
 accelerate = ["accelerate"]
@@ -102,7 +102,7 @@ flash-attn = [
 ]
 
 [build-system]
-requires = ["uv_build>=0.12.8,<0.12.9"]
+requires = ["uv_build>=0.12.9,<0.12.10"]
 build-backend = "uv_build"
 
 [tool.uv.build-backend]

{eval_framework-0.11.6 → eval_framework-0.11.8}/src/eval_framework/evaluation_generator.py

@@ -136,9 +136,12 @@ class EvaluationGenerator:
             error_free_ratio = float(len(data_subset_error_free) / total_count)
             aggregated_results[f"ErrorFreeRatio {metric}"] = error_free_ratio
 
+            # Default average is weighted by key and subject (macro average).
             # aggregate by key and subject first to have equal weights for all key / subject combinations
             key_subject_mean = data_subset_error_free.groupby(["key", "subject"]).mean()
             aggregated_results[f"Average {metric}"] = float(key_subject_mean[["value"]].mean()["value"])
+            # Additionally, report the plain mean over all samples (micro average or per-sample average).
+            aggregated_results[f"Average {metric} (micro)"] = float(data_subset_error_free["value"].mean())
 
             if error_free_ratio < 1.0:
                 # Treat error samples (with value=None) as 0 for the "including errors" average
@@ -274,12 +277,15 @@ class EvaluationGenerator:
                 raise ValueError(f"Metric {metric_name} not found in metrics list")
 
             for aggregator in current_metric.AGGREGATORS:
+                # Compute the aggregator per problem (collapsing the repeats of each problem into one score).
+                per_problem = aggregator(metric_group, ["subject", "problem"])
+                # Macro average: mean per key/subject group, then mean over groups, giving equal weight to every group.
                 aggregated_results[f"{aggregator.name} {current_metric_class}.{metric_name}"] = (
-                    aggregator(metric_group, ["subject", "problem"])  # Compute the aggregator per problem...
-                    .groupby(["key", "subject"])  # ... then group by key, subject...
-                    .agg({"value": "mean"})["value"]  # ...and average scores over each key, subject group...
-                    .mean()  # ...and lastly average the scores across all groups giving equal weight to every
-                    .item()  # key, subject group.
+                    per_problem.groupby(["key", "subject"]).agg({"value": "mean"})["value"].mean().item()
+                )
+                # Micro average: plain mean over all problems.
+                aggregated_results[f"{aggregator.name} {current_metric_class}.{metric_name} (micro)"] = (
+                    per_problem["value"].mean().item()
                 )
 
         # Loop to additionally compute per-subject/per-key breakdown metric scores, e.g. for only subject="algebra"