ai2-olmo-eval 0.7.1__py3-none-any.whl → 0.7.2__py3-none-any.whl

{ai2_olmo_eval-0.7.1.dist-info → ai2_olmo_eval-0.7.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ai2-olmo-eval
-Version: 0.7.1
+Version: 0.7.2
 Summary: In-loop evaluation tasks for language modeling
 Author-email: Allen Institute for Artificial Intelligence <olmo@allenai.org>
 License:                                  Apache License

{ai2_olmo_eval-0.7.1.dist-info → ai2_olmo_eval-0.7.2.dist-info}/RECORD

@@ -1,10 +1,10 @@
-ai2_olmo_eval-0.7.1.dist-info/licenses/LICENSE,sha256=YvuKOpYh3COIF0yqq-nCMXtpS7mh1GyYvPVlW2j1G-M,11359
+ai2_olmo_eval-0.7.2.dist-info/licenses/LICENSE,sha256=YvuKOpYh3COIF0yqq-nCMXtpS7mh1GyYvPVlW2j1G-M,11359
 olmo_eval/__init__.py,sha256=49RxnAaJNk8U9XP3SF5MjyFIxLSkxH0vXQuZgnEOi44,283
 olmo_eval/metrics.py,sha256=NcI_1B3BV-DC9RXjsSIftU-2GeF8vvU6SNyJnlYlKwU,18705
-olmo_eval/tasks.py,sha256=ro8h6qk56JSBjO_FVm6vnf5co2sxi8ak7WhJwT6HMhw,78384
+olmo_eval/tasks.py,sha256=QGLyF7JA2-T9mkh-N4cZGNOQp9si90yQSS41T3x5Lak,79630
 olmo_eval/tokenizer.py,sha256=PnkidE0nAtEA1QZjuQpE_bIwgAsHxodnaJRALAPqrJQ,5127
 olmo_eval/util.py,sha256=ARmZmRQl8VOvnKQoUprb3cOunzcApeNhRdV4BMXZuvo,3856
-olmo_eval/version.py,sha256=nns1NDKRJC67qXO0tgzazKqjYTd9JEBha9a4HkUXVAE,308
+olmo_eval/version.py,sha256=QWjPfx79C2NOQw2G7iDEsM4FKsLiGLCLNDzEx7EImf8,308
 olmo_eval/hf_datasets/ai2_arc/ARC-Challenge/validation/data-00000-of-00001.arrow,sha256=TPWbMhBmticWjYp7TA3etcKbXbaoCDBWhxuqlD1bDJA,98080
 olmo_eval/hf_datasets/ai2_arc/ARC-Challenge/validation/dataset_info.json,sha256=iZumP5Udu8LD7cbew3o7nNpnGu-o9jPaMxUrNDDNIVY,1795
 olmo_eval/hf_datasets/ai2_arc/ARC-Challenge/validation/state.json,sha256=6Q1XhM-HMZcymuGAKBC_8RjMBKgJSaR_6lLUO9Z8XwE,255
@@ -600,6 +600,18 @@ olmo_eval/oe_eval_tasks/arc_easy/val_mc_5shot/config.json,sha256=CEgPNm226vxmMim
 olmo_eval/oe_eval_tasks/arc_easy/val_mc_5shot/requests.jsonl.gz,sha256=LZ7XuWwDo6zJTqhgpZgHNj6yi-xOXb-TQxl9yxB9gVg,114271
 olmo_eval/oe_eval_tasks/arc_easy/val_rc_5shot/config.json,sha256=LeNP534voujfcp9ph8SKHfnfYPjfSu8ik3HWiXt3TFM,761
 olmo_eval/oe_eval_tasks/arc_easy/val_rc_5shot/requests.jsonl.gz,sha256=28UmHQnAB2DIlfYbqhuhJ4AjAVLDAHWWoEmaHlI-UKU,202290
+olmo_eval/oe_eval_tasks/basic_skills_arithmetic/rc_5shot/config.json,sha256=_gSH-miyIWms4r3TSLCMihc42v7kt8tEPnqQJcgux-4,616
+olmo_eval/oe_eval_tasks/basic_skills_arithmetic/rc_5shot/requests.jsonl.gz,sha256=iiVqzSVTiEk5lbq0WAiR8ujvBHHv73azRpvfuCIrEfI,215180
+olmo_eval/oe_eval_tasks/basic_skills_coding/rc_5shot/config.json,sha256=19NZFpCouu7oEidoUBthKUekW87pT5pzR1bX1NJV77g,592
+olmo_eval/oe_eval_tasks/basic_skills_coding/rc_5shot/requests.jsonl.gz,sha256=suXkEgQLUT-XK_EDyQKIoniNYNJvo4vUpe8-jyeNe-w,274302
+olmo_eval/oe_eval_tasks/basic_skills_common_knowledge/rc_5shot/config.json,sha256=DOHsmlO6_OMBIl-oEfKT8O0yIj89I1gTV_uvOxdiT8M,652
+olmo_eval/oe_eval_tasks/basic_skills_common_knowledge/rc_5shot/requests.jsonl.gz,sha256=ie673jV3ShxUhrqux3Y8YRNfAazKa8ayGEjo7hxEp1Y,237402
+olmo_eval/oe_eval_tasks/basic_skills_logical_reasoning/rc_5shot/config.json,sha256=OB20jgxj00v3bvfsc1M1zyWGlEJvZdXBlg4L9NeGsZY,658
+olmo_eval/oe_eval_tasks/basic_skills_logical_reasoning/rc_5shot/requests.jsonl.gz,sha256=5ElEBHtBq6tBQ1hqEbg9---XkUFV3GjcMGHFXxP_urs,284843
+olmo_eval/oe_eval_tasks/basic_skills_pattern/rc_5shot/config.json,sha256=OAZyUX7pw7cEguIsSbs_fKXiuHh1sbEkpF7x9v6ZI80,598
+olmo_eval/oe_eval_tasks/basic_skills_pattern/rc_5shot/requests.jsonl.gz,sha256=FY1pf-fTh5BNnN5H7uN0ksm21tdC6ewKsOhaOpN3760,71330
+olmo_eval/oe_eval_tasks/basic_skills_string_operations/rc_5shot/config.json,sha256=TfwWhRHC_G17uqk60-pNROMNzzmd0rMTY5nPP0dje00,658
+olmo_eval/oe_eval_tasks/basic_skills_string_operations/rc_5shot/requests.jsonl.gz,sha256=FIBdOQSDoQ99gDEpHYHYTmhW5qfClVP-rh3ll2I0fDQ,231341
 olmo_eval/oe_eval_tasks/boolq/mc_5shot/config.json,sha256=87GTyDGser1tWfSWmktZ1X17jKXU1EZzHOJLMSbVspA,632
 olmo_eval/oe_eval_tasks/boolq/mc_5shot/requests.jsonl.gz,sha256=uZ9ZkbFkiUn4XcCzypgPscTFTrVDexVC1L-e6zBiEMg,393249
 olmo_eval/oe_eval_tasks/boolq/rc_0shot/config.json,sha256=d1GKQMIX1cUgnZHlUe9kgAZsgkMc1N2GnMlyhccO9pE,509
@@ -704,7 +716,7 @@ olmo_eval/oe_eval_tasks/winogrande/val_rc_5shot/config.json,sha256=ySjEVqTOj5GwC
 olmo_eval/oe_eval_tasks/winogrande/val_rc_5shot/requests.jsonl.gz,sha256=knTzcqigWCfdYLN1Pl0TfCm0Fi1lRASWAo_SC6KtXsc,115262
 olmo_eval/tokenizers/allenai_eleuther-ai-gpt-neox-20b-pii-special.json,sha256=yjXYcnpTO7Zjm_R4Gucrn9oA5paadiYM-ZZER5q_EXc,2114319
 olmo_eval/tokenizers/allenai_gpt-neox-olmo-dolma-v1_5.json,sha256=mtM7Szmp-Dlzw_jEKgGUjdW4d6KKyaU1aVbE_07QtxQ,2115113
-ai2_olmo_eval-0.7.1.dist-info/METADATA,sha256=ZIqB1IUyLb3SLKORyR_X9aKPAmwLuygiUm-nhcepY6k,14398
-ai2_olmo_eval-0.7.1.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-ai2_olmo_eval-0.7.1.dist-info/top_level.txt,sha256=Pryk28JTb89-j624Uy1gRZiE0YXI3czgbNIfJCl9-x0,10
-ai2_olmo_eval-0.7.1.dist-info/RECORD,,
+ai2_olmo_eval-0.7.2.dist-info/METADATA,sha256=PKJfkoDu4hrLzb6NA1MDfXOjZnUxQ4WFpJouWU1Cr_4,14398
+ai2_olmo_eval-0.7.2.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
+ai2_olmo_eval-0.7.2.dist-info/top_level.txt,sha256=Pryk28JTb89-j624Uy1gRZiE0YXI3czgbNIfJCl9-x0,10
+ai2_olmo_eval-0.7.2.dist-info/RECORD,,

{ai2_olmo_eval-0.7.1.dist-info → ai2_olmo_eval-0.7.2.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (78.1.0)
+Generator: setuptools (80.7.1)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

olmo_eval/oe_eval_tasks/basic_skills_arithmetic/rc_5shot/config.json

@@ -0,0 +1,23 @@
+{
+  "task_name": "basic_skills_arithmetic",
+  "task_hash": "56711b967c78d896ef51ba00aef5cfb0",
+  "task_config": {
+    "dataset_path": "basic_skills_arithmetic",
+    "primary_metric": "acc_per_token",
+    "split": "validation",
+    "num_shots": 5,
+    "metadata": {
+      "regimes": [
+        "OLMES-v0.1"
+      ],
+      "alias": "basic_skills_arithmetic:rc::olmes"
+    },
+    "generation_kwargs": {},
+    "context_kwargs": {},
+    "dataset_name": "arithmetic",
+    "task_name": "basic_skills_arithmetic",
+    "version": 0,
+    "task_core": "basic_skills_arithmetic"
+  },
+  "current_date": "2025-05-12 00:06:28 UTC"
+}

olmo_eval/oe_eval_tasks/basic_skills_coding/rc_5shot/config.json

@@ -0,0 +1,23 @@
+{
+  "task_name": "basic_skills_coding",
+  "task_hash": "d748d1d8ba506d3d234eed529ef62c3e",
+  "task_config": {
+    "dataset_path": "basic_skills_coding",
+    "primary_metric": "acc_per_token",
+    "split": "validation",
+    "num_shots": 5,
+    "metadata": {
+      "regimes": [
+        "OLMES-v0.1"
+      ],
+      "alias": "basic_skills_coding:rc::olmes"
+    },
+    "generation_kwargs": {},
+    "context_kwargs": {},
+    "dataset_name": "coding",
+    "task_name": "basic_skills_coding",
+    "version": 0,
+    "task_core": "basic_skills_coding"
+  },
+  "current_date": "2025-05-12 00:06:28 UTC"
+}

olmo_eval/oe_eval_tasks/basic_skills_common_knowledge/rc_5shot/config.json

@@ -0,0 +1,23 @@
+{
+  "task_name": "basic_skills_common_knowledge",
+  "task_hash": "51e88e759602f9085a8c779da375d833",
+  "task_config": {
+    "dataset_path": "basic_skills_common_knowledge",
+    "primary_metric": "acc_per_token",
+    "split": "validation",
+    "num_shots": 5,
+    "metadata": {
+      "regimes": [
+        "OLMES-v0.1"
+      ],
+      "alias": "basic_skills_common_knowledge:rc::olmes"
+    },
+    "generation_kwargs": {},
+    "context_kwargs": {},
+    "dataset_name": "common_knowledge",
+    "task_name": "basic_skills_common_knowledge",
+    "version": 0,
+    "task_core": "basic_skills_common_knowledge"
+  },
+  "current_date": "2025-05-12 00:06:28 UTC"
+}

olmo_eval/oe_eval_tasks/basic_skills_logical_reasoning/rc_5shot/config.json

@@ -0,0 +1,23 @@
+{
+  "task_name": "basic_skills_logical_reasoning",
+  "task_hash": "a3d406a2f4224604b7e6bbf68050691d",
+  "task_config": {
+    "dataset_path": "basic_skills_logical_reasoning",
+    "primary_metric": "acc_per_token",
+    "split": "validation",
+    "num_shots": 5,
+    "metadata": {
+      "regimes": [
+        "OLMES-v0.1"
+      ],
+      "alias": "basic_skills_logical_reasoning:rc::olmes"
+    },
+    "generation_kwargs": {},
+    "context_kwargs": {},
+    "dataset_name": "logical_reasoning",
+    "task_name": "basic_skills_logical_reasoning",
+    "version": 0,
+    "task_core": "basic_skills_logical_reasoning"
+  },
+  "current_date": "2025-05-12 00:06:28 UTC"
+}

olmo_eval/oe_eval_tasks/basic_skills_pattern/rc_5shot/config.json

@@ -0,0 +1,23 @@
+{
+  "task_name": "basic_skills_pattern",
+  "task_hash": "67983750bfb70a3b5cc34dcd67ee3c6a",
+  "task_config": {
+    "dataset_path": "basic_skills_pattern",
+    "primary_metric": "acc_per_token",
+    "split": "validation",
+    "num_shots": 5,
+    "metadata": {
+      "regimes": [
+        "OLMES-v0.1"
+      ],
+      "alias": "basic_skills_pattern:rc::olmes"
+    },
+    "generation_kwargs": {},
+    "context_kwargs": {},
+    "dataset_name": "pattern",
+    "task_name": "basic_skills_pattern",
+    "version": 0,
+    "task_core": "basic_skills_pattern"
+  },
+  "current_date": "2025-05-12 00:06:28 UTC"
+}

olmo_eval/oe_eval_tasks/basic_skills_string_operations/rc_5shot/config.json

@@ -0,0 +1,23 @@
+{
+  "task_name": "basic_skills_string_operations",
+  "task_hash": "8e5fdc7697f1bc7b0c9487a6fa682e45",
+  "task_config": {
+    "dataset_path": "basic_skills_string_operations",
+    "primary_metric": "acc_per_token",
+    "split": "validation",
+    "num_shots": 5,
+    "metadata": {
+      "regimes": [
+        "OLMES-v0.1"
+      ],
+      "alias": "basic_skills_string_operations:rc::olmes"
+    },
+    "generation_kwargs": {},
+    "context_kwargs": {},
+    "dataset_name": "string_operations",
+    "task_name": "basic_skills_string_operations",
+    "version": 0,
+    "task_core": "basic_skills_string_operations"
+  },
+  "current_date": "2025-05-12 00:06:28 UTC"
+}

olmo_eval/tasks.py

@@ -1704,6 +1704,46 @@ LABEL_TO_TASK_MAP_ORIG = {
         OEEvalTask,
         {"dataset_path": "arc_easy", "dataset_name": "rc_5shot", "metric_type": "acc"},
     ),
+    "basic_skills_arithmetic_rc_5shot": (
+        OEEvalTask,
+        {
+            "dataset_path": "basic_skills_arithmetic",
+            "dataset_name": "rc_5shot",
+            "metric_type": "acc",
+        },
+    ),
+    "basic_skills_coding_rc_5shot": (
+        OEEvalTask,
+        {"dataset_path": "basic_skills_coding", "dataset_name": "rc_5shot", "metric_type": "acc"},
+    ),
+    "basic_skills_common_knowledge_rc_5shot": (
+        OEEvalTask,
+        {
+            "dataset_path": "basic_skills_common_knowledge",
+            "dataset_name": "rc_5shot",
+            "metric_type": "acc",
+        },
+    ),
+    "basic_skills_logical_reasoning_rc_5shot": (
+        OEEvalTask,
+        {
+            "dataset_path": "basic_skills_logical_reasoning",
+            "dataset_name": "rc_5shot",
+            "metric_type": "acc",
+        },
+    ),
+    "basic_skills_pattern_rc_5shot": (
+        OEEvalTask,
+        {"dataset_path": "basic_skills_pattern", "dataset_name": "rc_5shot", "metric_type": "acc"},
+    ),
+    "basic_skills_string_operations_rc_5shot": (
+        OEEvalTask,
+        {
+            "dataset_path": "basic_skills_string_operations",
+            "dataset_name": "rc_5shot",
+            "metric_type": "acc",
+        },
+    ),
     "boolq_mc_5shot": (
         OEEvalTask,
         {"dataset_path": "boolq", "dataset_name": "mc_5shot", "metric_type": "acc"},

olmo_eval/version.py

@@ -1,6 +1,6 @@
 _MAJOR = "0"
 _MINOR = "7"
-_PATCH = "1"
+_PATCH = "2"
 _SUFFIX = ""
 
 VERSION_SHORT = "{0}.{1}".format(_MAJOR, _MINOR)