ai2-olmo-eval 0.8.4__tar.gz → 0.8.6__tar.gz

{ai2_olmo_eval-0.8.4 → ai2_olmo_eval-0.8.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ai2-olmo-eval
-Version: 0.8.4
+Version: 0.8.6
 Summary: In-loop evaluation tasks for language modeling
 Author-email: Allen Institute for Artificial Intelligence <olmo@allenai.org>
 License:                                  Apache License
@@ -210,16 +210,15 @@ Project-URL: Changelog, https://github.com/allenai/OLMo-in-loop-evals/blob/main/
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: numpy<2.0
 Requires-Dist: torch
 Requires-Dist: torchmetrics
-Requires-Dist: datasets
-Requires-Dist: tokenizers
-Requires-Dist: scikit-learn
+Requires-Dist: datasets<4,>=3.6.0
 Requires-Dist: cached-path
 Requires-Dist: requests
 Requires-Dist: packaging
 Requires-Dist: importlib_resources
+Requires-Dist: tokenizers<0.20,>=0.19.1
+Requires-Dist: pyarrow<20,>=19.0
 Provides-Extra: dev
 Requires-Dist: ruff; extra == "dev"
 Requires-Dist: mypy<1.4,>=1.0; extra == "dev"
@@ -245,3 +244,21 @@ Code for in-loop evaluation tasks used by the OLMo training team.
 ```
 pip install ai2-olmo-eval
 ```
+
+## Release process
+
+### Steps
+
+1. Update the version in `src/olmo_eval/version.py`.
+2. Run the release script:
+
+    ```bash
+    ./src/scripts/release.sh
+    ```
+
+    This will commit the changes to the CHANGELOG and `version.py` files and then create a new tag in git
+    which will trigger a workflow on GitHub Actions that handles the rest.
+
+### Fixing a failed release
+
+If for some reason the GitHub Actions release workflow failed with an error that needs to be fixed, you'll have to delete the tag on GitHub. Once you've pushed a fix you can simply repeat the steps above.

ai2_olmo_eval-0.8.6/README.md

@@ -0,0 +1,27 @@
+# OLMo-in-loop-evals
+
+Code for in-loop evaluation tasks used by the OLMo training team.
+
+## Installation
+
+```
+pip install ai2-olmo-eval
+```
+
+## Release process
+
+### Steps
+
+1. Update the version in `src/olmo_eval/version.py`.
+2. Run the release script:
+
+    ```bash
+    ./src/scripts/release.sh
+    ```
+
+    This will commit the changes to the CHANGELOG and `version.py` files and then create a new tag in git
+    which will trigger a workflow on GitHub Actions that handles the rest.
+
+### Fixing a failed release
+
+If for some reason the GitHub Actions release workflow failed with an error that needs to be fixed, you'll have to delete the tag on GitHub. Once you've pushed a fix you can simply repeat the steps above.

{ai2_olmo_eval-0.8.4 → ai2_olmo_eval-0.8.6}/pyproject.toml

@@ -13,16 +13,15 @@ authors = [
 requires-python = ">=3.9"
 license = { file = "LICENSE" }
 dependencies = [
-    "numpy<2.0",
     "torch",
     "torchmetrics",
-    "datasets",
-    "tokenizers",
-    "scikit-learn",
+    "datasets>=3.6.0,<4",
     "cached-path",
     "requests",
     "packaging",
     "importlib_resources",
+    "tokenizers>=0.19.1,<0.20",
+    "pyarrow>=19.0,<20",    # datasets brings pyarrow, but version 21 breaks things
 ]
 
 [project.urls]

{ai2_olmo_eval-0.8.4 → ai2_olmo_eval-0.8.6}/src/ai2_olmo_eval.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ai2-olmo-eval
-Version: 0.8.4
+Version: 0.8.6
 Summary: In-loop evaluation tasks for language modeling
 Author-email: Allen Institute for Artificial Intelligence <olmo@allenai.org>
 License:                                  Apache License
@@ -210,16 +210,15 @@ Project-URL: Changelog, https://github.com/allenai/OLMo-in-loop-evals/blob/main/
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: numpy<2.0
 Requires-Dist: torch
 Requires-Dist: torchmetrics
-Requires-Dist: datasets
-Requires-Dist: tokenizers
-Requires-Dist: scikit-learn
+Requires-Dist: datasets<4,>=3.6.0
 Requires-Dist: cached-path
 Requires-Dist: requests
 Requires-Dist: packaging
 Requires-Dist: importlib_resources
+Requires-Dist: tokenizers<0.20,>=0.19.1
+Requires-Dist: pyarrow<20,>=19.0
 Provides-Extra: dev
 Requires-Dist: ruff; extra == "dev"
 Requires-Dist: mypy<1.4,>=1.0; extra == "dev"
@@ -245,3 +244,21 @@ Code for in-loop evaluation tasks used by the OLMo training team.
 ```
 pip install ai2-olmo-eval
 ```
+
+## Release process
+
+### Steps
+
+1. Update the version in `src/olmo_eval/version.py`.
+2. Run the release script:
+
+    ```bash
+    ./src/scripts/release.sh
+    ```
+
+    This will commit the changes to the CHANGELOG and `version.py` files and then create a new tag in git
+    which will trigger a workflow on GitHub Actions that handles the rest.
+
+### Fixing a failed release
+
+If for some reason the GitHub Actions release workflow failed with an error that needs to be fixed, you'll have to delete the tag on GitHub. Once you've pushed a fix you can simply repeat the steps above.

{ai2_olmo_eval-0.8.4 → ai2_olmo_eval-0.8.6}/src/ai2_olmo_eval.egg-info/requires.txt

@@ -1,13 +1,12 @@
-numpy<2.0
 torch
 torchmetrics
-datasets
-tokenizers
-scikit-learn
+datasets<4,>=3.6.0
 cached-path
 requests
 packaging
 importlib_resources
+tokenizers<0.20,>=0.19.1
+pyarrow<20,>=19.0
 
 [all]
 ai2-olmo-eval[dev]

{ai2_olmo_eval-0.8.4 → ai2_olmo_eval-0.8.6}/src/olmo_eval/metrics.py

@@ -3,7 +3,6 @@ from typing import Any, Dict, List, Optional, Tuple, TypeVar
 
 import torch
 import torch.nn.functional as F
-from sklearn.metrics import f1_score
 from torchmetrics import Metric
 
 from .util import all_gather_object
@@ -395,8 +394,10 @@ class ICLMetric(Metric):
             assert preds is not None
             assert labels is not None
             # for NLI tasks, continuations are yes, no, neither, so idx=0 assigned to pos label
-            score = f1_score(labels, preds, pos_label=0)
-            score_no_leading_space = f1_score(labels, preds_no_leading_space, pos_label=0)
+            score = self.custom_f1_score(labels, preds, pos_label=0)
+            score_no_leading_space = self.custom_f1_score(
+                labels, preds_no_leading_space, pos_label=0
+            )
             return {
                 "f1_v1": torch.tensor(score),
                 "f1_v2": torch.tensor(score_no_leading_space),
@@ -432,3 +433,21 @@ class ICLMetric(Metric):
                 ),
                 "soft_log_v2": torch.tensor(sum(soft_log_score) / len(soft_log_score)),
             }
+
+    def custom_f1_score(self, y_true, y_pred, pos_label=1):
+        y_true = list(y_true)
+        y_pred = list(y_pred)
+        tp = sum((yt == pos_label) and (yp == pos_label) for yt, yp in zip(y_true, y_pred))
+        fp = sum((yt != pos_label) and (yp == pos_label) for yt, yp in zip(y_true, y_pred))
+        fn = sum((yt == pos_label) and (yp != pos_label) for yt, yp in zip(y_true, y_pred))
+
+        if tp + fp == 0 or tp + fn == 0:
+            return 0.0
+
+        precision = tp / (tp + fp)
+        recall = tp / (tp + fn)
+
+        if precision + recall == 0:
+            return 0.0
+
+        return 2 * precision * recall / (precision + recall)

{ai2_olmo_eval-0.8.4 → ai2_olmo_eval-0.8.6}/src/olmo_eval/version.py

@@ -1,6 +1,6 @@
 _MAJOR = "0"
 _MINOR = "8"
-_PATCH = "4"
+_PATCH = "6"
 _SUFFIX = ""
 
 VERSION_SHORT = "{0}.{1}".format(_MAJOR, _MINOR)

ai2_olmo_eval-0.8.4/README.md

@@ -1,9 +0,0 @@
-# OLMo-in-loop-evals
-
-Code for in-loop evaluation tasks used by the OLMo training team.
-
-## Installation
-
-```
-pip install ai2-olmo-eval
-```