subset2evaluate 1.0.21__tar.gz → 1.0.23__tar.gz

{subset2evaluate-1.0.21 → subset2evaluate-1.0.23}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: subset2evaluate
-Version: 1.0.21
+Version: 1.0.23
 Summary: Find informative examples to efficiently (human-)evaluate NLG models.
 Author-email: Vilém Zouhar <vilem.zouhar@gmail.com>
 License: MIT

{subset2evaluate-1.0.21 → subset2evaluate-1.0.23}/pyproject.toml

@@ -11,7 +11,7 @@ packages = ["subset2evaluate"]
 
 [project]
 name = "subset2evaluate"
-version = "1.0.21"
+version = "1.0.23"
 description = "Find informative examples to efficiently (human-)evaluate NLG models."
 license = {text = "MIT"}
 readme = "README.md"

{subset2evaluate-1.0.21 → subset2evaluate-1.0.23}/subset2evaluate/utils.py

@@ -1,6 +1,5 @@
 from typing import Any, Callable, Dict, List, Optional, Union, Literal
 import numpy as np
-from subset2evaluate.reference_info import year2std_refs
 
 PROPS = np.linspace(0.05, 0.5, 10)
 
@@ -16,6 +15,8 @@ def _data_minmax_normalize(data):
     for line in data:
         for met_all in line["scores"].values():
             for met_k, met_v in met_all.items():
+                if met_v is None:
+                    continue
                 data_flat[met_k].append(met_v)
 
     # normalize
@@ -24,10 +25,15 @@ def _data_minmax_normalize(data):
     for line in data:
         for model, met_all in line["scores"].items():
             for met_k, met_v in met_all.items():
-                # (x-min)/(max-min) normalize
-                line["scores"][model][met_k] = (met_v - data_flat[met_k][0]) / (
-                    data_flat[met_k][1] - data_flat[met_k][0]
-                )
+                if met_v is None:
+                    continue
+                if data_flat[met_k][1] - data_flat[met_k][0] == 0:
+                    line["scores"][model][met_k] = 0
+                else:
+                    # (x-min)/(max-min) normalize
+                    line["scores"][model][met_k] = (met_v - data_flat[met_k][0]) / (
+                        data_flat[met_k][1] - data_flat[met_k][0]
+                    )
 
 
 def confidence_interval(data, confidence=0.95):
@@ -616,6 +622,7 @@ def load_data_wmt(  # noqa: C901
     import pickle
     import contextlib
     import importlib.metadata
+    import json
 
     # temporarily change to the root directory, this requires Python 3.11
     with contextlib.chdir(os.path.dirname(os.path.realpath(__file__)) + "/../"):
@@ -637,6 +644,44 @@ def load_data_wmt(  # noqa: C901
                 ):
                     return cache["data"]
 
+        # special handling for WMT24++
+        if os.path.exists(f"data/mt-metrics-eval-v2/{year}/sources/{langs}.jsonl"):
+            with open(
+                f"data/mt-metrics-eval-v2/{year}/sources/{langs}.jsonl", "r"
+            ) as f:
+                data_tmp = f.readlines()
+            lines_src = [json.loads(line)["source"] for line in data_tmp]
+            with open(f"data/mt-metrics-eval-v2/{year}/sources/{langs}.txt", "w") as f:
+                for line in lines_src:
+                    f.write(line + "\n")
+        if os.path.exists(
+            f"data/mt-metrics-eval-v2/{year}/references/{langs}.refA.jsonl"
+        ):
+            with open(
+                f"data/mt-metrics-eval-v2/{year}/references/{langs}.refA.jsonl", "r"
+            ) as f:
+                data_tmp = f.readlines()
+            lines_ref = [json.loads(line)["target"] for line in data_tmp]
+            with open(
+                f"data/mt-metrics-eval-v2/{year}/references/{langs}.refA.txt", "w"
+            ) as f:
+                for line in lines_ref:
+                    f.write(line + "\n")
+        if os.path.exists(
+            f"data/mt-metrics-eval-v2/{year}/references/{langs}.posteditA.jsonl"
+        ):
+            with open(
+                f"data/mt-metrics-eval-v2/{year}/references/{langs}.posteditA.jsonl",
+                "r",
+            ) as f:
+                data_tmp = f.readlines()
+            lines_ref = [json.loads(line)["target"] for line in data_tmp]
+            with open(
+                f"data/mt-metrics-eval-v2/{year}/references/{langs}.posteditA.txt", "w"
+            ) as f:
+                for line in lines_ref:
+                    f.write(line + "\n")
+
         lines_src = open(
             f"data/mt-metrics-eval-v2/{year}/sources/{langs}.txt", "r"
         ).readlines()
@@ -646,14 +691,13 @@ def load_data_wmt(  # noqa: C901
         lines_ref = None
 
         refs_dir = f"data/mt-metrics-eval-v2/{year}/references"
-        selected_human_ref = (
-            file_reference if file_reference is not None else year2std_refs[year][langs]
-        )
-        file_reference_path = f"{refs_dir}/{langs}.{selected_human_ref}.txt"
-
-        if not os.path.exists(file_reference_path):
+        for file_reference in [file_reference, "refA", "refB", "refC", "ref"]:
+            if os.path.exists(f"{refs_dir}/{langs}.{file_reference}.txt"):
+                break
+        else:
             # did not find reference
             return []
+        file_reference_path = f"{refs_dir}/{langs}.{file_reference}.txt"
 
         lines_ref = open(file_reference_path, "r").readlines()
 
@@ -736,14 +780,14 @@ def load_data_wmt(  # noqa: C901
             f"data/mt-metrics-eval-v2/{year}/metric-scores/{langs}/*.seg.score"
         ):
             # among ref-based metrics, load only the scores for the selected human ref
-            if not f.endswith(f"-{selected_human_ref}.seg.score") and not f.endswith(
+            if not f.endswith(f"-{file_reference}.seg.score") and not f.endswith(
                 "-src.seg.score"
             ):
                 continue
             # remove suffix for both ref-based and ref-less metrics
             metric = (
                 f.split("/")[-1]
-                .removesuffix(f"-{selected_human_ref}.seg.score")
+                .removesuffix(f"-{file_reference}.seg.score")
                 .removesuffix("-src.seg.score")
             )
             for line_i, line_raw in enumerate(open(f, "r").readlines()):
@@ -898,7 +942,7 @@ def load_data_wmt_test(**kwargs):
     return data
 
 
-def load_data_wmt_all(min_items=100, **kwargs):
+def load_data_wmt_all(min_items=100, name_filter=lambda x: True, **kwargs):
     data = {
         args: load_data_wmt(*args, **kwargs)
         for args in [
@@ -918,6 +962,61 @@ def load_data_wmt_all(min_items=100, **kwargs):
             ("wmt25", "en-uk_UA"),
             ("wmt25", "en-zh_CN"),
             ("wmt25", "ja-zh_CN"),
+            ("wmt24pp", "en-ar_EG"),
+            ("wmt24pp", "en-bn_IN"),
+            ("wmt24pp", "en-da_DK"),
+            ("wmt24pp", "en-es_MX"),
+            ("wmt24pp", "en-fi_FI"),
+            ("wmt24pp", "en-fr_FR"),
+            ("wmt24pp", "en-hi_IN"),
+            ("wmt24pp", "en-id_ID"),
+            ("wmt24pp", "en-ja_JP"),
+            ("wmt24pp", "en-lt_LT"),
+            ("wmt24pp", "en-mr_IN"),
+            ("wmt24pp", "en-pa_IN"),
+            ("wmt24pp", "en-pt_PT"),
+            ("wmt24pp", "en-sk_SK"),
+            ("wmt24pp", "en-sv_SE"),
+            ("wmt24pp", "en-ta_IN"),
+            ("wmt24pp", "en-tr_TR"),
+            ("wmt24pp", "en-vi_VN"),
+            ("wmt24pp", "en-zu_ZA"),
+            ("wmt24pp", "en-ar_SA"),
+            ("wmt24pp", "en-ca_ES"),
+            ("wmt24pp", "en-de_DE"),
+            ("wmt24pp", "en-et_EE"),
+            ("wmt24pp", "en-fil_PH"),
+            ("wmt24pp", "en-gu_IN"),
+            ("wmt24pp", "en-hr_HR"),
+            ("wmt24pp", "en-is_IS"),
+            ("wmt24pp", "en-kn_IN"),
+            ("wmt24pp", "en-lv_LV"),
+            ("wmt24pp", "en-nl_NL"),
+            ("wmt24pp", "en-pl_PL"),
+            ("wmt24pp", "en-ro_RO"),
+            ("wmt24pp", "en-sl_SI"),
+            ("wmt24pp", "en-sw_KE"),
+            ("wmt24pp", "en-te_IN"),
+            ("wmt24pp", "en-uk_UA"),
+            ("wmt24pp", "en-zh_CN"),
+            ("wmt24pp", "en-bg_BG"),
+            ("wmt24pp", "en-cs_CZ"),
+            ("wmt24pp", "en-el_GR"),
+            ("wmt24pp", "en-fa_IR"),
+            ("wmt24pp", "en-fr_CA"),
+            ("wmt24pp", "en-he_IL"),
+            ("wmt24pp", "en-hu_HU"),
+            ("wmt24pp", "en-it_IT"),
+            ("wmt24pp", "en-ko_KR"),
+            ("wmt24pp", "en-ml_IN"),
+            ("wmt24pp", "en-no_NO"),
+            ("wmt24pp", "en-pt_BR"),
+            ("wmt24pp", "en-ru_RU"),
+            ("wmt24pp", "en-sr_RS"),
+            ("wmt24pp", "en-sw_TZ"),
+            ("wmt24pp", "en-th_TH"),
+            ("wmt24pp", "en-ur_PK"),
+            ("wmt24pp", "en-zh_TW"),
             ("wmt24", "cs-uk"),
             ("wmt24", "en-cs"),
             ("wmt24", "en-de"),
@@ -1025,6 +1124,7 @@ def load_data_wmt_all(min_items=100, **kwargs):
             ("wmt19", "en-de"),
             ("wmt19", "de-cs"),
         ]
+        if name_filter(args)
     }
     # filter out empty datasets
     # some years/langs have issues with human annotations coverage

{subset2evaluate-1.0.21 → subset2evaluate-1.0.23}/subset2evaluate.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: subset2evaluate
-Version: 1.0.21
+Version: 1.0.23
 Summary: Find informative examples to efficiently (human-)evaluate NLG models.
 Author-email: Vilém Zouhar <vilem.zouhar@gmail.com>
 License: MIT

{subset2evaluate-1.0.21 → subset2evaluate-1.0.23}/subset2evaluate.egg-info/SOURCES.txt

@@ -4,7 +4,6 @@ subset2evaluate/__init__.py
 subset2evaluate/evaluate.py
 subset2evaluate/methods.py
 subset2evaluate/methods_old.py
-subset2evaluate/reference_info.py
 subset2evaluate/select_subset.py
 subset2evaluate/test.py
 subset2evaluate/utils.py

subset2evaluate-1.0.21/subset2evaluate/reference_info.py

@@ -1,147 +0,0 @@
-"""Information about the standard human references selected for various WMT test sets."""
-
-year2std_refs = {
-
-    "wmt25": {
-        "cs-de_DE": "refA",
-        "cs-uk_UA": "refA",
-        "en-ar_EG": "refA",
-        "en-bho_IN": "refA",
-        "en-cs_CZ": "refA",
-        "en-et_EE": "refA",
-        "en-is_IS": "refA",
-        "en-it_IT": "refA",
-        "en-ja_JP": "refA",
-        "en-ko_KR": "refA",
-        "en-mas_KE": "refA",
-        "en-ru_RU": "refA",
-        "en-sr_Cyrl_RS": "refA",
-        "en-uk_UA": "refA",
-        "en-zh_CN": "refA",
-        "ja-zh_CN": "refA",
-    },
-    "wmt24": {
-        "en-de": "refB",
-        "en-es": "refA",
-        "ja-zh": "refA",
-        "cs-uk": "refA",
-        "en-cs": "refA",
-        "en-hi": "refA",
-        "en-is": "refA",
-        "en-ja": "refA",
-        "en-ru": "refA",
-        "en-uk": "refA",
-        "en-zh": "refA"
-    },
-    "wmt23.sent": {
-        "en-de": "refA"
-    },
-    "wmt23": {
-        "en-de": "refA",
-        "he-en": "refB",
-        "zh-en": "refA",
-        "cs-uk": "refA",
-        "de-en": "refA",
-        "en-cs": "refA",
-        "en-he": "refB",
-        "en-ja": "refA",
-        "en-ru": "refA",
-        "en-uk": "refA",
-        "en-zh": "refA",
-        "ja-en": "refA",
-        "ru-en": "refA",
-        "uk-en": "refA"
-    },
-    "wmt22": {
-        "en-de": "refA",
-        "en-ru": "refA",
-        "zh-en": "refA",
-        "cs-en": "refB",
-        "cs-uk": "refA",
-        "de-en": "refA",
-        "de-fr": "refA",
-        "en-cs": "refB",
-        "en-hr": "refA",
-        "en-ja": "refA",
-        "en-liv": "refA",
-        "en-uk": "refA",
-        "en-zh": "refA",
-        "fr-de": "refA",
-        "ja-en": "refA",
-        "liv-en": "refA",
-        "ru-en": "refA",
-        "ru-sah": "refA",
-        "sah-ru": "refA",
-        "uk-cs": "refA",
-        "uk-en": "refA"
-    },
-    "wmt21.news": {
-        "en-cs": "refA",
-        "en-de": "refC",
-        "en-ha": "refA",
-        "en-is": "refA",
-        "en-ja": "refA",
-        "en-ru": "refA",
-        "en-zh": "refA",
-        "cs-en": "refA",
-        "de-en": "refA",
-        "de-fr": "refA",
-        "fr-de": "refA",
-        "ha-en": "refA",
-        "is-en": "refA",
-        "ja-en": "refA",
-        "ru-en": "refA",
-        "zh-en": "refB"
-    },
-    "wmt21.tedtalks": {
-        "en-de": "refA",
-        "en-ru": "refA",
-        "zh-en": "refB"
-    },
-    "wmt21.flores": {
-        "bn-hi": "refA",
-        "hi-bn": "refA",
-        "xh-zu": "refA",
-        "zu-xh": "refA"
-    },
-    "wmt20": {
-        "cs-en": "ref",
-        "de-en": "ref",
-        "en-cs": "ref",
-        "en-de": "ref",
-        "en-iu": "ref",
-        "en-ja": "ref",
-        "en-pl": "ref",
-        "en-ru": "ref",
-        "en-ta": "ref",
-        "en-zh": "ref",
-        "iu-en": "ref",
-        "ja-en": "ref",
-        "km-en": "ref",
-        "pl-en": "ref",
-        "ps-en": "ref",
-        "ru-en": "ref",
-        "ta-en": "ref",
-        "zh-en": "ref"
-    },
-    "wmt19": {
-        "de-cs": "ref",
-        "de-en": "ref",
-        "de-fr": "ref",
-        "en-cs": "ref",
-        "en-de": "ref",
-        "en-fi": "ref",
-        "en-gu": "ref",
-        "en-kk": "ref",
-        "en-lt": "ref",
-        "en-ru": "ref",
-        "en-zh": "ref",
-        "fi-en": "ref",
-        "fr-de": "ref",
-        "gu-en": "ref",
-        "kk-en": "ref",
-        "lt-en": "ref",
-        "ru-en": "ref",
-        "zh-en": "ref"
-    }
-}