subset2evaluate 1.0.22__tar.gz → 1.0.24__tar.gz

{subset2evaluate-1.0.22 → subset2evaluate-1.0.24}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: subset2evaluate
-Version: 1.0.22
+Version: 1.0.24
 Summary: Find informative examples to efficiently (human-)evaluate NLG models.
 Author-email: Vilém Zouhar <vilem.zouhar@gmail.com>
 License: MIT

{subset2evaluate-1.0.22 → subset2evaluate-1.0.24}/pyproject.toml

@@ -11,7 +11,7 @@ packages = ["subset2evaluate"]
 
 [project]
 name = "subset2evaluate"
-version = "1.0.22"
+version = "1.0.24"
 description = "Find informative examples to efficiently (human-)evaluate NLG models."
 license = {text = "MIT"}
 readme = "README.md"

{subset2evaluate-1.0.22 → subset2evaluate-1.0.24}/subset2evaluate/utils.py

@@ -1,6 +1,5 @@
 from typing import Any, Callable, Dict, List, Optional, Union, Literal
 import numpy as np
-from subset2evaluate.reference_info import year2std_refs
 
 PROPS = np.linspace(0.05, 0.5, 10)
 
@@ -623,6 +622,7 @@ def load_data_wmt(  # noqa: C901
     import pickle
     import contextlib
     import importlib.metadata
+    import json
 
     # temporarily change to the root directory, this requires Python 3.11
     with contextlib.chdir(os.path.dirname(os.path.realpath(__file__)) + "/../"):
@@ -644,6 +644,44 @@ def load_data_wmt(  # noqa: C901
                 ):
                     return cache["data"]
 
+        # special handling for WMT24++
+        if os.path.exists(f"data/mt-metrics-eval-v2/{year}/sources/{langs}.jsonl"):
+            with open(
+                f"data/mt-metrics-eval-v2/{year}/sources/{langs}.jsonl", "r"
+            ) as f:
+                data_tmp = f.readlines()
+            lines_src = [json.loads(line)["source"] for line in data_tmp]
+            with open(f"data/mt-metrics-eval-v2/{year}/sources/{langs}.txt", "w") as f:
+                for line in lines_src:
+                    f.write(line + "\n")
+        if os.path.exists(
+            f"data/mt-metrics-eval-v2/{year}/references/{langs}.refA.jsonl"
+        ):
+            with open(
+                f"data/mt-metrics-eval-v2/{year}/references/{langs}.refA.jsonl", "r"
+            ) as f:
+                data_tmp = f.readlines()
+            lines_ref = [json.loads(line)["target"] for line in data_tmp]
+            with open(
+                f"data/mt-metrics-eval-v2/{year}/references/{langs}.refA.txt", "w"
+            ) as f:
+                for line in lines_ref:
+                    f.write(line + "\n")
+        if os.path.exists(
+            f"data/mt-metrics-eval-v2/{year}/references/{langs}.posteditA.jsonl"
+        ):
+            with open(
+                f"data/mt-metrics-eval-v2/{year}/references/{langs}.posteditA.jsonl",
+                "r",
+            ) as f:
+                data_tmp = f.readlines()
+            lines_ref = [json.loads(line)["target"] for line in data_tmp]
+            with open(
+                f"data/mt-metrics-eval-v2/{year}/references/{langs}.posteditA.txt", "w"
+            ) as f:
+                for line in lines_ref:
+                    f.write(line + "\n")
+
         lines_src = open(
             f"data/mt-metrics-eval-v2/{year}/sources/{langs}.txt", "r"
         ).readlines()
@@ -653,14 +691,13 @@ def load_data_wmt(  # noqa: C901
         lines_ref = None
 
         refs_dir = f"data/mt-metrics-eval-v2/{year}/references"
-        selected_human_ref = (
-            file_reference if file_reference is not None else year2std_refs[year][langs]
-        )
-        file_reference_path = f"{refs_dir}/{langs}.{selected_human_ref}.txt"
-
-        if not os.path.exists(file_reference_path):
+        for file_reference in [file_reference, "refA", "refB", "refC", "ref"]:
+            if os.path.exists(f"{refs_dir}/{langs}.{file_reference}.txt"):
+                break
+        else:
             # did not find reference
             return []
+        file_reference_path = f"{refs_dir}/{langs}.{file_reference}.txt"
 
         lines_ref = open(file_reference_path, "r").readlines()
 
@@ -675,23 +712,33 @@ def load_data_wmt(  # noqa: C901
                 human_refs.add(parts[1])
 
         # do not consider canary line
-        contain_canary_line = lines_src[0].lower().startswith("canary")
+        # for WTM24++ it's part of the translation dataset
+        contain_canary_line = (
+            lines_src[0].lower().startswith("canary") and year != "wmt24pp"
+        )
         if contain_canary_line:
             lines_src.pop(0)
             lines_doc.pop(0)
             lines_ref.pop(0)
 
         line_model = {}
-        for f in glob.glob(
-            f"data/mt-metrics-eval-v2/{year}/system-outputs/{langs}/*.txt"
+        for f in list(
+            glob.glob(f"data/mt-metrics-eval-v2/{year}/system-outputs/{langs}/*.txt")
+        ) + list(
+            glob.glob(f"data/mt-metrics-eval-v2/{year}/system-outputs/{langs}/*.jsonl")
         ):
-            model = f.split("/")[-1].removesuffix(".txt")
+            if f.endswith(".jsonl"):
+                model = f.split("/")[-1].removesuffix(".jsonl")
+                lines = [json.loads(line)["hypothesis"] for line in open(f, "r")]
+            else:
+                model = f.split("/")[-1].removesuffix(".txt")
+                lines = open(f, "r").readlines()
             if model in {"synthetic_ref", "chrf_bestmbr"}:
                 continue
             if model in human_refs and not include_ref:
                 continue
 
-            line_model[model] = open(f, "r").readlines()
+            line_model[model] = lines
             if contain_canary_line:
                 line_model[model].pop(0)
 
@@ -699,22 +746,18 @@ def load_data_wmt(  # noqa: C901
 
         lines_score = collections.defaultdict(list)
 
-        if file_protocol is not None:
-            f_protocols = [
-                f"data/mt-metrics-eval-v2/{year}/human-scores/{langs}.{file_protocol}.seg.score",
-            ]
-        else:
-            f_protocols = [
-                f"data/mt-metrics-eval-v2/{year}/human-scores/{langs}.mqm.seg.score",
-                f"data/mt-metrics-eval-v2/{year}/human-scores/{langs}.esa-merged.seg.score",
-                f"data/mt-metrics-eval-v2/{year}/human-scores/{langs}.esa.seg.score",
-                f"data/mt-metrics-eval-v2/{year}/human-scores/{langs}.da-sqm.seg.score",
-                f"data/mt-metrics-eval-v2/{year}/human-scores/{langs}.mqm.seg.score",
-                f"data/mt-metrics-eval-v2/{year}/human-scores/{langs}.wmt.seg.score",
-                f"data/mt-metrics-eval-v2/{year}/human-scores/{langs}.appraise.seg.score",
-                f"data/mt-metrics-eval-v2/{year}/human-scores/{langs}.wmt-raw.seg.score",
-                f"data/mt-metrics-eval-v2/{year}/human-scores/{langs}.wmt-appraise.seg.score",
-            ]
+        f_protocols = [
+            f"data/mt-metrics-eval-v2/{year}/human-scores/{langs}.{file_protocol}.seg.score",
+            f"data/mt-metrics-eval-v2/{year}/human-scores/{langs}.mqm.seg.score",
+            f"data/mt-metrics-eval-v2/{year}/human-scores/{langs}.esa-merged.seg.score",
+            f"data/mt-metrics-eval-v2/{year}/human-scores/{langs}.esa.seg.score",
+            f"data/mt-metrics-eval-v2/{year}/human-scores/{langs}.da-sqm.seg.score",
+            f"data/mt-metrics-eval-v2/{year}/human-scores/{langs}.mqm.seg.score",
+            f"data/mt-metrics-eval-v2/{year}/human-scores/{langs}.wmt.seg.score",
+            f"data/mt-metrics-eval-v2/{year}/human-scores/{langs}.appraise.seg.score",
+            f"data/mt-metrics-eval-v2/{year}/human-scores/{langs}.wmt-raw.seg.score",
+            f"data/mt-metrics-eval-v2/{year}/human-scores/{langs}.wmt-appraise.seg.score",
+        ]
         for fname in [*f_protocols, False]:
             if fname and os.path.exists(fname):
                 break
@@ -743,14 +786,14 @@ def load_data_wmt(  # noqa: C901
             f"data/mt-metrics-eval-v2/{year}/metric-scores/{langs}/*.seg.score"
         ):
             # among ref-based metrics, load only the scores for the selected human ref
-            if not f.endswith(f"-{selected_human_ref}.seg.score") and not f.endswith(
+            if not f.endswith(f"-{file_reference}.seg.score") and not f.endswith(
                 "-src.seg.score"
             ):
                 continue
             # remove suffix for both ref-based and ref-less metrics
             metric = (
                 f.split("/")[-1]
-                .removesuffix(f"-{selected_human_ref}.seg.score")
+                .removesuffix(f"-{file_reference}.seg.score")
                 .removesuffix("-src.seg.score")
             )
             for line_i, line_raw in enumerate(open(f, "r").readlines()):
@@ -771,6 +814,8 @@ def load_data_wmt(  # noqa: C901
 
                     model_line_i -= 1
 
+                if model_line_i >= len(lines_score[model]):
+                    print(year, langs, model_line_i, metric, model, contain_canary_line)
                 lines_score[model][model_line_i][metric] = float(score)
 
         # filter out lines that have no human score
@@ -905,7 +950,7 @@ def load_data_wmt_test(**kwargs):
     return data
 
 
-def load_data_wmt_all(min_items=100, **kwargs):
+def load_data_wmt_all(min_items=100, name_filter=lambda x: True, **kwargs):
     data = {
         args: load_data_wmt(*args, **kwargs)
         for args in [
@@ -925,6 +970,61 @@ def load_data_wmt_all(min_items=100, **kwargs):
             ("wmt25", "en-uk_UA"),
             ("wmt25", "en-zh_CN"),
             ("wmt25", "ja-zh_CN"),
+            ("wmt24pp", "en-ar_EG"),
+            ("wmt24pp", "en-bn_IN"),
+            ("wmt24pp", "en-da_DK"),
+            ("wmt24pp", "en-es_MX"),
+            ("wmt24pp", "en-fi_FI"),
+            ("wmt24pp", "en-fr_FR"),
+            ("wmt24pp", "en-hi_IN"),
+            ("wmt24pp", "en-id_ID"),
+            ("wmt24pp", "en-ja_JP"),
+            ("wmt24pp", "en-lt_LT"),
+            ("wmt24pp", "en-mr_IN"),
+            ("wmt24pp", "en-pa_IN"),
+            ("wmt24pp", "en-pt_PT"),
+            ("wmt24pp", "en-sk_SK"),
+            ("wmt24pp", "en-sv_SE"),
+            ("wmt24pp", "en-ta_IN"),
+            ("wmt24pp", "en-tr_TR"),
+            ("wmt24pp", "en-vi_VN"),
+            ("wmt24pp", "en-zu_ZA"),
+            ("wmt24pp", "en-ar_SA"),
+            ("wmt24pp", "en-ca_ES"),
+            ("wmt24pp", "en-de_DE"),
+            ("wmt24pp", "en-et_EE"),
+            ("wmt24pp", "en-fil_PH"),
+            ("wmt24pp", "en-gu_IN"),
+            ("wmt24pp", "en-hr_HR"),
+            ("wmt24pp", "en-is_IS"),
+            ("wmt24pp", "en-kn_IN"),
+            ("wmt24pp", "en-lv_LV"),
+            ("wmt24pp", "en-nl_NL"),
+            ("wmt24pp", "en-pl_PL"),
+            ("wmt24pp", "en-ro_RO"),
+            ("wmt24pp", "en-sl_SI"),
+            ("wmt24pp", "en-sw_KE"),
+            ("wmt24pp", "en-te_IN"),
+            ("wmt24pp", "en-uk_UA"),
+            ("wmt24pp", "en-zh_CN"),
+            ("wmt24pp", "en-bg_BG"),
+            ("wmt24pp", "en-cs_CZ"),
+            ("wmt24pp", "en-el_GR"),
+            ("wmt24pp", "en-fa_IR"),
+            ("wmt24pp", "en-fr_CA"),
+            ("wmt24pp", "en-he_IL"),
+            ("wmt24pp", "en-hu_HU"),
+            ("wmt24pp", "en-it_IT"),
+            ("wmt24pp", "en-ko_KR"),
+            ("wmt24pp", "en-ml_IN"),
+            ("wmt24pp", "en-no_NO"),
+            ("wmt24pp", "en-pt_BR"),
+            ("wmt24pp", "en-ru_RU"),
+            ("wmt24pp", "en-sr_RS"),
+            ("wmt24pp", "en-sw_TZ"),
+            ("wmt24pp", "en-th_TH"),
+            ("wmt24pp", "en-ur_PK"),
+            ("wmt24pp", "en-zh_TW"),
             ("wmt24", "cs-uk"),
             ("wmt24", "en-cs"),
             ("wmt24", "en-de"),
@@ -1032,6 +1132,7 @@ def load_data_wmt_all(min_items=100, **kwargs):
             ("wmt19", "en-de"),
             ("wmt19", "de-cs"),
         ]
+        if name_filter(args)
     }
     # filter out empty datasets
     # some years/langs have issues with human annotations coverage

{subset2evaluate-1.0.22 → subset2evaluate-1.0.24}/subset2evaluate.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: subset2evaluate
-Version: 1.0.22
+Version: 1.0.24
 Summary: Find informative examples to efficiently (human-)evaluate NLG models.
 Author-email: Vilém Zouhar <vilem.zouhar@gmail.com>
 License: MIT

{subset2evaluate-1.0.22 → subset2evaluate-1.0.24}/subset2evaluate.egg-info/SOURCES.txt

@@ -4,7 +4,6 @@ subset2evaluate/__init__.py
 subset2evaluate/evaluate.py
 subset2evaluate/methods.py
 subset2evaluate/methods_old.py
-subset2evaluate/reference_info.py
 subset2evaluate/select_subset.py
 subset2evaluate/test.py
 subset2evaluate/utils.py

subset2evaluate-1.0.22/subset2evaluate/reference_info.py

@@ -1,147 +0,0 @@
-"""Information about the standard human references selected for various WMT test sets."""
-
-year2std_refs = {
-
-    "wmt25": {
-        "cs-de_DE": "refA",
-        "cs-uk_UA": "refA",
-        "en-ar_EG": "refA",
-        "en-bho_IN": "refA",
-        "en-cs_CZ": "refA",
-        "en-et_EE": "refA",
-        "en-is_IS": "refA",
-        "en-it_IT": "refA",
-        "en-ja_JP": "refA",
-        "en-ko_KR": "refA",
-        "en-mas_KE": "refA",
-        "en-ru_RU": "refA",
-        "en-sr_Cyrl_RS": "refA",
-        "en-uk_UA": "refA",
-        "en-zh_CN": "refA",
-        "ja-zh_CN": "refA",
-    },
-    "wmt24": {
-        "en-de": "refB",
-        "en-es": "refA",
-        "ja-zh": "refA",
-        "cs-uk": "refA",
-        "en-cs": "refA",
-        "en-hi": "refA",
-        "en-is": "refA",
-        "en-ja": "refA",
-        "en-ru": "refA",
-        "en-uk": "refA",
-        "en-zh": "refA"
-    },
-    "wmt23.sent": {
-        "en-de": "refA"
-    },
-    "wmt23": {
-        "en-de": "refA",
-        "he-en": "refB",
-        "zh-en": "refA",
-        "cs-uk": "refA",
-        "de-en": "refA",
-        "en-cs": "refA",
-        "en-he": "refB",
-        "en-ja": "refA",
-        "en-ru": "refA",
-        "en-uk": "refA",
-        "en-zh": "refA",
-        "ja-en": "refA",
-        "ru-en": "refA",
-        "uk-en": "refA"
-    },
-    "wmt22": {
-        "en-de": "refA",
-        "en-ru": "refA",
-        "zh-en": "refA",
-        "cs-en": "refB",
-        "cs-uk": "refA",
-        "de-en": "refA",
-        "de-fr": "refA",
-        "en-cs": "refB",
-        "en-hr": "refA",
-        "en-ja": "refA",
-        "en-liv": "refA",
-        "en-uk": "refA",
-        "en-zh": "refA",
-        "fr-de": "refA",
-        "ja-en": "refA",
-        "liv-en": "refA",
-        "ru-en": "refA",
-        "ru-sah": "refA",
-        "sah-ru": "refA",
-        "uk-cs": "refA",
-        "uk-en": "refA"
-    },
-    "wmt21.news": {
-        "en-cs": "refA",
-        "en-de": "refC",
-        "en-ha": "refA",
-        "en-is": "refA",
-        "en-ja": "refA",
-        "en-ru": "refA",
-        "en-zh": "refA",
-        "cs-en": "refA",
-        "de-en": "refA",
-        "de-fr": "refA",
-        "fr-de": "refA",
-        "ha-en": "refA",
-        "is-en": "refA",
-        "ja-en": "refA",
-        "ru-en": "refA",
-        "zh-en": "refB"
-    },
-    "wmt21.tedtalks": {
-        "en-de": "refA",
-        "en-ru": "refA",
-        "zh-en": "refB"
-    },
-    "wmt21.flores": {
-        "bn-hi": "refA",
-        "hi-bn": "refA",
-        "xh-zu": "refA",
-        "zu-xh": "refA"
-    },
-    "wmt20": {
-        "cs-en": "ref",
-        "de-en": "ref",
-        "en-cs": "ref",
-        "en-de": "ref",
-        "en-iu": "ref",
-        "en-ja": "ref",
-        "en-pl": "ref",
-        "en-ru": "ref",
-        "en-ta": "ref",
-        "en-zh": "ref",
-        "iu-en": "ref",
-        "ja-en": "ref",
-        "km-en": "ref",
-        "pl-en": "ref",
-        "ps-en": "ref",
-        "ru-en": "ref",
-        "ta-en": "ref",
-        "zh-en": "ref"
-    },
-    "wmt19": {
-        "de-cs": "ref",
-        "de-en": "ref",
-        "de-fr": "ref",
-        "en-cs": "ref",
-        "en-de": "ref",
-        "en-fi": "ref",
-        "en-gu": "ref",
-        "en-kk": "ref",
-        "en-lt": "ref",
-        "en-ru": "ref",
-        "en-zh": "ref",
-        "fi-en": "ref",
-        "fr-de": "ref",
-        "gu-en": "ref",
-        "kk-en": "ref",
-        "lt-en": "ref",
-        "ru-en": "ref",
-        "zh-en": "ref"
-    }
-}