subset2evaluate 0.0.1a6__py3-none-any.whl

subset2evaluate/__init__.py

@@ -0,0 +1,6 @@
+# default imports
+# flake8: noqa F401
+import subset2evaluate.utils
+import subset2evaluate.evaluate
+import subset2evaluate.select_subset
+import subset2evaluate.methods

subset2evaluate/evaluate.py

@@ -0,0 +1,272 @@
+from typing import Dict, List, Tuple
+import numpy as np
+import subset2evaluate
+import subset2evaluate.utils as utils
+
+
+def run_evaluate_cluacc(data_new: List[Dict], data_old: List[Dict], metric="human", props: List[float]=utils.PROPS) -> Tuple[float, float]:
+    # both list or descriptor is fine
+    data_new = utils.load_data(data_new)
+    data_old = utils.load_data(data_old)
+
+    clu_new = []
+    acc_new = []
+    for prop in props:
+        k = int(len(data_old) * prop)
+        clu_new.append(eval_subset_clusters(data_new[:k], metric=metric))
+        acc_new.append(eval_subset_accuracy(data_new[:k], data_old, metric=metric))
+
+    return clu_new, acc_new
+
+
+def run_evaluate_cluacc_par(
+        data_new: List[Dict],
+        data_old: List[Dict],
+        clus_tgt: List[float],
+        accs_tgt: List[float],
+        metric="human",
+        props: List[float]=utils.PROPS,
+        workers=10,
+) -> Tuple[float, float]:
+    """
+    Evaluates the proportion of data that is needed to achieve parity with target.
+    """
+    import multiprocessing.pool
+
+    # both list or descriptor is fine
+    data_new = utils.load_data(data_new)
+    data_old = utils.load_data(data_old)
+
+    def _par_clu(data_new, clu_tgt, metric):
+        for k in range(5, len(data_new) + 1):
+            if eval_subset_clusters(data_new[:k], metric=metric) >= clu_tgt:
+                break
+        return k
+    
+    def _par_acc(data_new, data_old, acc_tgt, metric):
+        for k in range(5, len(data_new) + 1):
+            if eval_subset_accuracy(data_new[:k], data_old, metric=metric) >= acc_tgt:
+                break
+        return k
+
+    # multiprocess for each prop rather than k because the thread
+    # orchestration would be more expensive otherwise
+    with multiprocessing.pool.ThreadPool(min(workers, len(props))) as pool:
+        ks_clu_par = pool.starmap(
+            _par_clu,
+            [(data_new, clu_tgt, metric) for prop, clu_tgt in zip(props, clus_tgt)]
+        )
+        ks_clu_par = [k / (len(data_old) * prop) for k, prop in zip(ks_clu_par, props)]
+
+        ks_acc_par = pool.starmap(
+            _par_acc,
+            [(data_new, data_old, clu_tgt, metric) for prop, clu_tgt in zip(props, accs_tgt)]
+        )
+        ks_acc_par = [k / (len(data_old) * prop) for k, prop in zip(ks_acc_par, props)]
+    
+    return np.average(ks_clu_par), np.average(ks_acc_par)
+
+
+def precompute_randnorm(
+    data_old: List[Dict],
+    random_seeds=10,
+    metric="human",
+    workers=10,
+) -> Tuple[List[float], List[float], float, float]:
+    import subset2evaluate.select_subset
+
+    clu_random = []
+    acc_random = []
+    for seed in range(random_seeds):
+        clu_new, acc_new = run_evaluate_cluacc(
+            subset2evaluate.select_subset.run_select_subset(data_old, method="random", seed=seed),
+            data_old,
+            metric=metric,
+        )
+        clu_random.append(clu_new)
+        acc_random.append(acc_new)
+    clu_random = np.average(clu_random, axis=0)
+    acc_random = np.average(acc_random, axis=0)
+
+    pars_clu_rand = []
+    pars_acc_rand = []
+
+    for seed in range(random_seeds, 2*random_seeds):
+        par_clu_rand, par_acc_rand = run_evaluate_cluacc_par(
+            subset2evaluate.select_subset.run_select_subset(data_old, method="random", seed=seed),
+            data_old,
+            clu_random,
+            acc_random,
+            metric=metric,
+            workers=workers,
+        )
+        pars_clu_rand.append(par_clu_rand)
+        pars_acc_rand.append(par_acc_rand)
+
+    return (clu_random, acc_random), (np.average(pars_clu_rand), np.average(pars_acc_rand))
+
+def run_evaluate_cluacc_randnorm(
+    data_new: List[Dict],
+    data_old: List[Dict],
+    random_seeds=10,
+    metric="human",
+    cluacc_precomputed = None
+) -> Tuple[float, float]:
+
+    if cluacc_precomputed is not None:
+        (clu_random, acc_random), (clu_random_norm, acc_random_norm) = cluacc_precomputed
+    else:
+        (clu_random, acc_random), (clu_random_norm, acc_random_norm) = precompute_randnorm(data_old, random_seeds=random_seeds, metric=metric)
+
+    # compute the parity of the new data
+    par_clu, par_acc = run_evaluate_cluacc_par(
+        data_new, data_old,
+        clu_random, acc_random,
+        metric=metric
+    )
+
+    return par_clu/clu_random_norm, par_acc/acc_random_norm
+
+
+def run_evaluate_top_timebudget(data_new, data_old, metric="human"):
+    # both list or descriptor is fine
+    data_old = utils.load_data(data_old)
+    data_new = utils.load_data(data_new)
+
+    clu_new = []
+    acc_new = []
+    for prop in utils.PROPS:
+        k = int(len(data_old) * prop)
+        data_new_inbudget = []
+        budget = k
+        for item in data_new:
+            if item["time"] <= budget:
+                budget -= item["time"]
+                data_new_inbudget.append(item)
+            else:
+                break
+        clu_new.append(eval_subset_clusters(data_new_inbudget, metric=metric))
+        acc_new.append(eval_subset_accuracy(data_new_inbudget, data_old, metric=metric))
+
+    return clu_new, acc_new
+
+
+def eval_subset_accuracy(data_new: List[Dict], data_old: List[Dict], metric="human"):
+    # evaluates against ordering from data_old
+    import itertools
+
+    systems = list(data_old[0]["scores"].keys())
+
+    scores_old = get_sys_absolute(data_old, metric=metric)
+    scores_new = get_sys_absolute(data_new, metric=metric)
+
+    result = []
+    for sys1, sys2 in itertools.combinations(systems, 2):
+        result.append((scores_old[sys1] < scores_old[sys2]) == (scores_new[sys1] < scores_new[sys2]))
+
+    return np.average(result)
+
+
+def eval_subset_clusters(data: List[Dict], metric="human"):
+    from scipy.stats import wilcoxon
+    import warnings
+
+    # if we have just 3 samples, we can't say that there are clusters
+    if len(data) < 3:
+        return 1
+
+    # sort from top
+    sys_ord = list(get_sys_absolute(data, metric=metric).items())
+    sys_ord.sort(key=lambda x: x[1], reverse=True)
+    sys_ord = [sys for sys, _ in sys_ord]
+
+    def get_scores(system):
+        return [line["scores"][system][metric] for line in data]
+
+    clusters = [[get_scores(sys_ord.pop(0))]]
+    while sys_ord:
+        sys_scores = get_scores(sys_ord.pop(0))
+        diffs = [x - y for x, y in zip(sys_scores, clusters[-1][-1])]
+
+        with warnings.catch_warnings(action="ignore"):
+            if all([d == 0 for d in diffs]) or wilcoxon(diffs, alternative="less").pvalue < 0.05:
+                clusters.append([sys_scores])
+            else:
+                clusters[-1].append(sys_scores)
+                
+    return len(clusters)
+
+
+def get_sys_absolute(data_new, metric="human") -> Dict[str, float]:
+    import collections
+    import numpy as np
+
+    scores_new = collections.defaultdict(list)
+
+    systems = list(data_new[0]["scores"].keys())
+    for line in data_new:
+        for sys in systems:
+            scores_new[sys].append(line["scores"][sys][metric])
+
+    scores_new = {
+        sys: np.average(scores_new[sys])
+        for sys in systems
+    }
+
+    return scores_new
+
+
+def get_sys_ordering(data_new: List[Dict], metric="human"):
+    scores_new = get_sys_absolute(data_new, metric)
+
+    # sort to get ordering
+    scores_new = list(scores_new.items())
+    # sort from highest
+    scores_new.sort(key=lambda x: x[1], reverse=True)
+
+    sys_ord = {
+        sys: sys_i
+        for sys_i, (sys, sys_v) in enumerate(scores_new)
+    }
+
+    return sys_ord
+
+
+def eval_order_accuracy(scores_new: Dict[str, float], scores_old: Dict[str, float]):
+    # evaluates against ordering from data_old
+    import itertools
+    import numpy as np
+
+    systems = list(scores_old.keys())
+
+    result = []
+    for sys1, sys2 in itertools.combinations(systems, 2):
+        result.append((scores_old[sys1] < scores_old[sys2]) == (scores_new[sys1] < scores_new[sys2]))
+
+    return np.average(result)
+
+
+def main_cli():
+    import argparse
+
+    args = argparse.ArgumentParser(
+        description="Meta-evaluate subset selection methods with cluster count and system accuracy."
+    )
+    args.add_argument(
+        'data_old', type=str, default='wmt23/en-cs',
+        help="Original data descriptor or path."
+    )
+    args.add_argument(
+        'data_new', type=str, default='wmt23/en-cs',
+        help="Path to new ordered data."
+    )
+    args.add_argument(
+        '--metric', type=str, default='human',
+        help="Metric to evaluate against, e.g., human or human_consistency. Can also be a metric and not human score."
+    )
+    args = args.parse_args()
+
+    clu_new, acc_new = run_evaluate_cluacc(args.data_old, args.data_new, args.metric)
+
+    print(f"Clusters: {np.average(clu_new):.2f}")
+    print(f"Accuracy: {np.average(acc_new):.1%}")

subset2evaluate/methods.py

@@ -0,0 +1,363 @@
+from typing import Any, List, Tuple, Union
+from functools import partial
+import numpy as np
+import os
+os.environ["TOKENIZERS_PARALLELISM"] = "false"
+
+
+def random_subset(data, seed=None, **kwargs) -> List[float]:
+    import random
+    r = random.Random(seed)
+    return [r.random() for _ in data]
+
+
+def metric_avg(data, metric, **kwargs) -> List[float]:
+    return [
+        -np.average([sys_v[metric] for sys_v in item["scores"].values()])
+        for item in data
+    ]
+
+
+def metric_var(data, metric, **kwargs) -> List[float]:
+    return [
+        np.var([sys_v[metric] for sys_v in item["scores"].values()])
+        for item in data
+    ]
+
+
+def _fn_information_content(item_old, item_irt, data_irt) -> float:
+    information = 0
+    for theta in data_irt["systems"].values():
+        x1 = np.exp(item_irt["disc"] * (theta + item_irt["diff"]))
+        x2 = np.exp(item_irt["disc"] * item_irt["diff"])
+        x3 = np.exp(item_irt["disc"] * theta)
+        information += (item_irt["disc"]**2) * x1 / (x2 + x3)**2
+    return information
+
+
+def fn_irt_utility(item_old, item_irt, data_irt, fn_utility) -> float:
+    if fn_utility == "fisher_information_content":
+        return _fn_information_content(item_old, item_irt, data_irt)
+    elif fn_utility == "diff":
+        return -item_irt["diff"]
+    elif fn_utility == "disc":
+        return -item_irt["disc"]
+    elif fn_utility == "diffdisc":
+        return item_irt["diff"] * item_irt["disc"]
+    elif fn_utility == "feas":
+        return item_irt["feas"]
+
+
+def pyirt(data, metric, return_model=False, load_model=None, model="4pl_score", dropout=0.25, epochs=1000, enforce_positive_disc=False, **kwargs) -> Union[List[float], Tuple[List[float], Any]]:
+    import py_irt
+    import py_irt.config
+    import py_irt.dataset
+    import py_irt.io
+    import py_irt.training
+    import py_irt.models
+    import py_irt.models.abstract_model
+    import pandas as pd
+
+    if model not in py_irt.models.abstract_model._IRT_REGISTRY:
+        raise Exception("Please install py-irt with `pip install git+https://github.com/zouharvi/py-irt.git")
+
+    systems = list(data[0]["scores"].keys())
+
+    if load_model is not None:
+        data_irt = load_model
+    else:
+        # we need median binarization if we are not using 4pl_score model
+        median = np.median([
+            system_v[metric]
+            for line in data
+            for system_v in line["scores"].values()
+        ])
+        dataset = pd.DataFrame({
+            "system": systems,
+            **{
+                f"item_{line['i']}": [
+                    line["scores"][system][metric]
+                    if "_score" in model else
+                    line["scores"][system][metric] >= median
+                    for system in systems
+                ]
+                for line in data
+            }
+        })
+
+        embeddings = None
+        if "amortized_" in model:
+            import sentence_transformers
+            embd_model = sentence_transformers.SentenceTransformer("paraphrase-MiniLM-L12-v2")
+            embeddings = embd_model.encode([line["src"] for line in data])
+            embeddings = {f"item_{line['i']}": emb.tolist() for line, emb in zip(data, embeddings)}
+            del embd_model
+
+        dataset = py_irt.dataset.Dataset.from_pandas(
+            dataset,
+            subject_column="system",
+            item_columns=[f"item_{line['i']}" for line in data],
+            embeddings=embeddings,
+        )
+
+        config = py_irt.config.IrtConfig(
+            model_type=model,
+            log_every=100,
+            dropout=dropout,
+            priors="hiearchical",
+            seed=0,
+            deterministic=True,
+        )
+        trainer = py_irt.training.IrtModelTrainer(
+            config=config,
+            data_path=None,
+            dataset=dataset,
+            verbose=False
+        )
+        trainer.train(epochs=epochs, device='cuda')
+
+        params = trainer.best_params
+
+        # this flipping should not affect the predictions
+        if enforce_positive_disc and np.average(params["disc"]) < 0:
+            params["disc"] = -np.array(params["disc"])
+            params["ability"] = -np.array(params["ability"])
+            params["diff"] = -np.array(params["diff"])
+
+        # normalize naming
+        if "lambdas" in params:
+            params["feas"] = params.pop("lambdas")
+
+        # TODO: cross-check make sure that we do the predictions as the models were trained
+        if "feas" in params:
+            # 3PL/4PL
+            data_irt = {
+                "systems": {sys: sys_v for sys, sys_v in zip(systems, params["ability"])},
+                "items": [
+                    {"disc": disc, "diff": diff, "feas": feas}
+                    for disc, diff, feas in zip(
+                        params["disc"],
+                        params["diff"],
+                        params["feas"],
+                    )
+                ]
+            }
+        elif "disc" in params:
+            data_irt = {
+                "systems": {sys: sys_v for sys, sys_v in zip(systems, params["ability"])},
+                "items": [
+                    {"disc": disc, "diff": diff}
+                    for disc, diff in zip(
+                        params["disc"],
+                        params["diff"],
+                    )
+                ]
+            }
+        else:
+            data_irt = {
+                "systems": {sys: sys_v for sys, sys_v in zip(systems, params["ability"])},
+                "items": [
+                    {"diff": diff}
+                    for diff in params["diff"]
+                ]
+            }
+
+    scores = [
+        fn_irt_utility(item_old, item_irt, data_irt, kwargs["fn_utility"])
+        for item_old, item_irt in zip(data, data_irt["items"])
+    ]
+
+    if return_model:
+        return scores, data_irt
+    else:
+        return scores
+
+
+def _assert_comet_version():
+    import comet
+    if "HypothesislessRegression" not in dir(comet.models):
+        raise Exception("Please install COMET with `pip install git+https://github.com/zouharvi/comet-src.git`")
+
+
+def precomet(data, model_path, return_model=False, load_model=None, reverse=False, **kwargs) -> Union[List, Tuple[List, Any]]:
+    import os
+    prev_tqdm_setting = os.environ.get("TQDM_DISABLE", None)
+    os.environ["TQDM_DISABLE"] = "1"
+
+    import logging
+    import comet
+    import warnings
+
+    logging.disable(logging.INFO)
+    _assert_comet_version()
+
+    with warnings.catch_warnings(action="ignore"):
+        if load_model is not None:
+            model = load_model
+        elif os.path.exists(model_path):
+            model = comet.load_from_checkpoint(model_path)
+        else:
+            model = comet.load_from_checkpoint(comet.download_model(model_path))
+        scores = model.predict([
+            {"src": line["src"]}
+            for line in data
+        ], progress_bar=False).scores
+        if reverse:
+            scores = [-x for x in scores]
+    
+    logging.disable(logging.NOTSET)
+    if prev_tqdm_setting is not None:
+        os.environ["TQDM_DISABLE"] = prev_tqdm_setting
+    else:
+        os.environ.pop("TQDM_DISABLE")
+
+    if return_model:
+        return scores, model
+    else:
+        return scores
+
+
+def precomet_dual(data, model_path1, model_path2, return_model=False, load_model=None, reverse=False, **kwargs) -> Union[List, Tuple[List, Any]]:
+    import os
+    tqdm_disable_prev = os.environ.get("TQDM_DISABLE", None)
+    os.environ["TQDM_DISABLE"] = "1"
+
+    import comet
+    import warnings
+    import logging
+
+    logging.disable(logging.INFO)
+    _assert_comet_version()
+
+    with warnings.catch_warnings(action="ignore"):
+        if load_model is not None:
+            model1, model2 = load_model
+        else:
+            if os.path.exists(model_path1):
+                model1 = comet.load_from_checkpoint(model_path1)
+            else:
+                model1 = comet.load_from_checkpoint(comet.download_model(model_path1))
+
+            if os.path.exists(model_path2):
+                model2 = comet.load_from_checkpoint(model_path2)
+            else:
+                model2 = comet.load_from_checkpoint(comet.download_model(model_path2))
+        scores1 = model1.predict([
+            {"src": line["src"]}
+            for line in data
+        ], progress_bar=False).scores
+        scores2 = model2.predict([
+            {"src": line["src"]}
+            for line in data
+        ], progress_bar=False).scores
+
+    if reverse:
+        scores = [-s1 * s2 for s1, s2 in zip(scores1, scores2)]
+    else:
+        scores = [s1 * s2 for s1, s2 in zip(scores1, scores2)]
+    
+    logging.disable(logging.NOTSET)
+    if tqdm_disable_prev is not None:
+        os.environ["TQDM_DISABLE"] = tqdm_disable_prev
+    else:
+        os.environ.pop("TQDM_DISABLE")
+
+    if return_model:
+        return scores, (model1, model2)
+    else:
+        return scores
+
+
+def diversity_unigram(data, **kwargs) -> List[float]:
+    import itertools
+    import collections
+
+    def _f(line):
+        out = []
+        for text_a, text_b in itertools.combinations(line["tgt"].values(), 2):
+            text_a = collections.Counter(text_a.split())
+            text_b = collections.Counter(text_b.split())
+            if text_a.total() == 0 or text_b.total() == 0:
+                out.append(1)
+            else:
+                out.append(2 * (text_a & text_b).total() / (text_a.total() + text_b.total()))
+        return np.average(out)
+
+    # we prefer smallest similarity so flip
+    return [
+        -_f(line)
+        for line in data
+    ]
+
+
+def diversity_bleu(data, **kwargs) -> List[float]:
+    import itertools
+    import sacrebleu
+    metric = sacrebleu.metrics.BLEU(effective_order=True)
+
+    def _f(line):
+        return np.average([
+            metric.sentence_score(
+                text_a,
+                [text_b],
+            ).score
+            for text_a, text_b in itertools.product(line["tgt"].values(), line["tgt"].values())
+        ])
+
+    # we prefer smallest similarity so flip
+    return [
+        -_f(line)
+        for line in data
+    ]
+
+
+def diversity_chrf(data, **kwargs) -> List[float]:
+    import itertools
+    import sacrebleu
+    metric = sacrebleu.metrics.CHRF()
+
+    def _f(line):
+        return np.average([
+            metric.sentence_score(
+                text_a,
+                [text_b],
+            ).score
+            for text_a, text_b in itertools.product(line["tgt"].values(), line["tgt"].values())
+        ])
+
+    # we prefer smallest similarity so flip
+    return [
+        -_f(line)
+        for line in data
+    ]
+
+
+METHODS = {
+    "random": random_subset,
+    "metric_avg": metric_avg,
+    "metric_var": metric_var,
+    "diversity_bleu": diversity_bleu,
+    "diversity_chrf": diversity_chrf,
+    "diversity_unigram": diversity_unigram,
+
+    "pyirt_diff": partial(pyirt, fn_utility="diff"),
+    "pyirt_disc": partial(pyirt, fn_utility="disc"),
+    "pyirt_diffdisc": partial(pyirt, fn_utility="diffdisc"),
+    "pyirt_feas": partial(pyirt, fn_utility="feas"),
+    "pyirt_fic": partial(pyirt, fn_utility="fisher_information_content"),
+    "pyirt_experiment": partial(pyirt, fn_utility="experiment"),
+
+    "precomet_var": partial(precomet, model_path="zouharvi/PreCOMET-var", reverse=True),
+    "precomet_avg": partial(precomet, model_path="zouharvi/PreCOMET-avg", reverse=True),
+    "precomet_diversity": partial(precomet, model_path="zouharvi/PreCOMET-diversity", reverse=True),
+
+    "precomet_diff": partial(precomet, model_path="zouharvi/PreCOMET-diff", reverse=False),
+    "precomet_disc": partial(precomet, model_path="zouharvi/PreCOMET-disc", reverse=True),
+    "precomet_diffdisc_direct": partial(precomet, model_path="zouharvi/PreCOMET-diffdisc_direct", reverse=False),
+    "precomet_diffdisc": partial(
+        precomet_dual,
+        model_path1="zouharvi/PreCOMET-diff",
+        model_path2="zouharvi/PreCOMET-disc",
+        reverse=False,
+    ),
+}