peerannot 0.0.1.post39__tar.gz → 0.0.1.post41__tar.gz

{peerannot-0.0.1.post39/peerannot.egg-info → peerannot-0.0.1.post41}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: peerannot
-Version: 0.0.1.post39
+Version: 0.0.1.post41
 Summary: Crowdsourcing library
 Download-URL: https://github.com/peerannot/peerannot.git
 Maintainer: Tanguy Lefort

{peerannot-0.0.1.post39 → peerannot-0.0.1.post41}/README.rst

@@ -166,4 +166,4 @@ The ``dataset.py`` is not mandatory but is here to facilitate the dataset's inst
 .. |Documentation| image:: https://github.com/peerannot/peerannot.github.io/actions/workflows/deploy-jekyll.yml/badge.svg
    :target: https://peerannot.github.io
 .. |Codecov| image:: https://codecov.io/gh/peerannot/peerannot/graph/badge.svg?token=3U77QPSODB 
-   :target: https://codecov.io/gh/peerannot/peerannot
+   :target: https://codecov.io/gh/peerannot/peerannot

peerannot-0.0.1.post41/VERSION.txt

@@ -0,0 +1 @@
+0.0.1-41

{peerannot-0.0.1.post39 → peerannot-0.0.1.post41}/peerannot/models/__init__.py

@@ -6,7 +6,10 @@ from .aggregation.GLAD import GLAD
 from .aggregation.WDS import WDS
 from .aggregation.twothird import TwoThird
 from .aggregation.plantnet import PlantNet
+
 from .aggregation.Wawa import Wawa
+from .aggregation.IWMV import IWMV
+
 from .identification.WAUM_perworker import WAUM_perworker
 from .identification.WAUM import WAUM
 from .identification.AUM import AUM
@@ -26,6 +29,7 @@ agg_strategies = {
     "WDS": WDS,
     "PlantNet": PlantNet,
     "TwoThird": TwoThird,
+    "IWMV": IWMV,
     "Wawa": Wawa,
 }
 

peerannot-0.0.1.post39/peerannot/models/aggregation/Wawa.py → peerannot-0.0.1.post41/peerannot/models/aggregation/IWMV.py

@@ -4,16 +4,13 @@ Majority voting
 =========================
 Most answered label per task
 """
+
 from ..template import CrowdModel
 import numpy as np
 from tqdm.auto import tqdm
 
-# implementatio of
-# https://github.com/Toloka/crowd-kit/blob/v1.2.1/crowdkit/aggregation/classification/wawa.py
-# that takes into account large sparse datasets
-
 
-class Wawa(CrowdModel):
+class IWMV(CrowdModel):
     def __init__(self, answers, n_classes=2, sparse=False, **kwargs):
         super().__init__(answers)
         self.n_classes = n_classes
@@ -63,7 +60,7 @@ class Wawa(CrowdModel):
     def accuracy_by_mv(self):
         worker_score = np.zeros(self.n_workers)
         worker_n_ans = np.zeros(self.n_workers)
-        for (task_id, mv_lab) in zip(self.answers.keys(), self.ans):
+        for task_id, mv_lab in zip(self.answers.keys(), self.ans):
             for worker, lab in self.answers[task_id].items():
                 if int(lab) == mv_lab:
                     worker_score[int(worker)] += 1
@@ -76,10 +73,12 @@ class Wawa(CrowdModel):
         )
         self.worker_score = worker_score
 
-    def run(self, **kwargs):
+    def run(self, nb_iter=10, **kwargs):
         self.MV(weight=np.ones(self.n_workers))
         self.accuracy_by_mv()
-        self.MV(weight=self.worker_score)
+        for _ in range(nb_iter):
+            self.MV(weight=self.worker_score)
+            self.accuracy_by_mv()
 
     def get_answers(self):
         """Get labels obtained with majority voting aggregation

peerannot-0.0.1.post41/peerannot/models/aggregation/Wawa.py

@@ -0,0 +1,38 @@
+"""
+=========================
+Majority voting
+=========================
+Most answered label per task
+"""
+
+from ..aggregation.IWMV import IWMV
+import numpy as np
+from tqdm.auto import tqdm
+
+# implementatio of
+# https://github.com/Toloka/crowd-kit/blob/v1.2.1/crowdkit/aggregation/classification/wawa.py
+# that takes into account large sparse datasets
+
+
+class Wawa(IWMV):
+    def __init__(self, answers, n_classes=2, sparse=False, **kwargs):
+        super().__init__(answers, n_classes, sparse, **kwargs)
+
+    def run(self, **kwargs):
+        super().run(1, **kwargs)
+
+    def get_answers(self):
+        """Get labels obtained with majority voting aggregation
+
+        :return: Most answered labels per task
+        :rtype: numpy.ndarray
+        """
+        return np.array(self.ans)
+
+    def get_probas(self):
+        """Get labels obtained with majority voting aggregation
+
+        :return: Most answered labels per task
+        :rtype: numpy.ndarray
+        """
+        return self.get_answers()

peerannot-0.0.1.post41/peerannot/runners/datasets.py

@@ -0,0 +1,164 @@
+# based on https://benchopt.github.io CLI
+import click
+from pathlib import Path
+import numpy as np
+import importlib.util
+import inspect
+import sys
+
+datasets = click.Group(
+    name="Running peerannot datasets",
+    help="Commands related to datasets that can be used with the PeerAnnot library",
+)
+
+
+@datasets.command(
+    help="Install dataset from `.py` file",
+    epilog="""Each dataset is a folder with:
+
+    \b
+- name.py: python file containing how to download and format data
+- answers.json: json file containing each task voted labels
+- metadata.json: all metadata for dataset, at least the name, n_task and n_classes""",
+)
+@click.argument(
+    "path",
+    type=click.Path(),
+)
+@click.option(
+    "--no-task",
+    is_flag=True,
+    help="True if no task is associated with the dataset",
+    # (only an answers file as in the krippendorff example dataset)
+)
+@click.option(
+    "--answers-format",
+    default=0,
+    type=click.INT,
+    help="annotation file format",
+    # 0 == Rodrigues matrix format, 1 == JSON answers/worker format, 2 == JSON worker/answers format
+)
+@click.option(
+    "--answers",
+    default="",
+    type=click.Path(exists=False),
+    help="annotation file",
+)
+@click.option(
+    "--metadata",
+    default="",
+    type=click.Path(exists=False),
+    help="metadata information file",
+)
+@click.option(
+    "--label-names",
+    default="",
+    type=click.Path(exists=False),
+    help="path to label names files",
+)
+@click.option(
+    "--files-path",
+    default="",
+    type=click.Path(exists=False),
+    help="path to train filenames",
+)
+@click.option(
+    "--train-path",
+    default="",
+    type=click.Path(exists=False),
+    help="path to train data",
+)
+@click.option(
+    "--test-ground-truth-format",
+    default=-1,
+    type=click.INT,
+    help="annotation file format",
+    # 0 == Rodrigues matrix format, 1 == JSON answers/worker format, 2 == JSON worker/answers format
+)
+@click.option(
+    "--test-ground-truth",
+    default="",
+    type=click.Path(exists=False),
+    help="test ground truth file",
+)
+@click.option(
+    "--test-path",
+    default="",
+    type=click.Path(exists=False),
+    help="path to test data",
+)
+@click.option(
+    "--val-path",
+    default="",
+    type=click.Path(exists=False),
+    help="path to val data",
+)
+def install(
+    path,
+    no_task,
+    answers_format,
+    answers,
+    metadata,
+    label_names,
+    files_path,
+    train_path,
+    test_ground_truth_format,
+    test_ground_truth,
+    test_path,
+    val_path,
+):
+    """Download and install dataset
+
+    :param path: path to python file including the class with method `setfolders` to install the data
+    :type path: click.Path
+    """
+    pathFilename = Path(path).resolve().as_posix().split("/")[-1].split(".")[0]
+    if pathFilename == "customDataset":
+        if not no_task:
+            if train_path == "":
+                click.echo("Please provide a valid train path")
+                sys.exit(1)
+            if files_path == "":
+                click.echo("Please provide a valid filenames path")
+                sys.exit(1)
+        if answers == "":
+            click.echo("Please provide a valid answers file")
+            sys.exit(1)
+        if test_ground_truth_format == -1:
+            test_ground_truth_format = answers_format
+
+        spec = importlib.util.spec_from_file_location("dataset", path)
+        mydata = importlib.util.module_from_spec(spec)
+        sys.modules["dataset"] = mydata
+        spec.loader.exec_module(mydata)
+        mm = [
+            (name, cls)
+            for name, cls in inspect.getmembers(mydata, inspect.isclass)
+            if cls.__module__ == "dataset"
+        ][0]
+        df = mm[1]()
+        df.setfolders(
+            no_task,
+            answers_format,
+            answers,
+            metadata,
+            label_names,
+            files_path,
+            train_path,
+            test_ground_truth_format,
+            test_ground_truth,
+            test_path,
+            val_path,
+        )
+    else:
+        spec = importlib.util.spec_from_file_location("dataset", path)
+        mydata = importlib.util.module_from_spec(spec)
+        sys.modules["dataset"] = mydata
+        spec.loader.exec_module(mydata)
+        mm = [
+            (name, cls)
+            for name, cls in inspect.getmembers(mydata, inspect.isclass)
+            if cls.__module__ == "dataset"
+        ][0]
+        df = mm[1]()
+        df.setfolders()

{peerannot-0.0.1.post39 → peerannot-0.0.1.post41/peerannot.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: peerannot
-Version: 0.0.1.post39
+Version: 0.0.1.post41
 Summary: Crowdsourcing library
 Download-URL: https://github.com/peerannot/peerannot.git
 Maintainer: Tanguy Lefort

{peerannot-0.0.1.post39 → peerannot-0.0.1.post41}/peerannot.egg-info/SOURCES.txt

@@ -23,6 +23,7 @@ peerannot/models/agg_deep/__init__.py
 peerannot/models/aggregation/DS.py
 peerannot/models/aggregation/DS_clust.py
 peerannot/models/aggregation/GLAD.py
+peerannot/models/aggregation/IWMV.py
 peerannot/models/aggregation/MV.py
 peerannot/models/aggregation/NaiveSoft.py
 peerannot/models/aggregation/WDS.py
@@ -46,5 +47,6 @@ peerannot/runners/simulate.py
 peerannot/runners/train.py
 peerannot/training/__init__.py
 peerannot/training/load_data.py
+tests/test_filecreation.py
 tests/test_identification.py
 tests/test_models.py

peerannot-0.0.1.post41/tests/test_filecreation.py

@@ -0,0 +1,156 @@
+import numpy as np
+from pathlib import Path
+import os
+import os.path
+import json
+import shutil
+import subprocess
+
+
+dir_toydata = Path(__file__).parents[1] / "datasets" / "toy-data"
+
+
+def verify_json(path):
+    with open(path, "r") as f:
+        answers = json.load(f)
+
+    with open(dir_toydata / "answers.json", "r") as f:
+        answers_sure = json.load(f)
+
+    assert answers == answers_sure
+
+
+def test_custom_creation():
+    dir_train = dir_toydata / "train"
+    dir_test = dir_toydata / "test"
+    dir_val = dir_toydata / "val"
+    dir_temp = dir_toydata / "temp"
+
+    # Test the answers txt format (Rodrigues matrix format), with a test set and a val set
+    os.makedirs(dir_temp)
+    currentPath = Path(__file__).parents[1] / "datasets" / "customDataset.py"
+    result = subprocess.run(
+        [
+            "peerannot",
+            "install",
+            Path(__file__).parents[1] / "datasets" / "customDataset.py",
+            "--train-path",
+            dir_train,
+            "--test-path",
+            dir_test,
+            "--val-path",
+            dir_val,
+            "--answers",
+            dir_toydata / "answers.txt",
+            "--files-path",
+            dir_toydata / "filenames.txt",
+            "--label-names",
+            dir_toydata / "labelNames.txt",
+        ],
+        capture_output=True,
+        cwd=dir_temp,
+    )
+    nb_files = len([name for name in os.listdir(dir_temp)])
+    assert nb_files == 6
+
+    nb_labels = len([name for name in os.listdir(dir_temp / "train")])
+    assert nb_labels == 2
+
+    nb_data = len([name for name in os.listdir(dir_temp / "train" / "smiles")])
+    assert nb_data == 2
+
+    verify_json(dir_temp / "answers.json")
+    shutil.rmtree(dir_temp)  # cleanup
+
+    ############################################################################
+
+    # Test the Peerannot answers format (task->worker), with a train and no val and test set
+    os.makedirs(dir_temp)
+    result = subprocess.run(
+        [
+            "peerannot",
+            "install",
+            Path(__file__).parents[1] / "datasets" / "customDataset.py",
+            "--train-path",
+            dir_train,
+            "--answers-format",
+            "1",
+            "--files-path",
+            dir_toydata / "filenames.txt",
+            "--answers",
+            dir_toydata / "answers.json",
+            "--label-names",
+            dir_toydata / "labelNames.txt",
+        ],
+        capture_output=True,
+        cwd=dir_temp,
+    )
+    nb_files = len(
+        [
+            name
+            for name in os.listdir(dir_temp)
+            # if os.path.isfile(os.path.join(dir_temp, name))
+        ]
+    )
+    assert nb_files == 4
+    verify_json(dir_temp / "answers.json")
+    shutil.rmtree(dir_temp)  # cleanup
+
+    ############################################################################
+
+    # Test the Peerannot answers format (task->worker), with a test, train and no val set
+    os.makedirs(dir_temp)
+    result = subprocess.run(
+        [
+            "peerannot",
+            "install",
+            Path(__file__).parents[1] / "datasets" / "customDataset.py",
+            "--train-path",
+            dir_train,
+            "--test-path",
+            dir_test,
+            "--answers-format",
+            "1",
+            "--answers",
+            dir_toydata / "answers.json",
+            "--label-names",
+            dir_toydata / "labelNames.txt",
+            "--files-path",
+            dir_toydata / "filenames.txt",
+        ],
+        capture_output=True,
+        cwd=dir_temp,
+    )
+    verify_json(dir_temp / "answers.json")
+    shutil.rmtree(dir_temp)  # cleanup
+
+    ############################################################################
+
+    # Test the Peerannot inverse answers format (worker->task), with no sets
+    os.makedirs(dir_temp)
+    result = subprocess.run(
+        [
+            "peerannot",
+            "install",
+            Path(__file__).parents[1] / "datasets" / "customDataset.py",
+            "--answers-format",
+            "2",
+            "--answers",
+            dir_toydata / "answers_inversed.json",
+            "--no-task",
+        ],
+        capture_output=True,
+        cwd=dir_temp,
+    )
+    nb_files = len(
+        [
+            name
+            for name in os.listdir(dir_temp)
+            # if os.path.isfile(os.path.join(dir_temp, name))
+        ]
+    )
+    assert nb_files == 1
+
+    verify_json(dir_temp / "answers.json")
+    shutil.rmtree(dir_temp)  # cleanup
+    # assert True == False

{peerannot-0.0.1.post39 → peerannot-0.0.1.post41}/tests/test_models.py

@@ -19,7 +19,7 @@ def test_mv():
     from peerannot.models import MV
 
     for sparse_ in sparse:
-        mv = MV(ANSWERS, n_classses=2, sparse=sparse_)
+        mv = MV(ANSWERS, n_classes=2, sparse=sparse_)
         y = mv.get_answers()
         expected = np.array([1, 0, 1])
         assert all([e == y_ for e, y_ in zip(expected, y)])
@@ -55,12 +55,28 @@ def test_wawa():
 
     sparse = [True, False]
     for sparse_ in sparse:
-        wawa = Wawa(ANSWERS, n_classses=2, n_workers=4, sparse=sparse_)
+        wawa = Wawa(ANSWERS, n_classes=2, n_workers=4, sparse=sparse_)
         wawa.run()
         y = wawa.get_answers()
         expected = np.array([1, 0, 1])
         assert np.isclose((wawa.worker_score - np.array([1 / 2, 1, 1, 1 / 2])).sum(), 0)
         assert all([e == y_ for e, y_ in zip(expected, y)])
+        # assert True == False
+
+
+def test_IWMV():
+    from peerannot.models import IWMV
+
+    sparse = [True, False]
+    for sparse_ in sparse:
+        wawa_it = IWMV(ANSWERS, n_classes=2, n_workers=4, sparse=sparse_)
+        wawa_it.run(10)
+        y = wawa_it.get_answers()
+        expected = np.array([1, 0, 1])
+        assert np.isclose(
+            (wawa_it.worker_score - np.array([1 / 2, 1, 1, 1 / 2])).sum(), 0
+        )
+        assert all([e == y_ for e, y_ in zip(expected, y)])
 
 
 def test_twothird():
@@ -68,7 +84,7 @@ def test_twothird():
 
     sparse = [True, False]
     for sparse_ in sparse:
-        two = TwoThird(ANSWERS, n_classses=2, n_workers=4, sparse=sparse_)
+        two = TwoThird(ANSWERS, n_classes=2, n_workers=4, sparse=sparse_)
         y = two.get_answers()
         expected = np.array([1, 0, 1])
         assert all([e == y_ for e, y_ in zip(expected, y)])

peerannot-0.0.1.post39/VERSION.txt

@@ -1 +0,0 @@
-0.0.1-39

peerannot-0.0.1.post39/peerannot/runners/datasets.py

@@ -1,44 +0,0 @@
-# based on https://benchopt.github.io CLI
-import click
-from pathlib import Path
-import numpy as np
-import importlib.util
-import inspect
-import sys
-
-datasets = click.Group(
-    name="Running peerannot datasets",
-    help="Commands related to datasets that can be used with the PeerAnnot library",
-)
-
-
-@datasets.command(
-    help="Install dataset from `.py` file",
-    epilog="""Each dataset is a folder with:
-
-    \b
-- name.py: python file containing how to download and format data
-- answers.json: json file containing each task voted labels
-- metadata.json: all metadata for dataset, at least the name, n_task and n_classes""",
-)
-@click.argument(
-    "path",
-    type=click.Path(),
-)
-def install(path):
-    """Download and install dataset
-
-    :param path: path to python file including the class with method `setfolders` to install the data
-    :type path: click.Path
-    """
-    spec = importlib.util.spec_from_file_location("dataset", path)
-    mydata = importlib.util.module_from_spec(spec)
-    sys.modules["dataset"] = mydata
-    spec.loader.exec_module(mydata)
-    mm = [
-        (name, cls)
-        for name, cls in inspect.getmembers(mydata, inspect.isclass)
-        if cls.__module__ == "dataset"
-    ][0]
-    df = mm[1]()
-    df.setfolders()