pearmut 0.1.1__tar.gz → 0.1.2__tar.gz

{pearmut-0.1.1 → pearmut-0.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pearmut
-Version: 0.1.1
+Version: 0.1.2
 Summary: A tool for evaluation of model outputs, primarily MT.
 Author-email: Vilém Zouhar <vilem.zouhar@gmail.com>
 License: apache-2.0
@@ -63,7 +63,7 @@ One of the simplest ones, where each user has a pre-defined list of tasks (`task
 ```python
 {
   "info": {
-    "type": "task-based",
+    "assignment": "task-based",
     "template": "pointwise",
     "protocol_score": true,                 # we want scores [0...100] for each segment
     "protocol_error_spans": true,           # we want error spans
@@ -115,19 +115,19 @@ For the standard ones (ESA, DA, MQM), we expect each item to be a dictionary (co
 ... # definition of another item (document)
 ```
 
-We also support a super simple allocation of annotations (`task-single`, not yet ⚠️), where you simply pass a list of all examples to be evaluated and they are processed in parallel by all annotators:
+We also support a simple allocation where all annotators draw from the same pool (`single-stream`). Items are randomly assigned to annotators from the pool of unfinished items:
 ```python
 {
     "campaign_id": "my campaign 6",
     "info": {
-        "type": "task-single",
+        "assignment": "single-stream",
         "template": "pointwise",
         "protocol_score": True,                # collect scores
         "protocol_error_spans": True,          # collect error spans
         "protocol_error_categories": False,    # do not collect MQM categories, so ESA
-        "users": 50,
+        "num_users": 50,                       # number of annotators
     },
-    "data": [...], # list of all items
+    "data": [...], # list of all items (shared among all annotators)
 }
 ```
 
@@ -137,10 +137,10 @@ We also support dynamic allocation of annotations (`dynamic`, not yet ⚠️), w
 {
     "campaign_id": "my campaign 6",
     "info": {
-        "type": "dynamic",
+        "assignment": "dynamic",
         "template": "kway",
         "protocol_k": 5,
-        "users": 50,
+        "num_users": 50,
     },
     "data": [...], # list of all items
 }
@@ -185,7 +185,7 @@ To make changes locally, clone the repository and run the following, which will
 cd pearmut
 # watch the frontend for changes (in a separate terminal)
 npm install web/ --prefix web/
-npm run watch --prefix web/
+npm run build --prefix web/ # `watch` for rebuild on code change
 
 # install local package as editable
 pip3 install -e .

{pearmut-0.1.1 → pearmut-0.1.2}/README.md

@@ -42,7 +42,7 @@ One of the simplest ones, where each user has a pre-defined list of tasks (`task
 ```python
 {
   "info": {
-    "type": "task-based",
+    "assignment": "task-based",
     "template": "pointwise",
     "protocol_score": true,                 # we want scores [0...100] for each segment
     "protocol_error_spans": true,           # we want error spans
@@ -94,19 +94,19 @@ For the standard ones (ESA, DA, MQM), we expect each item to be a dictionary (co
 ... # definition of another item (document)
 ```
 
-We also support a super simple allocation of annotations (`task-single`, not yet ⚠️), where you simply pass a list of all examples to be evaluated and they are processed in parallel by all annotators:
+We also support a simple allocation where all annotators draw from the same pool (`single-stream`). Items are randomly assigned to annotators from the pool of unfinished items:
 ```python
 {
     "campaign_id": "my campaign 6",
     "info": {
-        "type": "task-single",
+        "assignment": "single-stream",
         "template": "pointwise",
         "protocol_score": True,                # collect scores
         "protocol_error_spans": True,          # collect error spans
         "protocol_error_categories": False,    # do not collect MQM categories, so ESA
-        "users": 50,
+        "num_users": 50,                       # number of annotators
     },
-    "data": [...], # list of all items
+    "data": [...], # list of all items (shared among all annotators)
 }
 ```
 
@@ -116,10 +116,10 @@ We also support dynamic allocation of annotations (`dynamic`, not yet ⚠️), w
 {
     "campaign_id": "my campaign 6",
     "info": {
-        "type": "dynamic",
+        "assignment": "dynamic",
         "template": "kway",
         "protocol_k": 5,
-        "users": 50,
+        "num_users": 50,
     },
     "data": [...], # list of all items
 }
@@ -164,7 +164,7 @@ To make changes locally, clone the repository and run the following, which will
 cd pearmut
 # watch the frontend for changes (in a separate terminal)
 npm install web/ --prefix web/
-npm run watch --prefix web/
+npm run build --prefix web/ # `watch` for rebuild on code change
 
 # install local package as editable
 pip3 install -e .

{pearmut-0.1.1 → pearmut-0.1.2}/pearmut.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pearmut
-Version: 0.1.1
+Version: 0.1.2
 Summary: A tool for evaluation of model outputs, primarily MT.
 Author-email: Vilém Zouhar <vilem.zouhar@gmail.com>
 License: apache-2.0
@@ -63,7 +63,7 @@ One of the simplest ones, where each user has a pre-defined list of tasks (`task
 ```python
 {
   "info": {
-    "type": "task-based",
+    "assignment": "task-based",
     "template": "pointwise",
     "protocol_score": true,                 # we want scores [0...100] for each segment
     "protocol_error_spans": true,           # we want error spans
@@ -115,19 +115,19 @@ For the standard ones (ESA, DA, MQM), we expect each item to be a dictionary (co
 ... # definition of another item (document)
 ```
 
-We also support a super simple allocation of annotations (`task-single`, not yet ⚠️), where you simply pass a list of all examples to be evaluated and they are processed in parallel by all annotators:
+We also support a simple allocation where all annotators draw from the same pool (`single-stream`). Items are randomly assigned to annotators from the pool of unfinished items:
 ```python
 {
     "campaign_id": "my campaign 6",
     "info": {
-        "type": "task-single",
+        "assignment": "single-stream",
         "template": "pointwise",
         "protocol_score": True,                # collect scores
         "protocol_error_spans": True,          # collect error spans
         "protocol_error_categories": False,    # do not collect MQM categories, so ESA
-        "users": 50,
+        "num_users": 50,                       # number of annotators
     },
-    "data": [...], # list of all items
+    "data": [...], # list of all items (shared among all annotators)
 }
 ```
 
@@ -137,10 +137,10 @@ We also support dynamic allocation of annotations (`dynamic`, not yet ⚠️), w
 {
     "campaign_id": "my campaign 6",
     "info": {
-        "type": "dynamic",
+        "assignment": "dynamic",
         "template": "kway",
         "protocol_k": 5,
-        "users": 50,
+        "num_users": 50,
     },
     "data": [...], # list of all items
 }
@@ -185,7 +185,7 @@ To make changes locally, clone the repository and run the following, which will
 cd pearmut
 # watch the frontend for changes (in a separate terminal)
 npm install web/ --prefix web/
-npm run watch --prefix web/
+npm run build --prefix web/ # `watch` for rebuild on code change
 
 # install local package as editable
 pip3 install -e .

{pearmut-0.1.1 → pearmut-0.1.2}/pearmut.egg-info/SOURCES.txt

@@ -8,12 +8,14 @@ pearmut.egg-info/entry_points.txt
 pearmut.egg-info/requires.txt
 pearmut.egg-info/top_level.txt
 server/app.py
+server/assignment.py
 server/cli.py
-server/protocols.py
 server/utils.py
 server/static/dashboard.bundle.js
 server/static/dashboard.html
 server/static/index.html
+server/static/listwise.bundle.js
+server/static/listwise.html
 server/static/pointwise.bundle.js
 server/static/pointwise.html
 server/static/assets/favicon.svg

{pearmut-0.1.1 → pearmut-0.1.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "pearmut"
-version = "0.1.1"
+version = "0.1.2"
 description = "A tool for evaluation of model outputs, primarily MT."
 readme = "README.md"
 license = { text = "apache-2.0" }

{pearmut-0.1.1 → pearmut-0.1.2}/server/app.py

@@ -8,7 +8,7 @@ from fastapi.responses import JSONResponse
 from fastapi.staticfiles import StaticFiles
 from pydantic import BaseModel
 
-from .protocols import get_next_item, reset_task, update_progress
+from .assignment import get_next_item, reset_task, update_progress
 from .utils import ROOT, load_progress_data, save_progress_data
 
 os.makedirs(f"{ROOT}/data/outputs", exist_ok=True)
@@ -111,19 +111,20 @@ async def _dashboard_data(request: DashboardDataRequest):
     if campaign_id not in progress_data:
         return JSONResponse(content={"error": "Unknown campaign ID"}, status_code=400)
 
-    progress_new = {
-        user_id: {
-            **user_val,
-            "total": len(tasks_data[campaign_id]["data"][user_id]),
-        } | (
-            # override if not privileged
-            {
-                "token_correct": None,
-                "token_incorrect": None,
-            } if not is_privileged else {}
-        )
-        for user_id, user_val in progress_data[campaign_id].items()
-    }
+    progress_new = {}
+    assignment = tasks_data[campaign_id]["info"]["assignment"]
+    if assignment not in ["task-based", "single-stream"]:
+        return JSONResponse(content={"error": "Unsupported campaign assignment type"}, status_code=400)
+
+    for user_id, user_val in progress_data[campaign_id].items():
+        # shallow copy
+        entry = dict(user_val)
+
+        if not is_privileged:
+            entry["token_correct"] = None
+            entry["token_incorrect"] = None
+
+        progress_new[user_id] = entry
 
     return JSONResponse(
         content={
@@ -190,19 +191,22 @@ async def _download_progress(
         return JSONResponse(content={"error": "Mismatched campaign_id and token count"}, status_code=400)
 
     output = {}
-    for campaign_id, campaign_id in enumerate(campaign_id):
-        if campaign_id not in progress_data:
-            return JSONResponse(content={"error": f"Unknown campaign ID {campaign_id}"}, status_code=400)
-        if token[campaign_id] != tasks_data[campaign_id]["token"]:
-            return JSONResponse(content={"error": f"Invalid token for campaign ID {campaign_id}"}, status_code=400)
+    for i, cid in enumerate(campaign_id):
+        if cid not in progress_data:
+            return JSONResponse(content={"error": f"Unknown campaign ID {cid}"}, status_code=400)
+        if token[i] != tasks_data[cid]["token"]:
+            return JSONResponse(content={"error": f"Invalid token for campaign ID {cid}"}, status_code=400)
 
-        output[campaign_id] = progress_data[campaign_id]
+        output[cid] = progress_data[cid]
 
     return JSONResponse(content=output, status_code=200)
 
+static_dir = f"{os.path.dirname(os.path.abspath(__file__))}/static/"
+if not os.path.exists(static_dir + "index.html"):
+    raise FileNotFoundError("Static directory not found. Please build the frontend first.")
 
 app.mount(
     "/",
-    StaticFiles(directory=f"{os.path.dirname(os.path.abspath(__file__))}/static/" , html=True, follow_symlink=True),
+    StaticFiles(directory=static_dir, html=True, follow_symlink=True),
     name="static",
 )

pearmut-0.1.2/server/assignment.py

@@ -0,0 +1,184 @@
+import random
+from typing import Any
+
+from fastapi.responses import JSONResponse
+
+
+def _completed_response(
+    progress_data: dict,
+    campaign_id: str,
+    user_id: str,
+) -> JSONResponse:
+    """Build a completed response with progress, time, and token."""
+    user_progress = progress_data[campaign_id][user_id]
+    # TODO: add check for data quality
+    is_ok = True
+    return JSONResponse(
+        content={
+            "status": "completed",
+            "progress": user_progress["progress"],
+            "time": user_progress["time"],
+            "token": user_progress["token_correct" if is_ok else "token_incorrect"],
+        },
+        status_code=200
+    )
+
+
+def get_next_item(
+    campaign_id: str,
+    user_id: str,
+    tasks_data: dict,
+    progress_data: dict,
+) -> JSONResponse:
+    """
+    Get the next item for the user in the specified campaign.
+    """
+    assignment = tasks_data[campaign_id]["info"]["assignment"]
+    if assignment == "task-based":
+        return get_next_item_taskbased(campaign_id, user_id, tasks_data, progress_data)
+    elif assignment == "single-stream":
+        return get_next_item_single_stream(campaign_id, user_id, tasks_data, progress_data)
+    elif assignment == "dynamic":
+        return get_next_item_dynamic(campaign_id, user_id, tasks_data, progress_data)
+    else:
+        return JSONResponse(content={"error": "Unknown campaign assignment type"}, status_code=400)
+
+
+def get_next_item_taskbased(
+    campaign_id: str,
+    user_id: str,
+    data_all: dict,
+    progress_data: dict,
+) -> JSONResponse:
+    """
+    Get the next item for task-based protocol.
+    """
+    user_progress = progress_data[campaign_id][user_id]
+    if all(user_progress["progress"]):
+        return _completed_response(progress_data, campaign_id, user_id)
+
+    # find first incomplete item
+    item_i = min([i for i, v in enumerate(user_progress["progress"]) if not v])
+    return JSONResponse(
+        content={
+            "status": "ok",
+            "progress": user_progress["progress"],
+            "time": user_progress["time"],
+            "info": {
+                "item_i": item_i,
+            } | {
+                k: v
+                for k, v in data_all[campaign_id]["info"].items()
+                if k.startswith("protocol")
+            },
+            "payload": data_all[campaign_id]["data"][user_id][item_i]},
+        status_code=200
+    )
+
+
+def get_next_item_dynamic(campaign_data: dict, user_id: str, progress_data: dict, data_all: dict):
+    raise NotImplementedError("Dynamic protocol is not implemented yet.")
+
+
+def get_next_item_single_stream(
+    campaign_id: str,
+    user_id: str,
+    data_all: dict,
+    progress_data: dict,
+) -> JSONResponse:
+    """
+    Get the next item for single-stream protocol.
+    In this mode, all users share the same pool of items.
+    Items are randomly selected from unfinished items.
+
+    Note: There is a potential race condition where multiple users could
+    receive the same item simultaneously. This is fine since we store all responses.
+    """
+    user_progress = progress_data[campaign_id][user_id]
+    progress = user_progress["progress"]
+
+    if all(progress):
+        return _completed_response(progress_data, campaign_id, user_id)
+
+    # find a random incomplete item
+    incomplete_indices = [i for i, v in enumerate(progress) if not v]
+    item_i = random.choice(incomplete_indices)
+
+    return JSONResponse(
+        content={
+            "status": "ok",
+            "time": user_progress["time"],
+            "progress": progress,
+            "info": {
+                "item_i": item_i,
+            } | {
+                k: v
+                for k, v in data_all[campaign_id]["info"].items()
+                if k.startswith("protocol")
+            },
+            "payload": data_all[campaign_id]["data"][item_i]},
+        status_code=200
+    )
+
+
+def _reset_user_time(progress_data: dict, campaign_id: str, user_id: str) -> None:
+    """Reset time tracking fields for a user."""
+    progress_data[campaign_id][user_id]["time"] = 0.0
+    progress_data[campaign_id][user_id]["time_start"] = None
+    progress_data[campaign_id][user_id]["time_end"] = None
+
+
+def reset_task(
+    campaign_id: str,
+    user_id: str,
+    tasks_data: dict,
+    progress_data: dict,
+) -> JSONResponse:
+    """
+    Reset the task progress for the user in the specified campaign.
+    """
+    assignment = tasks_data[campaign_id]["info"]["assignment"]
+    if assignment == "task-based":
+        progress_data[campaign_id][user_id]["progress"] = (
+            [False]*len(tasks_data[campaign_id]["data"][user_id])
+        )
+        _reset_user_time(progress_data, campaign_id, user_id)
+        return JSONResponse(content={"status": "ok"}, status_code=200)
+    elif assignment == "single-stream":
+        # for single-stream reset all progress
+        for uid in progress_data[campaign_id]:
+            progress_data[campaign_id][uid]["progress"] = (
+                [False]*len(tasks_data[campaign_id]["data"])
+            )
+        _reset_user_time(progress_data, campaign_id, user_id)
+        return JSONResponse(content={"status": "ok"}, status_code=200)
+    else:
+        return JSONResponse(content={"status": "error", "message": "Reset not supported for this assignment type"}, status_code=400)
+
+
+def update_progress(
+    campaign_id: str,
+    user_id: str,
+    tasks_data: dict,
+    progress_data: dict,
+    item_i: int,
+    payload: Any,
+) -> JSONResponse:
+    """
+    Log the user's response for the specified item in the campaign.
+    """
+    assignment = tasks_data[campaign_id]["info"]["assignment"]
+    if assignment == "task-based":
+        # even if it's already set it should be fine
+        progress_data[campaign_id][user_id]["progress"][item_i] = True
+        # TODO: log attention checks/quality?
+        return JSONResponse(content={"status": "ok"}, status_code=200)
+    elif assignment == "single-stream":
+        # progress all users
+        for uid in progress_data[campaign_id]:
+            progress_data[campaign_id][uid]["progress"][item_i] = True
+        return JSONResponse(content={"status": "ok"}, status_code=200)
+    elif assignment == "dynamic":
+        return JSONResponse(content={"status": "error", "message": "Dynamic protocol logging not implemented yet."}, status_code=400)
+    else:
+        return JSONResponse(content={"status": "error", "message": "Unknown campaign assignment type"}, status_code=400)

{pearmut-0.1.1 → pearmut-0.1.2}/server/cli.py

@@ -90,33 +90,41 @@ def _add_campaign(args_unknown):
         raise ValueError("Campaign data must contain 'info' field.")
     if "data" not in campaign_data:
         raise ValueError("Campaign data must contain 'data' field.")
-    if "type" not in campaign_data["info"]:
-        raise ValueError("Campaign 'info' must contain 'type' field.")
+    if "assignment" not in campaign_data["info"]:
+        raise ValueError("Campaign 'info' must contain 'assignment' field.")
     if "template" not in campaign_data["info"]:
         raise ValueError("Campaign 'info' must contain 'template' field.")
 
+    assignment = campaign_data["info"]["assignment"]
     # use random words for identifying users
     rng = random.Random(campaign_data["campaign_id"])
     rword = wonderwords.RandomWord(rng=rng)
-    if campaign_data["info"]["type"] == "task-based":
+    if assignment == "task-based":
         tasks = campaign_data["data"]
         if not isinstance(tasks, list):
-            raise ValueError("Task-based campaign 'data' must be a list of tasks.")
+            raise ValueError(
+                "Task-based campaign 'data' must be a list of tasks.")
         if not all(isinstance(task, list) for task in tasks):
-            raise ValueError("Each task in task-based campaign 'data' must be a list of items.")
-        amount = len(tasks)
-    elif campaign_data["info"]["type"] == "dynamic":
-        if "num_users" not in campaign_data:
-            raise ValueError("Dynamic campaigns must specify 'num_users'.")
+            raise ValueError(
+                "Each task in task-based campaign 'data' must be a list of items.")
+        num_users = len(tasks)
+    elif assignment == "single-stream":
+        tasks = campaign_data["data"]
+        if "num_users" not in campaign_data["info"]:
+            raise ValueError(
+                "Single-stream campaigns must specify 'num_users' in info.")
         if not isinstance(campaign_data["data"], list):
-            raise ValueError("Dynamic campaign 'data' must be a list of items.")
-        amount = campaign_data["num_users"]
+            raise ValueError(
+                "Single-stream campaign 'data' must be a list of items.")
+        num_users = campaign_data["info"]["num_users"]
+    elif assignment == "dynamic":
+        raise NotImplementedError(
+            "Dynamic campaign assignment is not yet implemented.")
     else:
-        raise ValueError(
-            f"Unknown campaign type: {campaign_data["info"]['type']}")
+        raise ValueError(f"Unknown campaign assignment type: {assignment}")
 
     user_ids = []
-    while len(user_ids) < amount:
+    while len(user_ids) < num_users:
         # generate random user IDs
         new_id = f"{rword.random_words(amount=1, include_parts_of_speech=['adjective'])[0]}-{rword.random_words(amount=1, include_parts_of_speech=['noun'])[0]}"
         if new_id not in user_ids:
@@ -126,10 +134,15 @@ def _add_campaign(args_unknown):
         for user_id in user_ids
     ]
 
-    campaign_data["data"] = {
-        user_id: task
-        for user_id, task in zip(user_ids, tasks)
-    }
+    # For task-based, data is a dict mapping user_id -> tasks
+    # For single-stream, data is a flat list (shared among all users)
+    if assignment == "task-based":
+        campaign_data["data"] = {
+            user_id: task
+            for user_id, task in zip(user_ids, tasks)
+        }
+    elif assignment == "single-stream":
+        campaign_data["data"] = tasks
 
     # generate a token for dashboard access if not present
     if "token" not in campaign_data:
@@ -139,7 +152,12 @@ def _add_campaign(args_unknown):
 
     user_progress = {
         user_id: {
-            "progress": [False]*len(campaign_data["data"][user_id]) if campaign_data["info"]["type"] == "task-based" else [],
+            # TODO: progress tracking could be based on the assignment type
+            "progress": (
+                [False]*len(campaign_data["data"][user_id]) if assignment == "task-based"
+                else [False]*len(campaign_data["data"]) if assignment == "single-stream"
+                else []
+            ),
             "time_start": None,
             "time_end": None,
             "time": 0,