pearmut 0.0.1__py3-none-any.whl

pearmut/cli.py

@@ -0,0 +1,150 @@
+import argparse
+import hashlib
+import json
+import os
+import urllib.parse
+
+from .utils import ROOT, load_progress_data
+
+os.makedirs(f"{ROOT}/data/tasks", exist_ok=True)
+load_progress_data(warn=None)
+
+
+def _run():
+    import uvicorn
+
+    from .run import app
+    uvicorn.run(
+        app,
+        host="127.0.0.1",
+        port=8001,
+        # reload=reload_enabled,
+        # log_level="info",
+        # app_dir="src",
+        # factory=False # factory=False means it expects 'app' to be a variable
+    )
+
+
+def _add_campaign(args_unknown):
+    import argparse
+    import random
+
+    import wonderwords
+
+    args = argparse.ArgumentParser()
+    args.add_argument('data_file', type=str,
+                      help='Path to the campaign data file')
+    args.add_argument("-o", "--overwrite", action="store_true",
+                      help="Overwrite existing campaign if it exists")
+    args = args.parse_args(args_unknown)
+
+    with open(args.data_file, 'r') as f:
+        campaign_data = json.load(f)
+
+    with open(f"{ROOT}/data/progress.json", "r") as f:
+        progress_data = json.load(f)
+
+    if campaign_data['campaign_id'] in progress_data and not args.overwrite:
+        print(
+            f"Campaign {campaign_data['campaign_id']} already exists.",
+            "Use -o to overwrite."
+        )
+        exit(1)
+
+    # use random words for identifying users
+    rng = random.Random(campaign_data["campaign_id"])
+    rword = wonderwords.RandomWord(rng=rng)
+    if campaign_data["info"]["type"] == "task-based":
+        tasks = campaign_data["data"]
+        amount = len(tasks)
+    elif campaign_data["info"]["type"] == "dynamic":
+        amount = campaign_data["num_users"]
+    else:
+        raise ValueError(
+            f"Unknown campaign type: {campaign_data["info"]['type']}")
+
+    user_ids = []
+    while len(user_ids) < amount:
+        new_id = f"{rword.random_words(amount=1, include_parts_of_speech=['adjective'])[0]}-{rword.random_words(amount=1, include_parts_of_speech=['noun'])[0]}"
+        if new_id not in user_ids:
+            user_ids.append(new_id)
+    user_ids = [
+        f"{user_id}-{rng.randint(0, 999):03d}"
+        for user_id in user_ids
+    ]
+
+    server_url = campaign_data["info"].get(
+        "url",
+        "127.0.0.1:8001",  # by default local server
+    ).removesuffix("/")
+
+    campaign_data["data"] = {
+        user_id: task
+        for user_id, task in zip(user_ids, tasks)
+    }
+
+    # generate a token for dashboard access if not present
+    if "token" not in campaign_data:
+        campaign_data["token"] = (
+            hashlib.sha256(random.randbytes(16)).hexdigest()[:10]
+        )
+
+    user_progress = {
+        user_id: {
+            "progress": [False]*len(campaign_data["data"][user_id]) if campaign_data["info"]["type"] == "task-based" else [],
+            "time_start": None,
+            "time_end": None,
+            "time": 0,
+            "url": (
+                f"{server_url}/{campaign_data["info"]["template"]}.html"
+                f"?campaign_id={urllib.parse.quote_plus(campaign_data['campaign_id'])}"
+                f"&user_id={user_id}"
+            ),
+            "token_correct": hashlib.sha256(random.randbytes(16)).hexdigest()[:10],
+            "token_incorrect": hashlib.sha256(random.randbytes(16)).hexdigest()[:10],
+        }
+        for user_id in user_ids
+    }
+
+    with open(f"{ROOT}/data/tasks/{campaign_data['campaign_id']}.json", "w") as f:
+        json.dump(campaign_data, f, indent=2, ensure_ascii=False)
+
+    progress_data[campaign_data['campaign_id']] = user_progress
+
+    with open(f"{ROOT}/data/progress.json", "w") as f:
+        json.dump(progress_data, f, indent=2, ensure_ascii=False)
+
+    print(
+        f"{server_url}/dashboard.html"
+        f"?campaign_id={urllib.parse.quote_plus(campaign_data['campaign_id'])}"
+        f"&token={campaign_data['token']}"
+    )
+    print("-"*10)
+    for user_id, user_val in user_progress.items():
+        # point to the protocol URL
+        print(user_val["url"])
+
+
+def main():
+    args = argparse.ArgumentParser()
+    args.add_argument('command', type=str, choices=['run', 'add', 'purge'])
+    args, args_unknown = args.parse_known_args()
+
+    if args.command == 'run':
+        _run()
+    elif args.command == 'add':
+        _add_campaign(args_unknown)
+    elif args.command == 'purge':
+        import shutil
+
+        confirm = input(
+            "Are you sure you want to purge all campaign data? This action cannot be undone. [y/n]"
+        )
+        if confirm.lower() == 'y':
+            shutil.rmtree(f"{ROOT}/data/tasks", ignore_errors=True)
+            shutil.rmtree(f"{ROOT}/data/outputs", ignore_errors=True)
+            if os.path.exists(f"{ROOT}/data/progress.json"):
+                os.remove(f"{ROOT}/data/progress.json")
+            print("All campaign data purged.")
+        else:
+            print("Cancelled.")

pearmut/model.py

@@ -0,0 +1,61 @@
+# ruff: noqa
+
+raise Exception("Deprecated")
+"""
+See scripts/models.py for a list of possible competition models.
+"""
+
+import json
+import os
+import random
+
+sys1, sys2 = random.sample(systems, 2)
+segment_registry[(sys1, sys2)] += 1
+# TODO: handle overflow better
+if segment_registry[(sys1, sys2)] >= len(data):
+    segment_registry[(sys1, sys2)] = 0
+
+line = data[segment_registry[(sys1, sys2)]]
+
+texts = [highlight_differences(a, b) for a, b in zip(
+    line["tgt_text"][sys1],
+    line["tgt_text"][sys2],
+)]
+
+return JSONResponse(content={
+    "doc_id": line["doc_id"],
+    # TODO: this is not good sentence splitting
+    "src": [line.replace(". ", ".<br><br>") for line in line["src_text"]],
+    "sys_a": sys1,
+    "out_a": [line_a.replace(". ", ".<br><br>") for line_a, line_b in texts],
+    "sys_b": sys2,
+    "out_b": [line_b.replace(". ", ".<br><br>") for line_a, line_b in texts],
+})
+
+class CompetitionModel():
+    def __init__(self, systems):
+        if os.path.exists("data/model_elo.json"):
+            with open("data/model_elo.json", "r") as f:
+                self.scores = json.load(f)
+        else:
+            print("Initializing new ELO model")
+            self.scores = {sys: [] for sys in systems}
+
+    def system_score(self, sys):
+        out = 1000
+        for opponent, result in self.scores[sys]:
+            out += opponent + result
+        return out/len(self.scores[sys]) if self.scores[sys] else out
+
+    def future_information(self, sys1, sys2):
+        pass
+
+    def record_result(self, sys1, sys2, result):
+        self.scores[sys1].append((self.system_score(sys2), 1600*result - 800))
+        self.scores[sys2].append((self.system_score(sys1), 1600*(1-result) - 800))
+
+        self.save()
+
+    def save(self):
+        with open("data/model_elo.json", "w") as f:
+            json.dump(self.scores, f)

pearmut/protocols.py

@@ -0,0 +1,108 @@
+from typing import Any
+from fastapi.responses import JSONResponse
+
+
+def get_next_item(
+    campaign_id: str,
+    user_id: str,
+    tasks_data: dict,
+    progress_data: dict,
+) -> JSONResponse:
+    if tasks_data[campaign_id]["info"]["type"] == "task-based":
+        return get_next_item_taskbased(campaign_id, user_id, tasks_data, progress_data)
+    elif tasks_data[campaign_id]["info"]["type"] == "dynamic":
+        return get_next_item_dynamic(campaign_id, user_id, tasks_data, progress_data)
+    else:
+        return JSONResponse(content={"error": "Unknown campaign type"}, status_code=400)
+
+
+def get_next_item_taskbased(
+    campaign_id: str,
+    user_id: str,
+    data_all: dict,
+    progress_data: dict,
+) -> JSONResponse:
+    if all(progress_data[campaign_id][user_id]["progress"]):
+        # all items completed
+        # TODO: add check for data quality
+        is_ok = True
+        return JSONResponse(
+            content={
+                "status": "completed",
+                "progress": {
+                    "completed": sum(progress_data[campaign_id][user_id]["progress"]),
+                    "time": progress_data[campaign_id][user_id]["time"],
+                    "total": len(data_all[campaign_id]["data"][user_id]),
+                },
+                "token":  progress_data[campaign_id][user_id]["token_correct" if is_ok else "token_incorrect"],
+            },
+            status_code=200
+        )
+
+    # find first incomplete item
+    item_i = min([i for i, v in enumerate(progress_data[campaign_id][user_id]["progress"]) if not v])
+    return JSONResponse(
+        content={
+            "status": "ok",
+            "progress": {
+                "completed": sum(progress_data[campaign_id][user_id]["progress"]),
+                "time": progress_data[campaign_id][user_id]["time"],
+                "total": len(data_all[campaign_id]["data"][user_id]),
+            },
+            "info": {
+                "status_message": data_all[campaign_id]["info"].get("status_message", ""),
+                "item_i": item_i,
+            } | {
+                k: v
+                for k, v in data_all[campaign_id]["info"].items()
+                if k.startswith("protocol")
+            },
+            "payload": data_all[campaign_id]["data"][user_id][item_i]},
+        status_code=200
+    )
+
+
+def get_next_item_dynamic(campaign_data: dict, user_id: str, progress_data: dict, data_all: dict):
+    raise NotImplementedError("Dynamic protocol is not implemented yet.")
+    pass
+
+
+def reset_task(
+    campaign_id: str,
+    user_id: str,
+    tasks_data: dict,
+    progress_data: dict,
+) -> JSONResponse:
+    if tasks_data[campaign_id]["info"]["type"] == "task-based":
+        progress_data[campaign_id][user_id]["progress"] = [False]*len(tasks_data[campaign_id]["data"][user_id])
+        progress_data[campaign_id][user_id]["time"] = 0.0
+        progress_data[campaign_id][user_id]["time_start"] = None
+        progress_data[campaign_id][user_id]["time_end"] = None
+        return JSONResponse(content={"status": "ok"}, status_code=200)
+    else:
+        progress_data[campaign_id][user_id]["progress"] = []
+        progress_data[campaign_id][user_id]["time"] = 0.0
+        progress_data[campaign_id][user_id]["time_start"] = None
+        progress_data[campaign_id][user_id]["time_end"] = None
+        return JSONResponse(content={"status": "ok"}, status_code=200)
+    
+
+
+def log_response(
+    campaign_id: str,
+    user_id: str,
+    tasks_data: dict,
+    progress_data: dict,
+    item_i: int,
+    payload: Any,
+) -> JSONResponse:
+    if tasks_data[campaign_id]["info"]["type"] == "task-based":
+        # even if it's already set it should be fine
+        progress_data[campaign_id][user_id]["progress"][item_i] = True
+        return JSONResponse(content={"status": "ok"}, status_code=200)
+    elif tasks_data[campaign_id]["info"]["type"] == "dynamic":
+        return JSONResponse(content={"status": "error", "message": "Dynamic protocol logging not implemented yet."}, status_code=400)
+    elif tasks_data[campaign_id]["info"]["type"] == "task-single":
+        return JSONResponse(content={"status": "error", "message": "Task-single protocol logging not implemented yet."}, status_code=400)
+    else:
+        return JSONResponse(content={"status": "error", "message": "Unknown campaign type"}, status_code=400)

pearmut/run.py

@@ -0,0 +1,217 @@
+import json
+import os
+import urllib
+from typing import Any
+
+from fastapi import FastAPI, Query
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import JSONResponse
+from fastapi.staticfiles import StaticFiles
+from pydantic import BaseModel
+from pynpm import NPMPackage
+
+from .protocols import get_next_item, reset_task, log_response
+from .utils import ROOT, load_progress_data, save_progress_data
+
+os.makedirs("data/outputs", exist_ok=True)
+
+# build frontend
+pkg = NPMPackage('src/web/package.json')
+pkg.install()
+pkg.run_script('build')
+
+app = FastAPI()
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+tasks_data = {}
+progress_data = load_progress_data(
+    warn="No progress.json found. Running, but no campaign will be available.")
+
+# load all tasks into data_all
+for campaign_id in progress_data.keys():
+    with open(f"{ROOT}/data/tasks/{campaign_id}.json", "r") as f:
+        tasks_data[campaign_id] = json.load(f)
+
+# print access dashboard URL for all campaigns
+print(
+    list(tasks_data.values())[0]["info"]["url"] + "/dashboard.html?" + "&".join([
+        f"campaign_id={urllib.parse.quote_plus(campaign_id)}&token={campaign_data["token"]}"
+        for campaign_id, campaign_data in tasks_data.items()
+    ])
+)
+
+
+class LogResponseRequest(BaseModel):
+    campaign_id: str
+    user_id: str
+    item_i: int
+    payload: Any
+
+
+@app.post("/log-response")
+async def _log_response(request: LogResponseRequest):
+    global progress_data
+
+    campaign_id = request.campaign_id
+    user_id = request.user_id
+
+    if campaign_id not in progress_data:
+        return JSONResponse(content={"error": "Unknown campaign ID"}, status_code=400)
+    if user_id not in progress_data[campaign_id]:
+        return JSONResponse(content={"error": "Unknown user ID"}, status_code=400)
+
+    with open(f"{ROOT}/data/outputs/{campaign_id}.jsonl", "a") as log_file:
+        log_file.write(json.dumps(request.payload, ensure_ascii=False) + "\n")
+
+    # if actions were submitted, we can log time data
+    if "actions" in request.payload:
+        times = [
+            x["time"] for x in request.payload["actions"]
+        ]
+        if progress_data[campaign_id][user_id]["time_start"] is None:
+            progress_data[campaign_id][user_id]["time_start"] = min(times)
+        progress_data[campaign_id][user_id]["time_end"] = max(times)
+        progress_data[campaign_id][user_id]["time"] += sum([
+            min(b - a, 60)
+            for a, b in zip(times, times[1:])
+        ])
+
+    log_response(campaign_id, user_id, tasks_data, progress_data, request.item_i, request.payload)
+    save_progress_data(progress_data)
+
+    return JSONResponse(content={"status": "ok"}, status_code=200)
+
+
+class NextItemRequest(BaseModel):
+    campaign_id: str
+    user_id: str
+
+
+@app.post("/get-next-item")
+async def _get_next_item(request: NextItemRequest):
+    campaign_id = request.campaign_id
+    user_id = request.user_id
+
+    if campaign_id not in progress_data:
+        return JSONResponse(content={"error": "Unknown campaign ID"}, status_code=400)
+    if user_id not in progress_data[campaign_id]:
+        return JSONResponse(content={"error": "Unknown user ID"}, status_code=400)
+
+    return get_next_item(
+        campaign_id,
+        user_id,
+        tasks_data,
+        progress_data,
+    )
+
+
+class DashboardDataRequest(BaseModel):
+    campaign_id: str
+    token: str | None = None
+
+
+@app.post("/dashboard-data")
+async def _dashboard_data(request: DashboardDataRequest):
+    campaign_id = request.campaign_id
+
+    is_privileged = (request.token == tasks_data[campaign_id]["token"])
+
+    if campaign_id not in progress_data:
+        return JSONResponse(content={"error": "Unknown campaign ID"}, status_code=400)
+
+    progress_new = {
+        user_id: {
+            **user_val,
+            "total": len(tasks_data[campaign_id]["data"][user_id]),
+        } | (
+            # override if not privileged
+            {
+                "token_correct": None,
+                "token_incorrect": None,
+            } if not is_privileged else {}
+        )
+        for user_id, user_val in progress_data[campaign_id].items()
+    }
+
+    return JSONResponse(
+        content={
+            "status": "ok",
+            "data": progress_new
+        },
+        status_code=200
+    )
+
+
+class ResetTaskRequest(BaseModel):
+    campaign_id: str
+    user_id: str
+    token: str
+
+
+@app.post("/reset-task")
+async def _reset_task(request: ResetTaskRequest):
+    # ruff: noqa: F841
+    campaign_id = request.campaign_id
+    user_id = request.user_id
+    token = request.token
+
+    if campaign_id not in progress_data:
+        return JSONResponse(content={"error": "Unknown campaign ID"}, status_code=400)
+    if token != tasks_data[campaign_id]["token"]:
+        return JSONResponse(content={"error": "Invalid token"}, status_code=400)
+    if user_id not in progress_data[campaign_id]:
+        return JSONResponse(content={"error": "Unknown user ID"}, status_code=400)
+
+    response = reset_task(campaign_id, user_id, tasks_data, progress_data)
+    save_progress_data(progress_data)
+    return response
+
+
+@app.get("/download-annotations")
+async def _download_annotations(
+    campaign_id: list[str] = Query(),
+    # NOTE: currently not checking tokens for progress download as it is non-destructive
+    # token: list[str] = Query()
+):
+
+    output = {}
+    for campaign_id in campaign_id:
+        output_path = f"{ROOT}/data/outputs/{campaign_id}.jsonl"
+        if campaign_id not in progress_data:
+            return JSONResponse(content={"error": f"Unknown campaign ID {campaign_id}"}, status_code=400)
+        if not os.path.exists(output_path):
+            output[campaign_id] = []
+        else:
+            with open(output_path, "r") as f:
+                output[campaign_id] = [json.loads(x) for x in f.readlines()]
+
+    return JSONResponse(content=output, status_code=200)
+
+
+@app.get("/download-progress")
+async def _download_progress(
+    campaign_id: list[str] = Query(),
+    token: list[str] = Query()
+):
+
+    if len(campaign_id) != len(token):
+        return JSONResponse(content={"error": "Mismatched campaign_id and token count"}, status_code=400)
+
+    output = {}
+    for campaign_id, campaign_id in enumerate(campaign_id):
+        if campaign_id not in progress_data:
+            return JSONResponse(content={"error": f"Unknown campaign ID {campaign_id}"}, status_code=400)
+        if token[campaign_id] != tasks_data[campaign_id]["token"]:
+            return JSONResponse(content={"error": f"Invalid token for campaign ID {campaign_id}"}, status_code=400)
+
+        output[campaign_id] = progress_data[campaign_id]
+
+    return JSONResponse(content=output, status_code=200)
+
+app.mount("/", StaticFiles(directory="src/static", html=True), name="static")

pearmut/utils.py

@@ -0,0 +1,48 @@
+import json
+import os
+
+ROOT = "."
+
+def highlight_differences(a, b):
+    """
+    Compares two strings and wraps their differences in HTML span tags.
+
+    Args:
+        a: The first string.
+        b: The second string.
+
+    Returns:
+        A tuple containing the two strings with their differences highlighted.
+    """
+    import difflib
+    # TODO: maybe on the level of words?
+    s = difflib.SequenceMatcher(None, a, b)
+    res_a, res_b = [], []
+    span_open = '<span class="difference">'
+    span_close = '</span>'
+
+    for tag, i1, i2, j1, j2 in s.get_opcodes():
+        if tag == 'equal' or (i2-i1 <= 2 and j2-j1 <= 2):
+            res_a.append(a[i1:i2])
+            res_b.append(b[j1:j2])
+        else:
+            if tag in ('replace', 'delete'):
+                res_a.append(f"{span_open}{a[i1:i2]}{span_close}")
+            if tag in ('replace', 'insert'):
+                res_b.append(f"{span_open}{b[j1:j2]}{span_close}")
+    
+    return "".join(res_a), "".join(res_b)
+
+
+def load_progress_data(warn: str | None = None):
+    if not os.path.exists(f"{ROOT}/data/progress.json"):
+        if warn is not None:
+            print(warn)
+        with open(f"{ROOT}/data/progress.json", "w") as f:
+            f.write(json.dumps({}))
+    with open(f"{ROOT}/data/progress.json", "r") as f:
+        return json.load(f)
+
+def save_progress_data(data):
+    with open(f"{ROOT}/data/progress.json", "w") as f:
+        json.dump(data, f, indent=2)

pearmut-0.0.1.dist-info/METADATA

@@ -0,0 +1,17 @@
+Metadata-Version: 2.4
+Name: pearmut
+Version: 0.0.1
+Summary: A tool for evaluation of model outputs, primarily MT.
+Author-email: Vilém Zouhar <vilem.zouhar@gmail.com>
+License: MIT
+Project-URL: Repository, https://github.com/zouharvi/pearmut
+Project-URL: Issues, https://github.com/zouharvi/pearmut/issues
+Keywords: evaluation,machine translation,human evaluation,annotation
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+Requires-Dist: fastapi>=0.110.0
+Requires-Dist: uvicorn>=0.29.0
+Requires-Dist: wonderwords>=3.0.0
+Requires-Dist: pynpm>=0.3.0
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"

pearmut-0.0.1.dist-info/RECORD

@@ -0,0 +1,10 @@
+pearmut/cli.py,sha256=xzENnTEhKnjDgjc7CbVVcpBZOwZGexfmdPgTHz4Me_g,4853
+pearmut/model.py,sha256=icC6uE9iQOtoPBtjlaTjCF9l7fb8qrTIlrGeM7k6m4M,1827
+pearmut/protocols.py,sha256=GoqWNXMGereEqxzN_5sZiKqdB3w-_UVR-wtx0J86KgA,4319
+pearmut/run.py,sha256=Y67mdY_v1SaqaHiBvx3QuBdFSCdBylXwBhG1DzaSrvo,6841
+pearmut/utils.py,sha256=6hfVenrVdGm1r-7uJIkWHhX9o0ztWjqPse_j_MqkgBw,1443
+pearmut-0.0.1.dist-info/METADATA,sha256=JfYfNDNZ9qlZn_WyJBl_Mda7pAs9sbSiBCljJETYDEg,621
+pearmut-0.0.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+pearmut-0.0.1.dist-info/entry_points.txt,sha256=eEA9LVWsS3neQbMvL_nMvEw8I0oFudw8nQa1iqxOiWM,45
+pearmut-0.0.1.dist-info/top_level.txt,sha256=CdgtUM-SKQDt6o5g0QreO-_7XTBP9_wnHMS1P-Rl5Go,8
+pearmut-0.0.1.dist-info/RECORD,,

pearmut-0.0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

pearmut-0.0.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+pearmut = pearmut.cli:main

pearmut-0.0.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+pearmut