pearmut 0.2.7__py3-none-any.whl → 0.2.9__py3-none-any.whl

pearmut/app.py

@@ -1,5 +1,7 @@
+import collections
 import json
 import os
+import statistics
 from typing import Any
 
 from fastapi import FastAPI, Query
@@ -12,6 +14,7 @@ from .assignment import get_i_item, get_next_item, reset_task, update_progress
 from .utils import (
     ROOT,
     check_validation_threshold,
+    get_db_log,
     load_progress_data,
     save_db_payload,
     save_progress_data,
@@ -191,6 +194,58 @@ async def _dashboard_data(request: DashboardDataRequest):
     )
 
 
+class DashboardResultsRequest(BaseModel):
+    campaign_id: str
+    token: str
+
+
+@app.post("/dashboard-results")
+async def _dashboard_results(request: DashboardResultsRequest):
+    campaign_id = request.campaign_id
+    token = request.token
+
+    if campaign_id not in progress_data:
+        return JSONResponse(content="Unknown campaign ID", status_code=400)
+    
+    # Check if token is valid
+    if token != tasks_data[campaign_id]["token"]:
+        return JSONResponse(content="Invalid token", status_code=400)
+
+    # Compute model scores from annotations
+    model_scores = collections.defaultdict(dict)
+    
+    # Iterate through all tasks to find items with 'model' field
+    log = get_db_log(campaign_id)
+    for entry in log:
+        if "item" not in entry or "annotations" not in entry:
+            continue
+        for item, annotation in zip(entry["item"], entry["annotations"]):
+            if "model" in item:
+                # pointwise
+                if "score" in annotation:
+                    # make sure to only keep the latest score for each item
+                    # json.dumps(item) creates a unique item key
+                    model_scores[item["model"]][json.dumps(item)] = annotation["score"]
+            elif "models" in item:
+                # listwise
+                for model, annotation_cand in zip(item["models"], annotation):
+                    if "score" in annotation_cand:
+                        model_scores[model][json.dumps(item)] = (
+                            annotation_cand["score"]
+                        )
+
+    results = [
+        {
+            "model": model,
+            "score": statistics.mean(scores.values()),
+            "count": len(scores),
+        }
+        for model, scores in model_scores.items()
+    ]
+    results.sort(key=lambda x: x["score"], reverse=True)
+    return JSONResponse(content=results, status_code=200)
+
+
 class ResetTaskRequest(BaseModel):
     campaign_id: str
     user_id: str

pearmut/cli.py

@@ -41,7 +41,9 @@ def _run(args_unknown):
             args.server + "/dashboard.html?" + "&".join([
                 f"campaign_id={urllib.parse.quote_plus(campaign_id)}&token={campaign_data["token"]}"
                 for campaign_id, campaign_data in tasks_data.items()
-            ])
+            ]),
+            # this is important to flush
+            flush=True,
         )
 
     uvicorn.run(
@@ -278,26 +280,24 @@ def _add_single_campaign(data_file, overwrite, server):
             )
         
         # Symlink path is based on the destination, stripping the 'assets/' prefix
-        symlink_path = f"{STATIC_DIR}/{assets_destination}"
+        symlink_path = f"{STATIC_DIR}/{assets_destination}".rstrip("/")
 
         # Remove existing symlink if present and we are overriding the same campaign
         if os.path.lexists(symlink_path):
             # Check if any other campaign is using this destination
             current_campaign_id = campaign_data['campaign_id']
-            tasks_dir = f"{ROOT}/data/tasks"
-            if os.path.exists(tasks_dir):
-                for task_file in os.listdir(tasks_dir):
-                    if task_file.endswith('.json'):
-                        other_campaign_id = task_file[:-5]
-                        if other_campaign_id != current_campaign_id:
-                            with open(f"{tasks_dir}/{task_file}", "r") as f:
-                                other_campaign = json.load(f)
-                            other_assets = other_campaign.get("info", {}).get("assets")
-                            if other_assets and isinstance(other_assets, dict):
-                                if other_assets.get("destination") == assets_destination:
-                                    raise ValueError(
-                                        f"Assets destination '{assets_destination}' is already used by campaign '{other_campaign_id}'."
-                                    )
+
+            for other_campaign_id in progress_data.keys():
+                if other_campaign_id == current_campaign_id:
+                    continue
+                with open(f"{ROOT}/data/tasks/{other_campaign_id}.json", "r") as f:
+                    other_campaign = json.load(f)
+                    other_assets = other_campaign.get("info", {}).get("assets")
+                    if other_assets:
+                        if other_assets.get("destination") == assets_destination:
+                            raise ValueError(
+                                f"Assets destination '{assets_destination}' is already used by campaign '{other_campaign_id}'."
+                            )
             # Only allow overwrite if it's the same campaign
             if overwrite:
                 os.remove(symlink_path)
@@ -305,7 +305,8 @@ def _add_single_campaign(data_file, overwrite, server):
                 raise ValueError(f"Assets destination '{assets_destination}' is already taken.")
         
         # Ensure the assets directory exists
-        os.makedirs(f"{STATIC_DIR}/assets", exist_ok=True)
+        # get parent of symlink_path dir
+        os.makedirs(os.path.dirname(symlink_path), exist_ok=True)
 
         os.symlink(assets_real_path, symlink_path, target_is_directory=True)
         print(f"Assets symlinked: {symlink_path} -> {assets_real_path}")
@@ -391,7 +392,7 @@ def main():
                 campaign_data = json.load(f)
             destination = campaign_data.get("info", {}).get("assets", {}).get("destination")
             if destination:
-                symlink_path = f"{STATIC_DIR}/{destination}"
+                symlink_path = f"{STATIC_DIR}/{destination}".rstrip("/")
                 if os.path.islink(symlink_path):
                     os.remove(symlink_path)
                     print(f"Assets symlink removed: {symlink_path}")
@@ -436,12 +437,9 @@ def main():
             )
             if confirm.lower() == 'y':
                 # Unlink all assets first
-                tasks_dir = f"{ROOT}/data/tasks"
-                if os.path.exists(tasks_dir):
-                    for task_file in os.listdir(tasks_dir):
-                        if task_file.endswith('.json'):
-                            campaign_id = task_file[:-5]
-                            _unlink_assets(campaign_id)
+                progress_data = load_progress_data()
+                for campaign_id in progress_data.keys():
+                    _unlink_assets(campaign_id)
                 shutil.rmtree(f"{ROOT}/data/tasks", ignore_errors=True)
                 shutil.rmtree(f"{ROOT}/data/outputs", ignore_errors=True)
                 if os.path.exists(f"{ROOT}/data/progress.json"):