pearmut 0.1.0__py3-none-any.whl → 0.1.2__py3-none-any.whl

pearmut/static/pointwise.html

@@ -28,10 +28,6 @@
       text-align: center;
     }
 
-    .output_block {
-      margin-bottom: 30pt;
-    }
-
     .output_labels {
       vertical-align: top;
       display: inline-block;
@@ -48,7 +44,7 @@
       width: 20px;
     }
 
-    /* on displays with less than 1100px */
+    /* on displays with less than 1000px */
     @media (max-width: 1000px) {
       .output_srctgt {
         width: 80%;
@@ -63,109 +59,6 @@
       }
     }
 
-    .tgt_char:hover {
-      background-color: #ccc;
-      cursor: pointer;
-      ;
-    }
-
-    .src_char:hover {
-      background-color: #ccc;
-    }
-
-    .error_unknown {
-      background-color: #ddf;
-    }
-
-    .error_neutral {
-      background-color: #edd;
-    }
-
-    .error_minor {
-      background-color: #fcc;
-    }
-
-    .error_major {
-      background-color: #e88;
-    }
-
-    .src_char.highlighted,
-    .tgt_char.highlighted {
-      background-color: #ccc;
-    }
-
-    #button_error_minor:hover,
-    #button_error_major:hover {
-      opacity: 0.8;
-      cursor: pointer;
-    }
-
-    .highlighted_active {
-      background-color: #aaf !important;
-    }
-
-    /* span toolbox section */
-    .span_toolbox {
-      background-color: white;
-      padding: 5px;
-      border-radius: 8px;
-      box-shadow: 0 4px 6px #0005;
-    }
-
-    .span_toolbox_parent {
-      position: absolute;
-      padding-left: 20px;
-      padding-right: 20px;
-      padding-top: 10px;
-      /* always maximally spread out children */
-      min-width: max-content;
-    }
-
-    input[type="button"].error_neutral {
-      background-color: #ecc9 !important;
-      width: 100%;
-      text-align: center;
-      border-radius: 8px;
-    }
-
-    input[type="button"].error_neutral:hover {
-      background-color: #ecc !important;
-    }
-
-    input[type="button"].error_minor {
-      background-color: #fcc !important;
-      width: 100%;
-      text-align: center;
-      border-radius: 8px;
-    }
-
-    input[type="button"].error_minor:hover {
-      background-color: #daa !important;
-    }
-
-    input[type="button"].error_major {
-      background-color: #e88 !important;
-      width: 100%;
-      text-align: center;
-      border-radius: 8px;
-    }
-
-    input[type="button"].error_major:hover {
-      background-color: #c66 !important;
-    }
-
-    input[type="button"].error_delete {
-      background-color: #ddd !important;
-      font-size: 10pt;
-      width: 100%;
-      text-align: center;
-      border-radius: 8px;
-    }
-
-    input[type="button"].error_delete:hover {
-      background-color: #ccc !important;
-    }
-
     input[type=range][orient=vertical] {
       position: relative;
       top: -5px;
@@ -173,63 +66,4 @@
       direction: rtl;
       width: 16px;
       height: 200px;
-    }
-
-    /* progress bar */
-    #progress span {
-      font-size: 0pt;
-      display: inline-block;
-      border-radius: 50%;
-      text-align: center;
-      line-height: 2em;
-      margin-left: 2px;
-      box-shadow: 0 1px 1px #0002;
-      width: 10px;
-      height: 10px;
-    }
-
-    #progress span:hover,
-    #progress span.progress_current,
-    #progress span.progress_incomplete:last-child {
-      cursor: pointer;
-      user-select: none;
-      position: relative;
-      top: 7.5px;
-      width: 22px;
-      height: 22px;
-      font-size: 8pt;
-      margin-left: -5px;
-      margin-right: -5px;
-      box-shadow: 0 1px 3px #0002;
-    }
-
-    #progress span:hover {
-      z-index: 100;
-    }
-
-    #progress span.progress_complete {
-      color: white;
-      background: #3b5238;
-    }
-
-
-    #progress span.progress_complete:hover {
-      background: #2e3e2b;
-    }
-
-    #progress span.progress_current {
-      background: #91b08d;
-    }
-
-    #progress span.progress_current:hover {
-      background: #739c6f;
-    }
-
-    #progress span.progress_incomplete {
-      background: #bbb;
-      color: #555;
-    }
-
-    #progress span.progress_incomplete:hover {
-      background: #aaa;
-    }</style><script defer="defer" src="pointwise.bundle.js"></script></head><body><div style="max-width: 1600px; min-width: 900px; margin-left: auto; margin-right: auto; margin-top: 20px; padding-left: 10px;"><div class="white-box" style="margin-right: 30px; background-color: #e7e2cf; padding: 5px 15px 5px 5px;"><span id="instructions_global" style="display: inline-block; font-size: 11pt; width: calc(100% - 170px);"><ul id="instructions_spans"><li>Error spans:<ul><li><strong>Select</strong> the part of translation where you have identified a <strong>translation error</strong> (drag or click start & end).</li><li><strong>Click</strong> on the highlight to change error severity (minor/major) or remove the highlight.</li></ul>Choose error severity:<ul><li><span class="instruction_sev" id="instruction_sev_minor">Minor errors:</span> Style, grammar, word choice could be better or more natural.</li><li><span class="instruction_sev" id="instruction_sev_major">Major errors:</span>: The meaning is changed significantly and/or the part is really hard to understand.</li></ul><strong>Tip</strong>: Highlight the word or general area of the error (it doesn't need to be exact). Use separate highlights for different errors.<br></li><li id="instructions_score">Score the translation: Please use the slider and set an overall score based on meaning preservation and general quality:</li><ul><li>0: <strong>No meaning preserved</strong>: most information is lost.</li><li>33%: <strong>Some meaning preserved</strong>: major gaps and narrative issues.</li><li>66%: <strong>Most meaning preserved</strong>: minor issues with grammar or consistency.</li><li>100%: <strong>Perfect</strong>: meaning and grammar align completely with the source.</li></ul><li id="instructions_categories">Error types: After highlighting an error fragment, you will be asked to select the specific error type (main category and subcategory). If you are unsure about which errors fall under which categories, please consult the <a href="https://themqm.org/the-mqm-typology/" style="font-weight: bold; text-decoration: underline;">typology definitions</a>.</li></ul></span><div style="width: 170px; display: inline-block; vertical-align: top; text-align: right; padding-top: 5px;"><span id="time" style="width: 135px; text-align: left; display: inline-block; font-size: 11pt;" title="Approximation of total annotation time.">Time: 0m</span> <input type="button" value="⚙️" id="button_settings" style="height: 1.5em; width: 30px;"><br><br><div id="progress" style="text-align: center;"></div><br><br><input type="button" value="Next 🛠️" id="button_next" disabled="disabled" style="width: 170px; height: 2.5em;" title="Finish annotating all examples first."> <input type="button" value="skip tutorial" id="button_skip_tutorial" style="width: 170px; font-size: 11pt; height: 30px; margin-top: 10px; display: none;" title="Skip tutorial only if you completed it already."></div></div><div id="settings_div" class="white-box" style="margin-right: 20px; margin-top: 10px; display: none; background-color: #e7e2cf; font-size: 11pt;"><input type="checkbox" id="settings_approximate_alignment"> <label for="settings_approximate_alignment">Show approximate alignment</label></div><div id="output_div" style="margin-top: 100px;"></div><br><br><br></div></body></html>
+    }</style><script defer="defer" src="pointwise.bundle.js"></script></head><body><div style="max-width: 1600px; min-width: 900px; margin-left: auto; margin-right: auto; margin-top: 20px; padding-left: 10px;"><div class="white-box" style="margin-right: 30px; background-color: #e7e2cf; padding: 5px 15px 5px 5px;"><span id="instructions_global" style="display: inline-block; font-size: 11pt; width: calc(100% - 170px);"><ul id="instructions_spans"><li>Error spans:<ul><li><strong>Select</strong> the part of translation where you have identified a <strong>translation error</strong> (drag or click start & end).</li><li><strong>Click</strong> on the highlight to change error severity (minor/major) or remove the highlight.</li></ul>Choose error severity:<ul><li><span class="instruction_sev" id="instruction_sev_minor">Minor errors:</span> Style, grammar, word choice could be better or more natural.</li><li><span class="instruction_sev" id="instruction_sev_major">Major errors:</span>: The meaning is changed significantly and/or the part is really hard to understand.</li></ul><strong>Tip</strong>: Highlight the word or general area of the error (it doesn't need to be exact). Use separate highlights for different errors.<br></li><li id="instructions_score">Score the translation: Please use the slider and set an overall score based on meaning preservation and general quality:</li><ul><li>0: <strong>No meaning preserved</strong>: most information is lost.</li><li>33%: <strong>Some meaning preserved</strong>: major gaps and narrative issues.</li><li>66%: <strong>Most meaning preserved</strong>: minor issues with grammar or consistency.</li><li>100%: <strong>Perfect</strong>: meaning and grammar align completely with the source.</li></ul><li id="instructions_categories">Error types: After highlighting an error fragment, you will be asked to select the specific error type (main category and subcategory). If you are unsure about which errors fall under which categories, please consult the <a href="https://themqm.org/the-mqm-typology/" style="font-weight: bold; text-decoration: none; color: black;">typology definitions</a>.</li></ul></span><div style="width: 170px; display: inline-block; vertical-align: top; text-align: right; padding-top: 5px;"><span id="time" style="width: 135px; text-align: left; display: inline-block; font-size: 11pt;" title="Approximation of total annotation time.">Time: 0m</span> <input type="button" value="⚙️" id="button_settings" style="height: 1.5em; width: 30px;"><br><br><div id="progress" style="text-align: center;"></div><br><br><input type="button" value="Next 🛠️" id="button_next" disabled="disabled" style="width: 170px; height: 2.5em;" title="Finish annotating all examples first."> <input type="button" value="skip tutorial" id="button_skip_tutorial" style="width: 170px; font-size: 11pt; height: 30px; margin-top: 10px; display: none;" title="Skip tutorial only if you completed it already."></div></div><div id="settings_div" class="white-box" style="margin-right: 20px; margin-top: 10px; display: none; background-color: #e7e2cf; font-size: 11pt;"><input type="checkbox" id="settings_approximate_alignment"> <label for="settings_approximate_alignment">Show approximate alignment</label></div><div id="output_div" style="margin-top: 100px;"></div><br><br><br></div></body></html>

{pearmut-0.1.0.dist-info → pearmut-0.1.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pearmut
-Version: 0.1.0
+Version: 0.1.2
 Summary: A tool for evaluation of model outputs, primarily MT.
 Author-email: Vilém Zouhar <vilem.zouhar@gmail.com>
 License: apache-2.0
@@ -63,7 +63,7 @@ One of the simplest ones, where each user has a pre-defined list of tasks (`task
 ```python
 {
   "info": {
-    "type": "task-based",
+    "assignment": "task-based",
     "template": "pointwise",
     "protocol_score": true,                 # we want scores [0...100] for each segment
     "protocol_error_spans": true,           # we want error spans
@@ -115,19 +115,19 @@ For the standard ones (ESA, DA, MQM), we expect each item to be a dictionary (co
 ... # definition of another item (document)
 ```
 
-We also support a super simple allocation of annotations (`task-single`, not yet ⚠️), where you simply pass a list of all examples to be evaluated and they are processed in parallel by all annotators:
+We also support a simple allocation where all annotators draw from the same pool (`single-stream`). Items are randomly assigned to annotators from the pool of unfinished items:
 ```python
 {
     "campaign_id": "my campaign 6",
     "info": {
-        "type": "task-single",
+        "assignment": "single-stream",
         "template": "pointwise",
         "protocol_score": True,                # collect scores
         "protocol_error_spans": True,          # collect error spans
         "protocol_error_categories": False,    # do not collect MQM categories, so ESA
-        "users": 50,
+        "num_users": 50,                       # number of annotators
     },
-    "data": [...], # list of all items
+    "data": [...], # list of all items (shared among all annotators)
 }
 ```
 
@@ -137,10 +137,10 @@ We also support dynamic allocation of annotations (`dynamic`, not yet ⚠️), w
 {
     "campaign_id": "my campaign 6",
     "info": {
-        "type": "dynamic",
+        "assignment": "dynamic",
         "template": "kway",
         "protocol_k": 5,
-        "users": 50,
+        "num_users": 50,
     },
     "data": [...], # list of all items
 }
@@ -185,7 +185,7 @@ To make changes locally, clone the repository and run the following, which will
 cd pearmut
 # watch the frontend for changes (in a separate terminal)
 npm install web/ --prefix web/
-npm run watch --prefix web/
+npm run build --prefix web/ # `watch` for rebuild on code change
 
 # install local package as editable
 pip3 install -e .

pearmut-0.1.2.dist-info/RECORD

@@ -0,0 +1,19 @@
+pearmut/app.py,sha256=s_xv7Nq9dm3ObApH_Iz9myS-H_q4oXsFKqwiwVbQYuY,6740
+pearmut/assignment.py,sha256=IgGXmZKFASoGW8jVeXXUN3meY8Two-Txwg4nMwZEOnA,6422
+pearmut/cli.py,sha256=mV76uw6BywckbU7QEKIKTboukcALEdZp7l-kskJnBVA,7683
+pearmut/utils.py,sha256=6hfVenrVdGm1r-7uJIkWHhX9o0ztWjqPse_j_MqkgBw,1443
+pearmut/static/dashboard.bundle.js,sha256=6389gsHLCFh6JqiKdU3ng-Lm6VICRvfJgCSYM61H75U,91257
+pearmut/static/dashboard.html,sha256=tUP1yYvbKySRz0mxFtGq2Si4hTMhJkUCWeTpnq91Nf4,1789
+pearmut/static/index.html,sha256=ieCRLK83MVe-f-gtjYiOlvE-kKd8VnFF2xgyi6FoZpU,872
+pearmut/static/listwise.bundle.js,sha256=_KWKocPZjkDHHoiixKFOZzmD0qlw-nqFheBPcbED0HM,100788
+pearmut/static/listwise.html,sha256=zipFfGus26qWEdFbuNQmaG-NR5S1yaczv2XpD8j843U,5203
+pearmut/static/pointwise.bundle.js,sha256=1mks6kD4P2w7uQqeze4GttKVc-JZvsLYKRktV6Em6R0,100431
+pearmut/static/pointwise.html,sha256=dhmfgpWvCFB833Y4kj08_aBZyCN33SayYcS1ckL2-FU,5009
+pearmut/static/assets/favicon.svg,sha256=gVPxdBlyfyJVkiMfh8WLaiSyH4lpwmKZs8UiOeX8YW4,7347
+pearmut/static/assets/style.css,sha256=-B-RySjt8qccqkwvLT0PDy6IRoE1xytLLKAFtR_S-Tg,3967
+pearmut-0.1.2.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+pearmut-0.1.2.dist-info/METADATA,sha256=cuHpmxeRqYF9H6s5ukP6RZBEx4tzy7bzipdhmbtIBVc,8923
+pearmut-0.1.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+pearmut-0.1.2.dist-info/entry_points.txt,sha256=eEA9LVWsS3neQbMvL_nMvEw8I0oFudw8nQa1iqxOiWM,45
+pearmut-0.1.2.dist-info/top_level.txt,sha256=CdgtUM-SKQDt6o5g0QreO-_7XTBP9_wnHMS1P-Rl5Go,8
+pearmut-0.1.2.dist-info/RECORD,,

pearmut/protocols.py

@@ -1,122 +0,0 @@
-from typing import Any
-
-from fastapi.responses import JSONResponse
-
-
-def get_next_item(
-    campaign_id: str,
-    user_id: str,
-    tasks_data: dict,
-    progress_data: dict,
-) -> JSONResponse:
-    """
-    Get the next item for the user in the specified campaign.
-    """
-    if tasks_data[campaign_id]["info"]["type"] == "task-based":
-        return get_next_item_taskbased(campaign_id, user_id, tasks_data, progress_data)
-    elif tasks_data[campaign_id]["info"]["type"] == "dynamic":
-        return get_next_item_dynamic(campaign_id, user_id, tasks_data, progress_data)
-    else:
-        return JSONResponse(content={"error": "Unknown campaign type"}, status_code=400)
-
-
-def get_next_item_taskbased(
-    campaign_id: str,
-    user_id: str,
-    data_all: dict,
-    progress_data: dict,
-) -> JSONResponse:
-    """
-    Get the next item for task-based protocol.
-    """
-    if all(progress_data[campaign_id][user_id]["progress"]):
-        # all items completed
-        # TODO: add check for data quality
-        is_ok = True
-        return JSONResponse(
-            content={
-                "status": "completed",
-                "progress": {
-                    "completed": sum(progress_data[campaign_id][user_id]["progress"]),
-                    "time": progress_data[campaign_id][user_id]["time"],
-                    "total": len(data_all[campaign_id]["data"][user_id]),
-                },
-                "token":  progress_data[campaign_id][user_id]["token_correct" if is_ok else "token_incorrect"],
-            },
-            status_code=200
-        )
-
-    # find first incomplete item
-    item_i = min([i for i, v in enumerate(progress_data[campaign_id][user_id]["progress"]) if not v])
-    return JSONResponse(
-        content={
-            "status": "ok",
-            "progress": {
-                "completed": sum(progress_data[campaign_id][user_id]["progress"]),
-                "time": progress_data[campaign_id][user_id]["time"],
-                "total": len(data_all[campaign_id]["data"][user_id]),
-            },
-            "info": {
-                "instructions": data_all[campaign_id]["info"].get("instructions", ""),
-                "item_i": item_i,
-            } | {
-                k: v
-                for k, v in data_all[campaign_id]["info"].items()
-                if k.startswith("protocol")
-            },
-            "payload": data_all[campaign_id]["data"][user_id][item_i]},
-        status_code=200
-    )
-
-
-def get_next_item_dynamic(campaign_data: dict, user_id: str, progress_data: dict, data_all: dict):
-    raise NotImplementedError("Dynamic protocol is not implemented yet.")
-    pass
-
-
-def reset_task(
-    campaign_id: str,
-    user_id: str,
-    tasks_data: dict,
-    progress_data: dict,
-) -> JSONResponse:
-    """
-    Reset the task progress for the user in the specified campaign.
-    """
-    if tasks_data[campaign_id]["info"]["type"] == "task-based":
-        progress_data[campaign_id][user_id]["progress"] = [False]*len(tasks_data[campaign_id]["data"][user_id])
-        progress_data[campaign_id][user_id]["time"] = 0.0
-        progress_data[campaign_id][user_id]["time_start"] = None
-        progress_data[campaign_id][user_id]["time_end"] = None
-        return JSONResponse(content={"status": "ok"}, status_code=200)
-    else:
-        progress_data[campaign_id][user_id]["progress"] = []
-        progress_data[campaign_id][user_id]["time"] = 0.0
-        progress_data[campaign_id][user_id]["time_start"] = None
-        progress_data[campaign_id][user_id]["time_end"] = None
-        return JSONResponse(content={"status": "ok"}, status_code=200)
-    
-
-
-def update_progress(
-    campaign_id: str,
-    user_id: str,
-    tasks_data: dict,
-    progress_data: dict,
-    item_i: int,
-    payload: Any,
-) -> JSONResponse:
-    """
-    Log the user's response for the specified item in the campaign.
-    """
-    if tasks_data[campaign_id]["info"]["type"] == "task-based":
-        # even if it's already set it should be fine
-        progress_data[campaign_id][user_id]["progress"][item_i] = True
-        # TODO: log attention checks/quality?
-        return JSONResponse(content={"status": "ok"}, status_code=200)
-    elif tasks_data[campaign_id]["info"]["type"] == "dynamic":
-        return JSONResponse(content={"status": "error", "message": "Dynamic protocol logging not implemented yet."}, status_code=400)
-    elif tasks_data[campaign_id]["info"]["type"] == "task-single":
-        return JSONResponse(content={"status": "error", "message": "Task-single protocol logging not implemented yet."}, status_code=400)
-    else:
-        return JSONResponse(content={"status": "error", "message": "Unknown campaign type"}, status_code=400)

pearmut-0.1.0.dist-info/RECORD

@@ -1,17 +0,0 @@
-pearmut/app.py,sha256=p438nGXTqdGHTfTA3DHRA7GF71DsBT77D3pcCCOW_Ow,6535
-pearmut/cli.py,sha256=PSF6JxAzGuV2ir6hdL6Q0Enf2G6KUgaf59_uS18LbZk,6929
-pearmut/protocols.py,sha256=qd0X2l9c2_x4YKtK8RswNHdwc7bUb59IP2-GlDA1_R0,4674
-pearmut/utils.py,sha256=6hfVenrVdGm1r-7uJIkWHhX9o0ztWjqPse_j_MqkgBw,1443
-pearmut/static/dashboard.bundle.js,sha256=d_uoUgbBaAzcnT-hTGhxmZ1GxtSvfCI1xw6c8F8AAMY,91230
-pearmut/static/dashboard.html,sha256=tUP1yYvbKySRz0mxFtGq2Si4hTMhJkUCWeTpnq91Nf4,1789
-pearmut/static/index.html,sha256=ieCRLK83MVe-f-gtjYiOlvE-kKd8VnFF2xgyi6FoZpU,872
-pearmut/static/pointwise.bundle.js,sha256=zdVJ2S9vBcd5-BHzbpCM-fCghAmB6pGB21lQE6butOc,100218
-pearmut/static/pointwise.html,sha256=F4mWSnUf9DPf7LGyzQTvI1MgKb84ov9NItWzoFn1eIU,8314
-pearmut/static/assets/favicon.svg,sha256=gVPxdBlyfyJVkiMfh8WLaiSyH4lpwmKZs8UiOeX8YW4,7347
-pearmut/static/assets/style.css,sha256=p1gu_IVTdJMsiW9UtFQwtpiQ-4XillAbU9XLw1KJeGA,964
-pearmut-0.1.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-pearmut-0.1.0.dist-info/METADATA,sha256=2SCD_wBHfpjAo1gr7Ygsc1ESvVKeZqkB2E_0wsIBGfM,8811
-pearmut-0.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-pearmut-0.1.0.dist-info/entry_points.txt,sha256=eEA9LVWsS3neQbMvL_nMvEw8I0oFudw8nQa1iqxOiWM,45
-pearmut-0.1.0.dist-info/top_level.txt,sha256=CdgtUM-SKQDt6o5g0QreO-_7XTBP9_wnHMS1P-Rl5Go,8
-pearmut-0.1.0.dist-info/RECORD,,