pearmut 0.1.2__py3-none-any.whl → 0.1.3__py3-none-any.whl

pearmut/utils.py

@@ -3,6 +3,7 @@ import os
 
 ROOT = "."
 
+
 def highlight_differences(a, b):
     """
     Compares two strings and wraps their differences in HTML span tags.
@@ -30,7 +31,7 @@ def highlight_differences(a, b):
                 res_a.append(f"{span_open}{a[i1:i2]}{span_close}")
             if tag in ('replace', 'insert'):
                 res_b.append(f"{span_open}{b[j1:j2]}{span_close}")
-    
+
     return "".join(res_a), "".join(res_b)
 
 
@@ -43,6 +44,58 @@ def load_progress_data(warn: str | None = None):
     with open(f"{ROOT}/data/progress.json", "r") as f:
         return json.load(f)
 
+
 def save_progress_data(data):
     with open(f"{ROOT}/data/progress.json", "w") as f:
-        json.dump(data, f, indent=2)
+        json.dump(data, f, indent=2)
+
+
+_logs = {}
+
+
+def get_db_log(campaign_id: str) -> list[dict]:
+    """
+    Returns up to date log for the given campaign_id.
+    """
+    if campaign_id not in _logs:
+        # create a new one if it doesn't exist
+        log_path = f"{ROOT}/data/outputs/{campaign_id}.jsonl"
+        if os.path.exists(log_path):
+            with open(log_path, "r") as f:
+                _logs[campaign_id] = [
+                    json.loads(line) for line in f.readlines()
+                ]
+        else:
+            _logs[campaign_id] = []
+
+    return _logs[campaign_id]
+
+
+def get_db_log_item(campaign_id: str, user_id: str | None, item_i: int | None) -> list[dict]:
+    """
+    Returns the log item for the given campaign_id, user_id and item_i.
+    Can be empty.
+    """
+    log = get_db_log(campaign_id)
+    return [
+        entry for entry in log
+        if (
+            (user_id is None or entry.get("user_id") == user_id) and
+            (item_i is None or entry.get("item_i") == item_i)
+        )
+    ]
+
+
+def save_db_payload(campaign_id: str, payload: dict):
+    """
+    Saves the given payload to the log for the given campaign_id, user_id and item_i.
+    Saves both on disk and in-memory.
+    """
+
+    log_path = f"{ROOT}/data/outputs/{campaign_id}.jsonl"
+    with open(log_path, "a") as log_file:
+        log_file.write(json.dumps(payload, ensure_ascii=False,) + "\n")
+
+    log = get_db_log(campaign_id)
+    # copy to avoid mutation issues
+    log.append(payload)

{pearmut-0.1.2.dist-info → pearmut-0.1.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pearmut
-Version: 0.1.2
+Version: 0.1.3
 Summary: A tool for evaluation of model outputs, primarily MT.
 Author-email: Vilém Zouhar <vilem.zouhar@gmail.com>
 License: apache-2.0
@@ -23,7 +23,7 @@ Dynamic: license-file
 
 Pearmut is a **Platform for Evaluation and Reviewing of Multilingual Tasks**.
 It evaluates model outputs, primarily translation but also various other NLP tasks.
-Supports multimodality (text, video, audio, images) and a variety of annotation protocols (DA, ESA, MQM, paired ESA, etc).
+Supports multimodality (text, video, audio, images) and a variety of annotation protocols ([DA](https://aclanthology.org/N15-1124/), [ESA](https://aclanthology.org/2024.wmt-1.131/), [ESA<sup>AI</sup>](https://aclanthology.org/2025.naacl-long.255/), [MQM](https://doi.org/10.1162/tacl_a_00437), paired ESA, etc).
 
 [![PyPi version](https://badgen.net/pypi/v/pearmut/)](https://pypi.org/project/pearmut)
 &nbsp;
@@ -31,7 +31,7 @@ Supports multimodality (text, video, audio, images) and a variety of annotation
 &nbsp;
 [![PyPi license](https://badgen.net/pypi/license/pearmut/)](https://pypi.org/project/pearmut/)
 &nbsp;
-[![build status](https://github.com/zouharvi/pearmut/actions/workflows/ci.yml/badge.svg)](https://github.com/zouharvi/pearmut/actions/workflows/ci.yml)
+[![build status](https://github.com/zouharvi/pearmut/actions/workflows/test.yml/badge.svg)](https://github.com/zouharvi/pearmut/actions/workflows/test.yml)
 
 <img width="1000" alt="Screenshot of ESA/MQM interface" src="https://github.com/user-attachments/assets/f14c91a5-44d7-4248-ada9-387e95ca59d0" />
 
@@ -115,6 +115,38 @@ For the standard ones (ESA, DA, MQM), we expect each item to be a dictionary (co
 ... # definition of another item (document)
 ```
 
+## Pre-filled Error Spans (ESA<sup>AI</sup> Support)
+
+For workflows where you want to provide pre-filled error annotations (e.g., ESA<sup>AI</sup>), you can include an `error_spans` key in each item.
+These spans will be loaded into the interface as existing annotations that users can review, modify, or delete.
+
+```python
+{
+  "src": "The quick brown fox jumps over the lazy dog.",
+  "tgt": "Rychlá hnědá liška skáče přes líného psa.",
+  "error_spans": [
+    {
+      "start_i": 0,         # character index start (inclusive)
+      "end_i": 5,           # character index end (inclusive)
+      "severity": "minor",  # "minor", "major", "neutral", or null
+      "category": null      # MQM category string or null
+    },
+    {
+      "start_i": 27,
+      "end_i": 32,
+      "severity": "major",
+      "category": null
+    }
+  ]
+}
+```
+
+For **listwise** template, `error_spans` is a 2D array where each inner array corresponds to error spans for that candidate.
+
+See [examples/esaai_prefilled.json](examples/esaai_prefilled.json) for a complete example.
+
+## Single-stream Assignment
+
 We also support a simple allocation where all annotators draw from the same pool (`single-stream`). Items are randomly assigned to annotators from the pool of unfinished items:
 ```python
 {
@@ -138,7 +170,7 @@ We also support dynamic allocation of annotations (`dynamic`, not yet ⚠️), w
     "campaign_id": "my campaign 6",
     "info": {
         "assignment": "dynamic",
-        "template": "kway",
+        "template": "listwise",
         "protocol_k": 5,
         "num_users": 50,
     },
@@ -154,6 +186,25 @@ pearmut add my_campaign_4.json
 pearmut run
 ```
 
+## Campaign options
+
+In summary, you can select from the assignment types
+
+- `task-based`: each user has a predefined set of items
+- `single-stream`: all users are annotating together the same set of items
+- `dynamic`: WIP ⚠️
+
+and independently of that select your protocol template:
+
+- `pointwise`: evaluate a single output given a single output
+  - `protocol_score`: ask for score 0 to 100
+  - `protocol_error_spans`: ask for highlighting error spans
+  - `protocol_error_categories`: ask for highlighting error categories
+- `listwise`: evaluate multiple outputs at the same time given a single output ⚠️
+  - `protocol_score`: ask for score 0 to 100
+  - `protocol_error_spans`: ask for highlighting error spans
+  - `protocol_error_categories`: ask for highlighting error categories
+
 ## Campaign management
 
 When adding new campaigns or launching pearmut, a management link is shown that gives an overview of annotator progress but also an easy access to the annotation links or resetting the task progress (no data will be lost).
@@ -170,7 +221,7 @@ An intentionally incorrect token can be shown if the annotations don't pass qual
 
 We also support anything HTML-compatible both on the input and on the output.
 This includes embedded YouTube videos, or even simple `<video ` tags that point to some resource somewhere.
-For an example, try [examples/mock_multimodal.json](examples/mock_multimodal.json).
+For an example, try [examples/multimodal.json](examples/multimodal.json).
 Tip: make sure the elements are already appropriately styled.
 
 <img width="800" alt="Preview of multimodal elements in Pearmut" src="https://github.com/user-attachments/assets/f34a1a3e-ad95-4114-95ee-8a49e8003faf" />

pearmut-0.1.3.dist-info/RECORD

@@ -0,0 +1,19 @@
+pearmut/app.py,sha256=ymRlnpKrWSiwdc51Tw4PBDDFFOY1bmdeU-xJ2VlOl-Q,7393
+pearmut/assignment.py,sha256=aOQNlGYzzPNgunAmIIwlcF4qY-l-w6Wmy7hGquArAsc,10623
+pearmut/cli.py,sha256=mV76uw6BywckbU7QEKIKTboukcALEdZp7l-kskJnBVA,7683
+pearmut/utils.py,sha256=gk8b4biPc9TTvZiQMQ_8xh1_FsWuwrhtPzeK3NpzhZc,2902
+pearmut/static/dashboard.bundle.js,sha256=6389gsHLCFh6JqiKdU3ng-Lm6VICRvfJgCSYM61H75U,91257
+pearmut/static/dashboard.html,sha256=tUP1yYvbKySRz0mxFtGq2Si4hTMhJkUCWeTpnq91Nf4,1789
+pearmut/static/index.html,sha256=ieCRLK83MVe-f-gtjYiOlvE-kKd8VnFF2xgyi6FoZpU,872
+pearmut/static/listwise.bundle.js,sha256=Qcz3TSA8C5QRFI-ui47y99WF87wf_4tMKHZ3TyfiYa8,103790
+pearmut/static/listwise.html,sha256=MNS4gV1Fqx7JXZikLhrWgL0z1OPdqgumlOfTcmGnXEI,5212
+pearmut/static/pointwise.bundle.js,sha256=doa3DC8n9L7IIV2ttWxV-TBKVMQHgjTQgSR3Pjozy3k,106133
+pearmut/static/pointwise.html,sha256=dhmfgpWvCFB833Y4kj08_aBZyCN33SayYcS1ckL2-FU,5009
+pearmut/static/assets/favicon.svg,sha256=gVPxdBlyfyJVkiMfh8WLaiSyH4lpwmKZs8UiOeX8YW4,7347
+pearmut/static/assets/style.css,sha256=-B-RySjt8qccqkwvLT0PDy6IRoE1xytLLKAFtR_S-Tg,3967
+pearmut-0.1.3.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+pearmut-0.1.3.dist-info/METADATA,sha256=XhlUE5eAzWzZ1MQX4RmPQuM5Kijk_LwYahgQvTbmmp4,10990
+pearmut-0.1.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+pearmut-0.1.3.dist-info/entry_points.txt,sha256=eEA9LVWsS3neQbMvL_nMvEw8I0oFudw8nQa1iqxOiWM,45
+pearmut-0.1.3.dist-info/top_level.txt,sha256=CdgtUM-SKQDt6o5g0QreO-_7XTBP9_wnHMS1P-Rl5Go,8
+pearmut-0.1.3.dist-info/RECORD,,

pearmut-0.1.2.dist-info/RECORD

@@ -1,19 +0,0 @@
-pearmut/app.py,sha256=s_xv7Nq9dm3ObApH_Iz9myS-H_q4oXsFKqwiwVbQYuY,6740
-pearmut/assignment.py,sha256=IgGXmZKFASoGW8jVeXXUN3meY8Two-Txwg4nMwZEOnA,6422
-pearmut/cli.py,sha256=mV76uw6BywckbU7QEKIKTboukcALEdZp7l-kskJnBVA,7683
-pearmut/utils.py,sha256=6hfVenrVdGm1r-7uJIkWHhX9o0ztWjqPse_j_MqkgBw,1443
-pearmut/static/dashboard.bundle.js,sha256=6389gsHLCFh6JqiKdU3ng-Lm6VICRvfJgCSYM61H75U,91257
-pearmut/static/dashboard.html,sha256=tUP1yYvbKySRz0mxFtGq2Si4hTMhJkUCWeTpnq91Nf4,1789
-pearmut/static/index.html,sha256=ieCRLK83MVe-f-gtjYiOlvE-kKd8VnFF2xgyi6FoZpU,872
-pearmut/static/listwise.bundle.js,sha256=_KWKocPZjkDHHoiixKFOZzmD0qlw-nqFheBPcbED0HM,100788
-pearmut/static/listwise.html,sha256=zipFfGus26qWEdFbuNQmaG-NR5S1yaczv2XpD8j843U,5203
-pearmut/static/pointwise.bundle.js,sha256=1mks6kD4P2w7uQqeze4GttKVc-JZvsLYKRktV6Em6R0,100431
-pearmut/static/pointwise.html,sha256=dhmfgpWvCFB833Y4kj08_aBZyCN33SayYcS1ckL2-FU,5009
-pearmut/static/assets/favicon.svg,sha256=gVPxdBlyfyJVkiMfh8WLaiSyH4lpwmKZs8UiOeX8YW4,7347
-pearmut/static/assets/style.css,sha256=-B-RySjt8qccqkwvLT0PDy6IRoE1xytLLKAFtR_S-Tg,3967
-pearmut-0.1.2.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-pearmut-0.1.2.dist-info/METADATA,sha256=cuHpmxeRqYF9H6s5ukP6RZBEx4tzy7bzipdhmbtIBVc,8923
-pearmut-0.1.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-pearmut-0.1.2.dist-info/entry_points.txt,sha256=eEA9LVWsS3neQbMvL_nMvEw8I0oFudw8nQa1iqxOiWM,45
-pearmut-0.1.2.dist-info/top_level.txt,sha256=CdgtUM-SKQDt6o5g0QreO-_7XTBP9_wnHMS1P-Rl5Go,8
-pearmut-0.1.2.dist-info/RECORD,,