pearmut 1.0.1__py3-none-any.whl → 1.0.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (20) hide show
  1. pearmut/app.py +56 -25
  2. pearmut/assignment.py +340 -105
  3. pearmut/cli.py +185 -104
  4. pearmut/results_export.py +1 -1
  5. pearmut/static/annotate.bundle.js +1 -0
  6. pearmut/static/annotate.html +164 -0
  7. pearmut/static/dashboard.bundle.js +1 -1
  8. pearmut/static/dashboard.html +6 -1
  9. pearmut/static/index.html +1 -1
  10. pearmut/static/style.css +46 -0
  11. pearmut/utils.py +40 -21
  12. {pearmut-1.0.1.dist-info → pearmut-1.0.3.dist-info}/METADATA +119 -65
  13. pearmut-1.0.3.dist-info/RECORD +20 -0
  14. {pearmut-1.0.1.dist-info → pearmut-1.0.3.dist-info}/WHEEL +1 -1
  15. pearmut/static/basic.bundle.js +0 -1
  16. pearmut/static/basic.html +0 -133
  17. pearmut-1.0.1.dist-info/RECORD +0 -20
  18. {pearmut-1.0.1.dist-info → pearmut-1.0.3.dist-info}/entry_points.txt +0 -0
  19. {pearmut-1.0.1.dist-info → pearmut-1.0.3.dist-info}/licenses/LICENSE +0 -0
  20. {pearmut-1.0.1.dist-info → pearmut-1.0.3.dist-info}/top_level.txt +0 -0
pearmut/static/dashboard.html CHANGED
@@ -20,6 +20,11 @@
20
20
  cursor: pointer;
21
21
  }
22
22
 
23
+ .abutton[disabled] {
24
+ pointer-events: none;
25
+ opacity: 0.6;
26
+ }
27
+
23
28
  .ranking-content input[type="button"]:not(:last-child) {
24
29
  margin-right: 5px;
25
30
  }
@@ -93,4 +98,4 @@
93
98
  .white-box {
94
99
  overflow-x: auto;
95
100
  }
96
- }</style><script defer="defer" src="dashboard.bundle.js?0d289122fd490c931aec"></script><link href="style.css?0d289122fd490c931aec" rel="stylesheet"></head><body style="padding-top: 50px; padding-bottom: 50px;"><div id="main_div" style="width: calc(100% - 200px); min-width: 1300px; margin-left: auto; margin-right: auto;"><div style="margin-bottom: 15px; font-size: 0.9em; color: #555;">💤not started &nbsp;&nbsp; ✍️in progress &nbsp;&nbsp; ✅completed & passed &nbsp;&nbsp;❌completed & failed &nbsp;&nbsp; 🔗annotator link &nbsp;&nbsp; 👁️anotator link (view-only) &nbsp;&nbsp; 🗑️reset progress/campaign &nbsp;&nbsp; ⚖️show model ranking</div><div id="dashboard_div"></div><br><a class="abutton" id="download_progress" style="margin-right: 20px;">Download progress metadata 💾</a> <a class="abutton" id="download_annotations" style="margin-right: 20px;">Download all annotations 💾</a> <button class="abutton" id="add_campaign" style="width: 180px; margin-right: 20px;">Add new campaign 📤</button> <input type="file" id="campaign_file_input" accept=".json" style="display: none;"></div></body></html>
101
+ }</style><script defer="defer" src="dashboard.bundle.js?ee803f203a3b5ee1305d"></script><link href="style.css?ee803f203a3b5ee1305d" rel="stylesheet"></head><body style="padding-top: 50px; padding-bottom: 50px;"><div id="main_div" style="width: calc(100% - 200px); min-width: 1300px; margin-left: auto; margin-right: auto;"><div style="margin-bottom: 15px; font-size: 0.9em; color: #555;">💤not started &nbsp;&nbsp; ✍️in progress &nbsp;&nbsp; ✅completed & passed &nbsp;&nbsp;❌completed & failed &nbsp;&nbsp; 🔗annotator link &nbsp;&nbsp; 👁️anotator link (view-only) &nbsp;&nbsp; 🗑️reset progress/campaign &nbsp;&nbsp; ⚖️show model ranking</div><div id="dashboard_div"></div><br><a class="abutton" id="download_progress" style="margin-right: 20px;">Download progress metadata 💾</a> <a class="abutton" id="download_annotations" style="margin-right: 20px;">Download all annotations 💾</a> <button class="abutton" id="add_campaign" style="width: 180px; margin-right: 20px;">Add new campaign 📤</button> <input type="file" id="campaign_file_input" accept=".json" style="display: none;"></div></body></html>
pearmut/static/index.html CHANGED
@@ -1 +1 @@
1
- <!doctype html><html lang="en" style="height: 100%;"><head><meta charset="UTF-8"><meta name="viewport" content="width=900px"><title>Pearmut Evaluation</title><link rel="icon" type="image/svg+xml" href="favicon.svg"><script defer="defer" src="index.bundle.js?0d289122fd490c931aec"></script><link href="style.css?0d289122fd490c931aec" rel="stylesheet"></head><body><div class="white-box" style="width: max-content; font-size: large; position: absolute; top: 50%; left: 50%; transform: translate(-50%, -50%);">You have reached the Pearmut🍐 evaluation interface.<ul><li>If you are an annotator, you should have received a specialized link that takes you to the annotations.</li><li>If you are annotation manager, then you should distribute these links.</li></ul><br><br>See the <a href="https://github.com/zouharvi/pearmut">Pearmut project on GitHub</a>. Made with 💚 by Vilém Zouhar and others in 2025-2026.</div></body></html>
1
+ <!doctype html><html lang="en" style="height: 100%;"><head><meta charset="UTF-8"><meta name="viewport" content="width=900px"><title>Pearmut Evaluation</title><link rel="icon" type="image/svg+xml" href="favicon.svg"><script defer="defer" src="index.bundle.js?ee803f203a3b5ee1305d"></script><link href="style.css?ee803f203a3b5ee1305d" rel="stylesheet"></head><body><div class="white-box" style="width: max-content; font-size: large; position: absolute; top: 50%; left: 50%; transform: translate(-50%, -50%);">You have reached the Pearmut🍐 evaluation interface.<ul><li>If you are an annotator, you should have received a specialized link that takes you to the annotations.</li><li>If you are annotation manager, then you should distribute these links.</li></ul><br><br>See the <a href="https://github.com/zouharvi/pearmut">Pearmut project on GitHub</a>. Made with 💚 by Vilém Zouhar and others in 2025-2026.</div></body></html>
pearmut/static/style.css CHANGED
@@ -3,6 +3,12 @@ body {
3
3
  padding: 0;
4
4
  background: linear-gradient(135deg, #b9e2a1 0%, #e7e2cf 100%);
5
5
  background-attachment: fixed;
6
+
7
+ /* never rescale for phone */
8
+ text-size-adjust: none;
9
+ -webkit-text-size-adjust: none;
10
+ -ms-text-size-adjust: none;
11
+ -moz-text-size-adjust: none;
6
12
  }
7
13
 
8
14
  * {
@@ -63,6 +69,14 @@ label {
63
69
  margin-bottom: 30pt;
64
70
  }
65
71
 
72
+ .model_name {
73
+ font-weight: bold;
74
+ font-size: 0.9em;
75
+ color: #555;
76
+ margin-bottom: 5px;
77
+ padding: 2px 0;
78
+ }
79
+
66
80
  /* Character hover and highlighting */
67
81
  .tgt_char:hover {
68
82
  background-color: #ccc;
@@ -236,3 +250,35 @@ input[type="button"].error_delete:hover {
236
250
  .char_missing {
237
251
  font-family: monospace;
238
252
  }
253
+
254
+ /* Form styling */
255
+ .form-container {
256
+ max-width: 600px;
257
+ margin: 20px auto;
258
+ padding: 20px;
259
+ }
260
+
261
+ .form-field {
262
+ margin-bottom: 20px;
263
+ }
264
+
265
+ .form-label {
266
+ font-size: 14pt;
267
+ margin-bottom: 8px;
268
+ color: #333;
269
+ }
270
+
271
+ .form-input {
272
+ width: 100%;
273
+ padding: 10px;
274
+ font-size: 12pt;
275
+ border: 1px solid #ccc;
276
+ border-radius: 4px;
277
+ box-sizing: border-box;
278
+ }
279
+
280
+ .form-input:focus {
281
+ outline: none;
282
+ border-color: #4CAF50;
283
+ box-shadow: 0 0 5px rgba(76, 175, 80, 0.3);
284
+ }
pearmut/utils.py CHANGED
@@ -1,10 +1,13 @@
1
+ import hashlib
1
2
  import json
2
3
  import os
4
+ import random
3
5
 
4
6
  ROOT = "."
5
7
 
6
8
  # Sentinel value to indicate a task reset - masks all prior annotations
7
9
  RESET_MARKER = "__RESET__"
10
+ TOKEN_MAIN = hashlib.sha256(random.randbytes(16)).hexdigest()[:10]
8
11
 
9
12
 
10
13
  def load_progress_data(warn: str | None = None):
@@ -15,7 +18,7 @@ def load_progress_data(warn: str | None = None):
15
18
  f.write(json.dumps({}))
16
19
  with open(f"{ROOT}/data/progress.json", "r") as f:
17
20
  data = json.load(f)
18
-
21
+
19
22
  return data
20
23
 
21
24
 
@@ -36,42 +39,43 @@ def get_db_log(campaign_id: str) -> list[dict]:
36
39
  log_path = f"{ROOT}/data/outputs/{campaign_id}.jsonl"
37
40
  if os.path.exists(log_path):
38
41
  with open(log_path, "r") as f:
39
- _logs[campaign_id] = [
40
- json.loads(line) for line in f.readlines()
41
- ]
42
+ _logs[campaign_id] = [json.loads(line) for line in f.readlines()]
42
43
  else:
43
44
  _logs[campaign_id] = []
44
45
 
45
46
  return _logs[campaign_id]
46
47
 
47
48
 
48
- def get_db_log_item(campaign_id: str, user_id: str | None, item_i: int | None) -> list[dict]:
49
+ def get_db_log_item(
50
+ campaign_id: str, user_id: str | None, item_i: int | str | None
51
+ ) -> list[dict]:
49
52
  """
50
53
  Returns the log item for the given campaign_id, user_id and item_i.
51
54
  Can be empty. Respects reset markers - if a reset marker is found,
52
55
  only entries after the last reset are returned.
53
56
  """
54
57
  log = get_db_log(campaign_id)
55
-
58
+
56
59
  # Filter matching entries
57
60
  matching = [
58
- entry for entry in log
61
+ entry
62
+ for entry in log
59
63
  if (
60
- (user_id is None or entry.get("user_id") == user_id) and
61
- (item_i is None or entry.get("item_i") == item_i)
64
+ (user_id is None or entry.get("user_id") == user_id)
65
+ and (item_i is None or entry.get("item_i") == item_i)
62
66
  )
63
67
  ]
64
-
68
+
65
69
  # Find the last reset marker for this user (if any)
66
70
  last_reset_idx = -1
67
71
  for i, entry in enumerate(matching):
68
72
  if entry.get("annotation") == RESET_MARKER:
69
73
  last_reset_idx = i
70
-
74
+
71
75
  # Return only entries after the last reset
72
76
  if last_reset_idx >= 0:
73
- matching = matching[last_reset_idx + 1:]
74
-
77
+ matching = matching[last_reset_idx + 1 :]
78
+
75
79
  return matching
76
80
 
77
81
 
@@ -87,7 +91,13 @@ def save_db_payload(campaign_id: str, payload: dict):
87
91
  log_path = f"{ROOT}/data/outputs/{campaign_id}.jsonl"
88
92
  os.makedirs(os.path.dirname(log_path), exist_ok=True)
89
93
  with open(log_path, "a") as log_file:
90
- log_file.write(json.dumps(payload, ensure_ascii=False,) + "\n")
94
+ log_file.write(
95
+ json.dumps(
96
+ payload,
97
+ ensure_ascii=False,
98
+ )
99
+ + "\n"
100
+ )
91
101
 
92
102
  log.append(payload)
93
103
 
@@ -100,20 +110,20 @@ def check_validation_threshold(
100
110
  ) -> bool:
101
111
  """
102
112
  Check if user passes the validation threshold.
103
-
113
+
104
114
  The threshold is defined in campaign info as 'validation_threshold':
105
115
  - If integer: pass if number of failed checks <= threshold
106
- - If float in [0, 1): pass if proportion of failed checks <= threshold
116
+ - If float in [0, 1): pass if proportion of failed checks <= threshold
107
117
  - If float >= 1: always fail
108
118
  - If None/not set: defaults to 0 (fail on any failed check)
109
-
119
+
110
120
  Returns True if validation passes, False otherwise.
111
121
  """
112
122
  threshold = tasks_data[campaign_id]["info"].get("validation_threshold", 0)
113
-
123
+
114
124
  user_progress = progress_data[campaign_id][user_id]
115
125
  validations = user_progress.get("validations", {})
116
-
126
+
117
127
  # Count failed checks (validations is dict of item_i -> list of bools)
118
128
  total_checks = 0
119
129
  failed_checks = 0
@@ -126,11 +136,11 @@ def check_validation_threshold(
126
136
  # If no validation checks exist, pass
127
137
  if total_checks == 0:
128
138
  return True
129
-
139
+
130
140
  # Float >= 1: always fail
131
141
  if isinstance(threshold, float) and threshold >= 1:
132
142
  return False
133
-
143
+
134
144
  # Check threshold based on type
135
145
  if isinstance(threshold, float):
136
146
  # Float in [0, 1): proportion-based, pass if failed proportion <= threshold
@@ -138,3 +148,12 @@ def check_validation_threshold(
138
148
  else:
139
149
  # Integer: count-based, pass if failed count <= threshold
140
150
  return failed_checks <= threshold
151
+
152
+
153
+ def is_form_document(items):
154
+ """Check if a document contains form items instead of evaluation items."""
155
+ if not items:
156
+ return False
157
+ # Check if first item has 'text' and 'form' keys (form item)
158
+ first_item = items[0]
159
+ return "text" in first_item and "form" in first_item
{pearmut-1.0.1.dist-info → pearmut-1.0.3.dist-info}/METADATA RENAMED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: pearmut
3
- Version: 1.0.1
3
+ Version: 1.0.3
4
4
  Summary: A tool for evaluation of model outputs, primarily MT.
5
5
  Author-email: Vilém Zouhar <vilem.zouhar@gmail.com>
6
6
  License: MIT
@@ -19,7 +19,7 @@ Provides-Extra: dev
19
19
  Requires-Dist: pytest; extra == "dev"
20
20
  Dynamic: license-file
21
21
 
22
- # 🍐Pearmut &nbsp; &nbsp; [![PyPi version](https://badgen.net/pypi/v/pearmut/)](https://pypi.org/project/pearmut) [![PyPI download/month](https://img.shields.io/pypi/dm/pearmut.svg)](https://pypi.python.org/pypi/pearmut/) [![PyPi license](https://badgen.net/pypi/license/pearmut/)](https://pypi.org/project/pearmut/) [![build status](https://github.com/zouharvi/pearmut/actions/workflows/test.yml/badge.svg)](https://github.com/zouharvi/pearmut/actions/workflows/test.yml)
22
+ # 🍐Pearmut <br> [![PyPi version](https://badgen.net/pypi/v/pearmut/)](https://pypi.org/project/pearmut) [![PyPI download/month](https://img.shields.io/pypi/dm/pearmut.svg)](https://pypi.python.org/pypi/pearmut/) [![PyPi license](https://badgen.net/pypi/license/pearmut/)](https://pypi.org/project/pearmut/) [![build status](https://github.com/zouharvi/pearmut/actions/workflows/test.yml/badge.svg)](https://github.com/zouharvi/pearmut/actions/workflows/test.yml) [![arXiv](https://img.shields.io/badge/arXiv-2601.02933-b31b1b.svg?style=flat)](https://arxiv.org/abs/2601.02933)
23
23
 
24
24
  **Platform for Evaluation and Reviewing of Multilingual Tasks**: Evaluate model outputs for translation and NLP tasks with support for multimodal data (text, video, audio, images) and multiple annotation protocols ([DA](https://aclanthology.org/N15-1124/), [ESA](https://aclanthology.org/2024.wmt-1.131/), [ESA<sup>AI</sup>](https://aclanthology.org/2025.naacl-long.255/), [MQM](https://doi.org/10.1162/tacl_a_00437), and more!).
25
25
 
@@ -35,12 +35,15 @@ Dynamic: license-file
35
35
  - [Assignment Types](#assignment-types)
36
36
  - [Advanced Features](#advanced-features)
37
37
  - [Pre-filled Error Spans (ESA<sup>AI</sup>)](#pre-filled-error-spans-esaai)
38
+ - [Custom MQM Taxonomy](#custom-mqm-taxonomy)
38
39
  - [Tutorial and Attention Checks](#tutorial-and-attention-checks)
40
+ - [Form Items for User Metadata](#form-items-for-user-metadata)
39
41
  - [Pre-defined User IDs and Tokens](#pre-defined-user-ids-and-tokens)
40
42
  - [Multimodal Annotations](#multimodal-annotations)
41
43
  - [Hosting Assets](#hosting-assets)
42
44
  - [Campaign Management](#campaign-management)
43
45
  - [Custom Completion Messages](#custom-completion-messages)
46
+ - [Prolific Integration](#prolific-integration)
44
47
  - [CLI Commands](#cli-commands)
45
48
  - [Terminology](#terminology)
46
49
  - [Development](#development)
@@ -141,6 +144,22 @@ The `shuffle` parameter in campaign `info` controls this behavior:
141
144
  "data": [...]
142
145
  }
143
146
  ```
147
+ Documents in `data_welcome` are not shuffled and so don't require to have the same models in all documents.
148
+
149
+ ### Showing Model Names
150
+
151
+ By default, model names are hidden to avoid biasing annotators. To display model names on top of each output block, set `show_model_names` to `true`:
152
+ ```python
153
+ {
154
+ "info": {
155
+ "assignment": "task-based",
156
+ "protocol": "ESA",
157
+ "show_model_names": true # Default: false.
158
+ },
159
+ "campaign_id": "my_campaign",
160
+ "data": [...]
161
+ }
162
+ ```
144
163
 
145
164
  ### Custom Score Sliders
146
165
 
@@ -163,6 +182,52 @@ For multi-dimensional evaluation tasks (e.g., assessing fluency on a Likert scal
163
182
 
164
183
  When `sliders` is specified, only the custom sliders are shown. Each slider must have `name`, `min`, `max`, and `step` properties. All sliders must be answered before proceeding.
165
184
 
185
+ ### Textfield for Post-editing/Translation
186
+
187
+ Enable a textfield for post-editing or translation tasks using the `textfield` parameter in `info`. The textfield content is stored in annotations alongside scores and error spans.
188
+
189
+ ```python
190
+ {
191
+ "info": {
192
+ "protocol": "DA",
193
+ "textfield": "prefilled" # Options: null, "hidden", "visible", "prefilled"
194
+ }
195
+ }
196
+ ```
197
+
198
+ **Textfield modes:**
199
+ - `null` or omitted: No textfield (default)
200
+ - `"hidden"`: Textfield hidden by default, shown by clicking a button
201
+ - `"visible"`: Textfield always visible
202
+ - `"prefilled"`: Textfield visible and pre-filled with model output for post-editing
203
+
204
+ ### Custom MQM Taxonomy
205
+
206
+ For MQM protocol campaigns, you can define a custom error taxonomy instead of using the default MQM categories. Specify `mqm_categories` in the campaign `info` section as a dictionary mapping main categories to lists of subcategories:
207
+
208
+
209
+ ```python
210
+ {
211
+ "info": {
212
+ "assignment": "task-based",
213
+ "protocol": "MQM",
214
+ "mqm_categories": {
215
+ "": [], # Empty selection option
216
+ "General": ["", "Accuracy", "Fluency"],
217
+ "Audio-specific": ["", "Inaudible", "Background noise", "Speaker overlap", "Misinterpretation"],
218
+ "Style": ["", "Awkward", "Embarassing"],
219
+ "Unknown": [] # Category with no subcategories
220
+ }
221
+ },
222
+ "campaign_id": "custom_mqm_example",
223
+ "data": [...]
224
+ }
225
+ ```
226
+
227
+ If `mqm_categories` is not provided, the default MQM taxonomy will be used. The empty string key `""` provides an unselected state in the dropdown. Categories with empty subcategory lists (e.g., `"Style": []`) do not require a subcategory selection.
228
+
229
+ See [examples/custom_mqm.json](examples/custom_mqm.json) for a complete example.
230
+
166
231
  ### Custom Instructions
167
232
 
168
233
  Set campaign-level instructions using the `instructions` field in `info` (supports HTML).
@@ -252,6 +317,34 @@ The `score_greaterthan` field specifies the index of the candidate that must hav
252
317
  See [examples/tutorial/esa_deen.json](examples/tutorial/esa_deen.json) for a mock campaign with a fully prepared ESA tutorial.
253
318
  To use it, simply extract the `data` attribute and prefix it to each task in your campaign.
254
319
 
320
+ #### Universal Tutorial Items with `data_welcome`
321
+
322
+ Use `data_welcome` to add tutorial items that users must complete before starting regular tasks. The structure is a list of documents (same as `data`). Welcome items have IDs `welcome_0`, `welcome_1`, etc. and are tracked separately via `progress_welcome`.
323
+
324
+ ### Form Items for User Metadata
325
+
326
+ Collect user information (demographics, expertise) before annotation tasks using form items in `data_welcome`.
327
+ Form items have `text` (label/question) and `form` (field type: `null`, `"string"`, `"number"`, `"choices"`, and `"script"`).
328
+ Documents must be homogeneous: all form items or all evaluation items.
329
+
330
+ ```python
331
+ {
332
+ "data_welcome": [
333
+ [
334
+ {"text": "What is your native language?", "form": "string"},
335
+ {"text": "Rate your expertise (1-10)", "form": "number"}
336
+ ]
337
+ ]
338
+ }
339
+ ```
340
+
341
+ <img width="400" alt="Screenshot of a user form" src="https://github.com/user-attachments/assets/2310e8dc-98e9-4abf-8a27-6781b0094efe" />
342
+
343
+
344
+ It is possible to automatically collect additional information from the host system using `"script"` field type.
345
+ Typically such a form document (or their sequence) would be stored in `"data_welcome"` such that it is both mandatory and show to all users.
346
+ See [examples/user_info_form.json](examples/user_info_form.json).
347
+
255
348
  ### Single-stream Assignment
256
349
 
257
350
  All annotators draw from a shared pool with random assignment:
@@ -265,11 +358,14 @@ All annotators draw from a shared pool with random assignment:
265
358
  # ESA: error spans and scores
266
359
  "protocol": "ESA",
267
360
  "users": 50, # number of annotators (can also be a list, see below)
361
+ "docs_per_user": 10, # optional: show goodbye after N documents per user
268
362
  },
269
363
  "data": [...], # list of all items (shared among all annotators)
270
364
  }
271
365
  ```
272
366
 
367
+ Set `docs_per_user` to limit how many documents each user annotates before seeing the goodbye message (for single-stream, this is the number of documents).
368
+
273
369
  ### Dynamic Assignment
274
370
 
275
371
  The `dynamic` assignment type intelligently selects items based on current model performance to focus annotation effort on top-performing models using contrastive comparisons.
@@ -286,11 +382,14 @@ All items must contain outputs from all models for this assignment type to work
286
382
  "dynamic_contrastive_models": 2, # how many models to compare per item (optional, default: 1)
287
383
  "dynamic_first": 5, # annotations per model before dynamic kicks in (optional, default: 5)
288
384
  "dynamic_backoff": 0.1, # probability of uniform sampling (optional, default: 0)
385
+ "docs_per_user": 20, # optional: show goodbye after N documents per user
289
386
  },
290
387
  "data": [...], # list of all items (shared among all annotators)
291
388
  }
292
389
  ```
293
390
 
391
+ Set `docs_per_user` to limit how many documents each user annotates before seeing the goodbye message (for dynamic, this is roughly the number of documents × models).
392
+
294
393
  **How it works:**
295
394
  1. Initial phase: Each model gets `dynamic_first` annotations with fully random contrastive evaluation
296
395
  2. Dynamic phase: After the initial phase, top `dynamic_top` models (by average score) are identified
@@ -378,6 +477,14 @@ When tokens are supplied, the dashboard will try to show model rankings based on
378
477
 
379
478
  Customize the goodbye message shown to users when they complete all annotations using the `instructions_goodbye` field in campaign info. Supports arbitrary HTML for styling and formatting with variable replacement: `${TOKEN}` (completion token) and `${USER_ID}` (user ID). Default: `"If someone asks you for a token of completion, show them: ${TOKEN}"`.
380
479
 
480
+ ### Prolific Integration
481
+
482
+ Use task-based assignment with Prolific. For each task, Pearmut generates a unique URL which can be uploaded to Prolific's interface. Add redirect (on completion) to `instructions_goodbye`:
483
+ ```json
484
+ "instructions_goodbye": "<a href='https://app.prolific.com/submissions/complete?cc=${TOKEN}'>Click here to return to Prolific</a>"
485
+ ```
486
+ The `${TOKEN}` is automatically replaced based on passing attention checks (see [Attention checks](#tutorial-and-attention-checks) and [Pre-defined tokens](#pre-defined-user-ids-and-tokens)).
487
+
381
488
  ## Terminology
382
489
 
383
490
  - **Campaign**: An annotation project that contains configuration, data, and user assignments. Each campaign has a unique identifier and is defined in a JSON file.
@@ -401,7 +508,7 @@ Customize the goodbye message shown to users when they complete all annotations
401
508
  - **Score**: Numeric quality rating (0-100)
402
509
  - **Error Spans**: Text highlights marking errors with severity (`minor`, `major`)
403
510
  - **Error Categories**: MQM taxonomy labels for errors
404
- - **Template**: The annotation interface type. The `basic` template supports comparing multiple outputs simultaneously.
511
+ - **Template**: The annotation interface type. The `annotate` template supports comparing multiple outputs simultaneously.
405
512
  - **Assignment**: The method for distributing items to users:
406
513
  - **Task-based**: Each user has predefined items
407
514
  - **Single-stream**: Users draw from a shared pool with random assignment
@@ -432,7 +539,7 @@ pearmut run
432
539
  2. Add build rule to `webpack.config.js`
433
540
  3. Reference as `info->template` in campaign JSON
434
541
 
435
- See [web/src/basic.ts](web/src/basic.ts) for example.
542
+ See [web/src/annotate.ts](web/src/annotate.ts) for example.
436
543
 
437
544
  ### Deployment
438
545
 
@@ -443,68 +550,15 @@ Run on public server or tunnel local port to public IP/domain and run locally.
443
550
  If you use this work in your paper, please cite as following.
444
551
  ```bibtex
445
552
  @misc{zouhar2026pearmut,
446
- author = {Zouhar, Vilém},
447
- title = {Pearmut: Human Evaluation of Translation Made Trivial},
448
- year = {2026}
553
+ title={Pearmut: Human Evaluation of Translation Made Trivial},
554
+ author={Vilém Zouhar and Tom Kocmi},
555
+ year={2026},
556
+ eprint={2601.02933},
557
+ archivePrefix={arXiv},
558
+ primaryClass={cs.CL},
559
+ url={https://arxiv.org/abs/2601.02933},
449
560
  }
450
561
  ```
451
562
 
452
563
  Contributions are welcome! Please reach out to [Vilém Zouhar](mailto:vilem.zouhar@gmail.com).
453
-
454
- # Changelog
455
-
456
- - v1.0.1
457
- - Support RTL languages
458
- - Add boxes for references
459
- - Add custom score sliders for multi-dimensional evaluation
460
- - Make instructions customizable and protocol-dependent
461
- - Support custom sliders
462
- - Purge/reset whole tasks from dashboard
463
- - Fix resetting individual users in single-stream/dynamic
464
- - Fix notification stacking
465
- - Add campaigns from dashboard
466
- - v0.3.3
467
- - Rename `doc_id` to `item_id`
468
- - Add Typst, LaTeX, and PDF export for model ranking tables. Hide them by default.
469
- - Add dynamic assignment type with contrastive model comparison
470
- - Add `instructions_goodbye` field with variable substitution
471
- - Add visual anchors at 33% and 66% on sliders
472
- - Add German→English ESA tutorial with attention checks
473
- - Validate document model consistency before shuffle
474
- - Fix UI block on any interaction
475
- - v0.3.2
476
- - Revert seeding of user IDs
477
- - Set ESA (Error Span Annotation) as default
478
- - Update server IP address configuration
479
- - Show approximate alignment by default
480
- - Unify pointwise and listwise interfaces into `basic`
481
- - Refactor protocol configuration (breaking change)
482
- - v0.2.11
483
- - Add comment field in settings panel
484
- - Add `score_gt` validation for listwise comparisons
485
- - Add Content-Disposition headers for proper download filenames
486
- - Add model results display to dashboard with rankings
487
- - Add campaign file structure validation
488
- - Purge command now unlinks assets
489
- - v0.2.6
490
- - Add frozen annotation links feature for view-only mode
491
- - Add word-level annotation mode toggle for error spans
492
- - Add `[missing]` token support
493
- - Improve frontend speed and cleanup toolboxes on item load
494
- - Host assets via symlinks
495
- - Add validation threshold for success/fail tokens
496
- - Implement reset masking for annotations
497
- - Allow pre-defined user IDs and tokens in campaign data
498
- - v0.1.1
499
- - Set server defaults and add VM launch scripts
500
- - Add warning dialog when navigating away with unsaved work
501
- - Add tutorial validation support for pointwise and listwise
502
- - Add ability to preview existing annotations via progress bar
503
- - Add support for ESA<sup>AI</sup> pre-filled error_spans
504
- - Rename pairwise to listwise and update layout
505
- - Implement single-stream assignment type
506
- - v0.0.3
507
- - Support multimodal inputs and outputs
508
- - Add dashboard
509
- - Implement ESA (Error Span Annotation) and MQM support
510
-
564
+ See changes in [CHANGELOG.md](CHANGELOG.md).
pearmut-1.0.3.dist-info/RECORD ADDED
@@ -0,0 +1,20 @@
1
+ pearmut/app.py,sha256=BWbzUHtE8G3jNkiSIW8MGlYWQeIu0PtN7wELO6E5M_0,13873
2
+ pearmut/assignment.py,sha256=o_66Gq6L6f-JIX_lzIDJi1cAMrvLoqS5x4orwlu2GnI,32244
3
+ pearmut/cli.py,sha256=OMNNc1gxX6ZplS0UoCiNN86v9nsIn9hkjzRoaity2Fg,29904
4
+ pearmut/constants.py,sha256=iYONCk2kyYcKy3kikhSKyXRKZ1lWVaVFdcWh6kUYTrQ,4844
5
+ pearmut/results_export.py,sha256=YoVE_mXDBNzsiv88CzZhZeWLMg5FWTOuH6NrbzUZQs4,5746
6
+ pearmut/utils.py,sha256=a5nLLXk4SDASoArlXEvVqKXCbxdSgHTn_4fnJKaNa4A,4778
7
+ pearmut/static/annotate.bundle.js,sha256=HwSm0RODsVnDmPD4xUBdmdLgOYF6-sRyPbC_EmN_EL8,121895
8
+ pearmut/static/annotate.html,sha256=4e_3Ol-swrp_FISaxnLYS-iN05XgN5obuZyy-a5C4K8,5698
9
+ pearmut/static/dashboard.bundle.js,sha256=27izHS7KUPMmQjw-rKGxxpu-RjaocyJYFwrZF8CcPFg,105264
10
+ pearmut/static/dashboard.html,sha256=wzRKZCvV9p3FkL97GEMC2Y23lRvUDQt1EwuAVr8dehI,3456
11
+ pearmut/static/favicon.svg,sha256=gVPxdBlyfyJVkiMfh8WLaiSyH4lpwmKZs8UiOeX8YW4,7347
12
+ pearmut/static/index.bundle.js,sha256=-koQkaoRCei-H40wozYnvf0PnrAoZbtOXHotJcTn5OM,346
13
+ pearmut/static/index.html,sha256=1ug6svDibld7Tl33PZ3e_VZUFjXKC86sXYLXWBH_TJg,930
14
+ pearmut/static/style.css,sha256=kTzbun0LkbS0tn-bdCf-oajIQvda5YJcYeDrCV1PR-o,4885
15
+ pearmut-1.0.3.dist-info/licenses/LICENSE,sha256=GtR6RcTdRn-P23h5pKFuWSLZrLPD0ytHAwSOBt7aLpI,1071
16
+ pearmut-1.0.3.dist-info/METADATA,sha256=9k_RBzLNkrWNkWB8BsI4kvDveeCS0POANRl58sW3VaU,24332
17
+ pearmut-1.0.3.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
18
+ pearmut-1.0.3.dist-info/entry_points.txt,sha256=eEA9LVWsS3neQbMvL_nMvEw8I0oFudw8nQa1iqxOiWM,45
19
+ pearmut-1.0.3.dist-info/top_level.txt,sha256=CdgtUM-SKQDt6o5g0QreO-_7XTBP9_wnHMS1P-Rl5Go,8
20
+ pearmut-1.0.3.dist-info/RECORD,,
{pearmut-1.0.1.dist-info → pearmut-1.0.3.dist-info}/WHEEL RENAMED
@@ -1,5 +1,5 @@
1
1
  Wheel-Version: 1.0
2
- Generator: setuptools (80.9.0)
2
+ Generator: setuptools (80.10.2)
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
5
5