pearmut 0.3.3__py3-none-any.whl → 1.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pearmut/app.py +119 -27
- pearmut/assignment.py +318 -55
- pearmut/cli.py +245 -135
- pearmut/constants.py +93 -0
- pearmut/results_export.py +210 -0
- pearmut/static/basic.bundle.js +1 -1
- pearmut/static/basic.html +39 -3
- pearmut/static/dashboard.bundle.js +1 -1
- pearmut/static/dashboard.html +27 -12
- pearmut/static/index.bundle.js +1 -1
- pearmut/static/index.html +1 -1
- pearmut/utils.py +3 -1
- {pearmut-0.3.3.dist-info → pearmut-1.0.1.dist-info}/METADATA +152 -34
- pearmut-1.0.1.dist-info/RECORD +20 -0
- pearmut-0.3.3.dist-info/RECORD +0 -18
- {pearmut-0.3.3.dist-info → pearmut-1.0.1.dist-info}/WHEEL +0 -0
- {pearmut-0.3.3.dist-info → pearmut-1.0.1.dist-info}/entry_points.txt +0 -0
- {pearmut-0.3.3.dist-info → pearmut-1.0.1.dist-info}/licenses/LICENSE +0 -0
- {pearmut-0.3.3.dist-info → pearmut-1.0.1.dist-info}/top_level.txt +0 -0
pearmut/static/dashboard.html
CHANGED
|
@@ -1,19 +1,41 @@
|
|
|
1
1
|
<html><head><meta charset="UTF-8"><meta name="viewport" content="width=900px"><title>Pearmut Evaluation</title><link rel="icon" type="image/svg+xml" href="favicon.svg"><style>.white-box {
|
|
2
|
-
margin-bottom:
|
|
2
|
+
margin-bottom: 10px;
|
|
3
3
|
width: max-content;
|
|
4
|
+
display: inline-block;
|
|
4
5
|
}
|
|
5
6
|
|
|
6
7
|
.abutton {
|
|
7
|
-
background-color:
|
|
8
|
+
background-color: #eee !important;
|
|
8
9
|
text-decoration: none;
|
|
9
10
|
color: black;
|
|
10
11
|
padding: 5px 10px;
|
|
11
12
|
border-radius: 8px;
|
|
12
13
|
box-shadow: 0 4px 6px #0003;
|
|
14
|
+
border: none;
|
|
15
|
+
font-size: smaller !important;
|
|
13
16
|
}
|
|
14
17
|
|
|
15
18
|
.abutton:hover {
|
|
16
|
-
background-color: #ffd;
|
|
19
|
+
background-color: #ffd !important;
|
|
20
|
+
cursor: pointer;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
.ranking-content input[type="button"]:not(:last-child) {
|
|
24
|
+
margin-right: 5px;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
.ranking-content input[type="button"] {
|
|
28
|
+
margin-bottom: 8px;
|
|
29
|
+
padding: 3px 5px;
|
|
30
|
+
box-shadow: 0 2px 2px #0003 !important;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
.ranking-content {
|
|
34
|
+
display: inline-block;
|
|
35
|
+
vertical-align: top;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
.show-ranking-btn:hover {
|
|
17
39
|
cursor: pointer;
|
|
18
40
|
}
|
|
19
41
|
|
|
@@ -54,17 +76,10 @@
|
|
|
54
76
|
}
|
|
55
77
|
|
|
56
78
|
.dashboard-content {
|
|
57
|
-
display:
|
|
58
|
-
gap: 30px;
|
|
59
|
-
align-items: flex-start;
|
|
60
|
-
}
|
|
61
|
-
|
|
62
|
-
.dashboard-table {
|
|
63
|
-
flex-shrink: 0;
|
|
79
|
+
display: inline-block;
|
|
64
80
|
}
|
|
65
81
|
|
|
66
82
|
.results-section {
|
|
67
|
-
flex-shrink: 0;
|
|
68
83
|
min-width: 300px;
|
|
69
84
|
}
|
|
70
85
|
|
|
@@ -78,4 +93,4 @@
|
|
|
78
93
|
.white-box {
|
|
79
94
|
overflow-x: auto;
|
|
80
95
|
}
|
|
81
|
-
}</style><script defer="defer" src="dashboard.bundle.js?
|
|
96
|
+
}</style><script defer="defer" src="dashboard.bundle.js?0d289122fd490c931aec"></script><link href="style.css?0d289122fd490c931aec" rel="stylesheet"></head><body style="padding-top: 50px; padding-bottom: 50px;"><div id="main_div" style="width: calc(100% - 200px); min-width: 1300px; margin-left: auto; margin-right: auto;"><div style="margin-bottom: 15px; font-size: 0.9em; color: #555;">💤not started ✍️in progress ✅completed & passed ❌completed & failed 🔗annotator link 👁️anotator link (view-only) 🗑️reset progress/campaign ⚖️show model ranking</div><div id="dashboard_div"></div><br><a class="abutton" id="download_progress" style="margin-right: 20px;">Download progress metadata 💾</a> <a class="abutton" id="download_annotations" style="margin-right: 20px;">Download all annotations 💾</a> <button class="abutton" id="add_campaign" style="width: 180px; margin-right: 20px;">Add new campaign 📤</button> <input type="file" id="campaign_file_input" accept=".json" style="display: none;"></div></body></html>
|
pearmut/static/index.bundle.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
(()=>{"use strict";var e={284
|
|
1
|
+
(()=>{"use strict";var e={284(e,r,t){t.r(r)}},r={};function t(o){var n=r[o];if(void 0!==n)return n.exports;var i=r[o]={exports:{}};return e[o](i,i.exports,t),i.exports}t.r=e=>{"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})};t(284)})();
|
pearmut/static/index.html
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
<!doctype html><html lang="en" style="height: 100%;"><head><meta charset="UTF-8"><meta name="viewport" content="width=900px"><title>Pearmut Evaluation</title><link rel="icon" type="image/svg+xml" href="favicon.svg"><script defer="defer" src="index.bundle.js?
|
|
1
|
+
<!doctype html><html lang="en" style="height: 100%;"><head><meta charset="UTF-8"><meta name="viewport" content="width=900px"><title>Pearmut Evaluation</title><link rel="icon" type="image/svg+xml" href="favicon.svg"><script defer="defer" src="index.bundle.js?0d289122fd490c931aec"></script><link href="style.css?0d289122fd490c931aec" rel="stylesheet"></head><body><div class="white-box" style="width: max-content; font-size: large; position: absolute; top: 50%; left: 50%; transform: translate(-50%, -50%);">You have reached the Pearmut🍐 evaluation interface.<ul><li>If you are an annotator, you should have received a specialized link that takes you to the annotations.</li><li>If you are annotation manager, then you should distribute these links.</li></ul><br><br>See the <a href="https://github.com/zouharvi/pearmut">Pearmut project on GitHub</a>. Made with 💚 by Vilém Zouhar and others in 2025-2026.</div></body></html>
|
pearmut/utils.py
CHANGED
|
@@ -14,7 +14,9 @@ def load_progress_data(warn: str | None = None):
|
|
|
14
14
|
with open(f"{ROOT}/data/progress.json", "w") as f:
|
|
15
15
|
f.write(json.dumps({}))
|
|
16
16
|
with open(f"{ROOT}/data/progress.json", "r") as f:
|
|
17
|
-
|
|
17
|
+
data = json.load(f)
|
|
18
|
+
|
|
19
|
+
return data
|
|
18
20
|
|
|
19
21
|
|
|
20
22
|
def save_progress_data(data):
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pearmut
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 1.0.1
|
|
4
4
|
Summary: A tool for evaluation of model outputs, primarily MT.
|
|
5
5
|
Author-email: Vilém Zouhar <vilem.zouhar@gmail.com>
|
|
6
6
|
License: MIT
|
|
@@ -14,23 +14,18 @@ Requires-Dist: fastapi>=0.110.0
|
|
|
14
14
|
Requires-Dist: uvicorn>=0.29.0
|
|
15
15
|
Requires-Dist: wonderwords>=3.0.0
|
|
16
16
|
Requires-Dist: psutil>=7.1.0
|
|
17
|
+
Requires-Dist: typst>=0.14.4
|
|
17
18
|
Provides-Extra: dev
|
|
18
19
|
Requires-Dist: pytest; extra == "dev"
|
|
19
20
|
Dynamic: license-file
|
|
20
21
|
|
|
21
|
-
# Pearmut
|
|
22
|
+
# 🍐Pearmut [](https://pypi.org/project/pearmut) [](https://pypi.python.org/pypi/pearmut/) [](https://pypi.org/project/pearmut/) [](https://github.com/zouharvi/pearmut/actions/workflows/test.yml)
|
|
22
23
|
|
|
23
24
|
**Platform for Evaluation and Reviewing of Multilingual Tasks**: Evaluate model outputs for translation and NLP tasks with support for multimodal data (text, video, audio, images) and multiple annotation protocols ([DA](https://aclanthology.org/N15-1124/), [ESA](https://aclanthology.org/2024.wmt-1.131/), [ESA<sup>AI</sup>](https://aclanthology.org/2025.naacl-long.255/), [MQM](https://doi.org/10.1162/tacl_a_00437), and more!).
|
|
24
25
|
|
|
25
|
-
[](https://pypi.org/project/pearmut)
|
|
26
|
-
|
|
27
|
-
[](https://pypi.python.org/pypi/pearmut/)
|
|
28
|
-
|
|
29
|
-
[](https://pypi.org/project/pearmut/)
|
|
30
|
-
|
|
31
|
-
[](https://github.com/zouharvi/pearmut/actions/workflows/test.yml)
|
|
32
26
|
|
|
33
|
-
<img width="1000" alt="Screenshot of ESA/MQM interface" src="https://github.com/user-attachments/assets/
|
|
27
|
+
<img width="1000" alt="Screenshot of ESA/MQM interface" src="https://github.com/user-attachments/assets/71334238-300b-4ffc-b777-7f3c242b1630" />
|
|
28
|
+
|
|
34
29
|
|
|
35
30
|
## Table of Contents
|
|
36
31
|
|
|
@@ -45,10 +40,13 @@ Dynamic: license-file
|
|
|
45
40
|
- [Multimodal Annotations](#multimodal-annotations)
|
|
46
41
|
- [Hosting Assets](#hosting-assets)
|
|
47
42
|
- [Campaign Management](#campaign-management)
|
|
43
|
+
- [Custom Completion Messages](#custom-completion-messages)
|
|
48
44
|
- [CLI Commands](#cli-commands)
|
|
49
45
|
- [Terminology](#terminology)
|
|
50
46
|
- [Development](#development)
|
|
51
47
|
- [Citation](#citation)
|
|
48
|
+
- [Changelog](#changelog)
|
|
49
|
+
|
|
52
50
|
|
|
53
51
|
## Quick Start
|
|
54
52
|
|
|
@@ -86,11 +84,13 @@ Campaigns are defined in JSON files (see [examples/](examples/)). The simplest c
|
|
|
86
84
|
{
|
|
87
85
|
"instructions": "Evaluate translation from en to cs_CZ", # message to show to users above the first item
|
|
88
86
|
"src": "This will be the year that Guinness loses its cool. Cheers to that!",
|
|
89
|
-
"tgt": {"modelA": "Nevím přesně, kdy jsem to poprvé zaznamenal. Možná to bylo ve chvíli, ..."}
|
|
87
|
+
"tgt": {"modelA": "Nevím přesně, kdy jsem to poprvé zaznamenal. Možná to bylo ve chvíli, ..."},
|
|
88
|
+
"item_id": "first item in first document"
|
|
90
89
|
},
|
|
91
90
|
{
|
|
92
91
|
"src": "I'm not sure I can remember exactly when I sensed it. Maybe it was when some...",
|
|
93
|
-
"tgt": {"modelA": "Tohle bude rok, kdy Guinness přijde o svůj „cool“ faktor. Na zdraví!"}
|
|
92
|
+
"tgt": {"modelA": "Tohle bude rok, kdy Guinness přijde o svůj „cool“ faktor. Na zdraví!"},
|
|
93
|
+
"item_id": "second item in first document"
|
|
94
94
|
}
|
|
95
95
|
...
|
|
96
96
|
],
|
|
@@ -105,20 +105,12 @@ Campaigns are defined in JSON files (see [examples/](examples/)). The simplest c
|
|
|
105
105
|
]
|
|
106
106
|
}
|
|
107
107
|
```
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
},
|
|
115
|
-
{
|
|
116
|
-
"src": "toto je pokračování stejného dokumentu",
|
|
117
|
-
"tgt": {"modelA": "this is a continuation of the same document"}
|
|
118
|
-
# Additional keys stored for analysis
|
|
119
|
-
}
|
|
120
|
-
]
|
|
121
|
-
```
|
|
108
|
+
|
|
109
|
+
Each item has to have `tgt` (dictionary from model names to strings, even for a single model evaluation).
|
|
110
|
+
Optionally, you can also include `src` (source string) and/or `ref` (reference string).
|
|
111
|
+
If neither `src` nor `ref` is provided, only the model outputs will be displayed.
|
|
112
|
+
For full Pearmut functionality (e.g. automatic statistical analysis), add `item_id` as well.
|
|
113
|
+
Any other keys that you add will simply be stored in the logs.
|
|
122
114
|
|
|
123
115
|
Load campaigns and start the server:
|
|
124
116
|
```bash
|
|
@@ -130,7 +122,7 @@ pearmut run
|
|
|
130
122
|
|
|
131
123
|
- **`task-based`**: Each user has predefined items
|
|
132
124
|
- **`single-stream`**: All users draw from a shared pool (random assignment)
|
|
133
|
-
- **`dynamic`**:
|
|
125
|
+
- **`dynamic`**: Items are dynamically assigned based on current model performance (see [Dynamic Assignment](#dynamic-assignment))
|
|
134
126
|
|
|
135
127
|
## Advanced Features
|
|
136
128
|
|
|
@@ -150,6 +142,40 @@ The `shuffle` parameter in campaign `info` controls this behavior:
|
|
|
150
142
|
}
|
|
151
143
|
```
|
|
152
144
|
|
|
145
|
+
### Custom Score Sliders
|
|
146
|
+
|
|
147
|
+
For multi-dimensional evaluation tasks (e.g., assessing fluency on a Likert scale), you can define custom sliders with specific ranges and steps:
|
|
148
|
+
|
|
149
|
+
```python
|
|
150
|
+
{
|
|
151
|
+
"info": {
|
|
152
|
+
"assignment": "task-based",
|
|
153
|
+
"protocol": "ESA",
|
|
154
|
+
"sliders": [
|
|
155
|
+
{"name": "Fluency", "min": 0, "max": 5, "step": 1},
|
|
156
|
+
{"name": "Adequacy", "min": 0, "max": 100, "step": 1}
|
|
157
|
+
]
|
|
158
|
+
},
|
|
159
|
+
"campaign_id": "my_campaign",
|
|
160
|
+
"data": [...]
|
|
161
|
+
}
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
When `sliders` is specified, only the custom sliders are shown. Each slider must have `name`, `min`, `max`, and `step` properties. All sliders must be answered before proceeding.
|
|
165
|
+
|
|
166
|
+
### Custom Instructions
|
|
167
|
+
|
|
168
|
+
Set campaign-level instructions using the `instructions` field in `info` (supports HTML).
|
|
169
|
+
Instructions default to protocol-specific ones (DA: scoring, ESA: error spans + scoring, MQM: error spans + categories + scoring).
|
|
170
|
+
```python
|
|
171
|
+
{
|
|
172
|
+
"info": {
|
|
173
|
+
"protocol": "DA",
|
|
174
|
+
"instructions": "Rate translation quality on a 0-100 scale.<br>Pay special attention to document-level phenomena."
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
```
|
|
178
|
+
|
|
153
179
|
### Pre-filled Error Spans (ESA<sup>AI</sup>)
|
|
154
180
|
|
|
155
181
|
Include `error_spans` to pre-fill annotations that users can review, modify, or delete:
|
|
@@ -244,6 +270,36 @@ All annotators draw from a shared pool with random assignment:
|
|
|
244
270
|
}
|
|
245
271
|
```
|
|
246
272
|
|
|
273
|
+
### Dynamic Assignment
|
|
274
|
+
|
|
275
|
+
The `dynamic` assignment type intelligently selects items based on current model performance to focus annotation effort on top-performing models using contrastive comparisons.
|
|
276
|
+
All items must contain outputs from all models for this assignment type to work properly.
|
|
277
|
+
|
|
278
|
+
```python
|
|
279
|
+
{
|
|
280
|
+
"campaign_id": "my dynamic campaign",
|
|
281
|
+
"info": {
|
|
282
|
+
"assignment": "dynamic",
|
|
283
|
+
"protocol": "ESA",
|
|
284
|
+
"users": 10, # number of annotators
|
|
285
|
+
"dynamic_top": 3, # how many top models to consider (required)
|
|
286
|
+
"dynamic_contrastive_models": 2, # how many models to compare per item (optional, default: 1)
|
|
287
|
+
"dynamic_first": 5, # annotations per model before dynamic kicks in (optional, default: 5)
|
|
288
|
+
"dynamic_backoff": 0.1, # probability of uniform sampling (optional, default: 0)
|
|
289
|
+
},
|
|
290
|
+
"data": [...], # list of all items (shared among all annotators)
|
|
291
|
+
}
|
|
292
|
+
```
|
|
293
|
+
|
|
294
|
+
**How it works:**
|
|
295
|
+
1. Initial phase: Each model gets `dynamic_first` annotations with fully random contrastive evaluation
|
|
296
|
+
2. Dynamic phase: After the initial phase, top `dynamic_top` models (by average score) are identified
|
|
297
|
+
3. Contrastive evaluation: From the top N models, `dynamic_contrastive_models` models are randomly selected for each item
|
|
298
|
+
4. Item prioritization: Items with the least annotations for the selected models are prioritized
|
|
299
|
+
5. Backoff: With probability `dynamic_backoff`, uniform random selection is used instead to maintain exploration
|
|
300
|
+
|
|
301
|
+
This approach efficiently focuses annotation resources on distinguishing between the best-performing models while ensuring all models get adequate baseline coverage. The contrastive evaluation allows for direct comparison of multiple models simultaneously.
|
|
302
|
+
For an example, see [examples/dynamic.json](examples/dynamic.json).
|
|
247
303
|
|
|
248
304
|
### Pre-defined User IDs and Tokens
|
|
249
305
|
|
|
@@ -264,6 +320,7 @@ The `users` field accepts:
|
|
|
264
320
|
}
|
|
265
321
|
```
|
|
266
322
|
|
|
323
|
+
|
|
267
324
|
### Multimodal Annotations
|
|
268
325
|
|
|
269
326
|
Support for HTML-compatible elements (YouTube embeds, `<video>` tags, images). Ensure elements are pre-styled. See [examples/multimodal.json](examples/multimodal.json).
|
|
@@ -317,6 +374,10 @@ Completion tokens are shown at annotation end for verification (download correct
|
|
|
317
374
|
|
|
318
375
|
When tokens are supplied, the dashboard will try to show model rankings based on the names in the dictionaries.
|
|
319
376
|
|
|
377
|
+
### Custom Completion Messages
|
|
378
|
+
|
|
379
|
+
Customize the goodbye message shown to users when they complete all annotations using the `instructions_goodbye` field in campaign info. Supports arbitrary HTML for styling and formatting with variable replacement: `${TOKEN}` (completion token) and `${USER_ID}` (user ID). Default: `"If someone asks you for a token of completion, show them: ${TOKEN}"`.
|
|
380
|
+
|
|
320
381
|
## Terminology
|
|
321
382
|
|
|
322
383
|
- **Campaign**: An annotation project that contains configuration, data, and user assignments. Each campaign has a unique identifier and is defined in a JSON file.
|
|
@@ -344,7 +405,7 @@ When tokens are supplied, the dashboard will try to show model rankings based on
|
|
|
344
405
|
- **Assignment**: The method for distributing items to users:
|
|
345
406
|
- **Task-based**: Each user has predefined items
|
|
346
407
|
- **Single-stream**: Users draw from a shared pool with random assignment
|
|
347
|
-
- **Dynamic**:
|
|
408
|
+
- **Dynamic**: Items are intelligently assigned based on model performance to focus on top models
|
|
348
409
|
|
|
349
410
|
## Development
|
|
350
411
|
|
|
@@ -377,16 +438,73 @@ See [web/src/basic.ts](web/src/basic.ts) for example.
|
|
|
377
438
|
|
|
378
439
|
Run on public server or tunnel local port to public IP/domain and run locally.
|
|
379
440
|
|
|
380
|
-
##
|
|
441
|
+
## Citation
|
|
381
442
|
|
|
382
443
|
If you use this work in your paper, please cite as following.
|
|
383
444
|
```bibtex
|
|
384
|
-
@misc{
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
year={2026},
|
|
445
|
+
@misc{zouhar2026pearmut,
|
|
446
|
+
author = {Zouhar, Vilém},
|
|
447
|
+
title = {Pearmut: Human Evaluation of Translation Made Trivial},
|
|
448
|
+
year = {2026}
|
|
389
449
|
}
|
|
390
450
|
```
|
|
391
451
|
|
|
392
452
|
Contributions are welcome! Please reach out to [Vilém Zouhar](mailto:vilem.zouhar@gmail.com).
|
|
453
|
+
|
|
454
|
+
# Changelog
|
|
455
|
+
|
|
456
|
+
- v1.0.1
|
|
457
|
+
- Support RTL languages
|
|
458
|
+
- Add boxes for references
|
|
459
|
+
- Add custom score sliders for multi-dimensional evaluation
|
|
460
|
+
- Make instructions customizable and protocol-dependent
|
|
461
|
+
- Support custom sliders
|
|
462
|
+
- Purge/reset whole tasks from dashboard
|
|
463
|
+
- Fix resetting individual users in single-stream/dynamic
|
|
464
|
+
- Fix notification stacking
|
|
465
|
+
- Add campaigns from dashboard
|
|
466
|
+
- v0.3.3
|
|
467
|
+
- Rename `doc_id` to `item_id`
|
|
468
|
+
- Add Typst, LaTeX, and PDF export for model ranking tables. Hide them by default.
|
|
469
|
+
- Add dynamic assignment type with contrastive model comparison
|
|
470
|
+
- Add `instructions_goodbye` field with variable substitution
|
|
471
|
+
- Add visual anchors at 33% and 66% on sliders
|
|
472
|
+
- Add German→English ESA tutorial with attention checks
|
|
473
|
+
- Validate document model consistency before shuffle
|
|
474
|
+
- Fix UI block on any interaction
|
|
475
|
+
- v0.3.2
|
|
476
|
+
- Revert seeding of user IDs
|
|
477
|
+
- Set ESA (Error Span Annotation) as default
|
|
478
|
+
- Update server IP address configuration
|
|
479
|
+
- Show approximate alignment by default
|
|
480
|
+
- Unify pointwise and listwise interfaces into `basic`
|
|
481
|
+
- Refactor protocol configuration (breaking change)
|
|
482
|
+
- v0.2.11
|
|
483
|
+
- Add comment field in settings panel
|
|
484
|
+
- Add `score_gt` validation for listwise comparisons
|
|
485
|
+
- Add Content-Disposition headers for proper download filenames
|
|
486
|
+
- Add model results display to dashboard with rankings
|
|
487
|
+
- Add campaign file structure validation
|
|
488
|
+
- Purge command now unlinks assets
|
|
489
|
+
- v0.2.6
|
|
490
|
+
- Add frozen annotation links feature for view-only mode
|
|
491
|
+
- Add word-level annotation mode toggle for error spans
|
|
492
|
+
- Add `[missing]` token support
|
|
493
|
+
- Improve frontend speed and cleanup toolboxes on item load
|
|
494
|
+
- Host assets via symlinks
|
|
495
|
+
- Add validation threshold for success/fail tokens
|
|
496
|
+
- Implement reset masking for annotations
|
|
497
|
+
- Allow pre-defined user IDs and tokens in campaign data
|
|
498
|
+
- v0.1.1
|
|
499
|
+
- Set server defaults and add VM launch scripts
|
|
500
|
+
- Add warning dialog when navigating away with unsaved work
|
|
501
|
+
- Add tutorial validation support for pointwise and listwise
|
|
502
|
+
- Add ability to preview existing annotations via progress bar
|
|
503
|
+
- Add support for ESA<sup>AI</sup> pre-filled error_spans
|
|
504
|
+
- Rename pairwise to listwise and update layout
|
|
505
|
+
- Implement single-stream assignment type
|
|
506
|
+
- v0.0.3
|
|
507
|
+
- Support multimodal inputs and outputs
|
|
508
|
+
- Add dashboard
|
|
509
|
+
- Implement ESA (Error Span Annotation) and MQM support
|
|
510
|
+
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
pearmut/app.py,sha256=R33IAsiElYhr_17eCJ2gNSa6XGmnp_qB1-RaNDqItR4,12963
|
|
2
|
+
pearmut/assignment.py,sha256=wuRtP3WSsyRPk432n8EbjJI9GgK0ruJq0PLKDmtM87w,22105
|
|
3
|
+
pearmut/cli.py,sha256=hvKX8et4jWMs0W_O3Da12d8zTb7E2Yw4mlPd49cXYfE,26400
|
|
4
|
+
pearmut/constants.py,sha256=iYONCk2kyYcKy3kikhSKyXRKZ1lWVaVFdcWh6kUYTrQ,4844
|
|
5
|
+
pearmut/results_export.py,sha256=UxtbbqbrqVJDBSYQf-aT3M7lMgQgQiepHkHy4TSZrGg,5743
|
|
6
|
+
pearmut/utils.py,sha256=Rkc08bzm4Z96n_Ks8R--2c2B9TUTWB9z9QQxuyL8bAA,4368
|
|
7
|
+
pearmut/static/basic.bundle.js,sha256=N6ECQMPGq6hLDrwmU-ys5aegiMJu-X20Cfg73vawk7I,114962
|
|
8
|
+
pearmut/static/basic.html,sha256=URS37Iwum2ttnJ886LuoFHs55EjoNTHyI6YKj1TwDTc,5075
|
|
9
|
+
pearmut/static/dashboard.bundle.js,sha256=lNQh_pcKCGOSHLsQoBkU1pahhs6hql8GFm9YqpQRv0Y,105990
|
|
10
|
+
pearmut/static/dashboard.html,sha256=qBQak_12-tQW3vdAXPm2cxdeddQIIOGo_jUXtWsWW3c,3356
|
|
11
|
+
pearmut/static/favicon.svg,sha256=gVPxdBlyfyJVkiMfh8WLaiSyH4lpwmKZs8UiOeX8YW4,7347
|
|
12
|
+
pearmut/static/index.bundle.js,sha256=-koQkaoRCei-H40wozYnvf0PnrAoZbtOXHotJcTn5OM,346
|
|
13
|
+
pearmut/static/index.html,sha256=ImMkxquH2zfNOaZEIPkUoPAga79pTdTIuOm3yKWLMoE,930
|
|
14
|
+
pearmut/static/style.css,sha256=NKdwsugsS946w3pREfLab2Rf3Av9hNk0fvkTxmhyGrQ,4102
|
|
15
|
+
pearmut-1.0.1.dist-info/licenses/LICENSE,sha256=GtR6RcTdRn-P23h5pKFuWSLZrLPD0ytHAwSOBt7aLpI,1071
|
|
16
|
+
pearmut-1.0.1.dist-info/METADATA,sha256=z67XRDVps98CLH5XLmrTUiKhO2wXHbksu2Ez6OtDDJA,21663
|
|
17
|
+
pearmut-1.0.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
18
|
+
pearmut-1.0.1.dist-info/entry_points.txt,sha256=eEA9LVWsS3neQbMvL_nMvEw8I0oFudw8nQa1iqxOiWM,45
|
|
19
|
+
pearmut-1.0.1.dist-info/top_level.txt,sha256=CdgtUM-SKQDt6o5g0QreO-_7XTBP9_wnHMS1P-Rl5Go,8
|
|
20
|
+
pearmut-1.0.1.dist-info/RECORD,,
|
pearmut-0.3.3.dist-info/RECORD
DELETED
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
pearmut/app.py,sha256=O4jfxaiBENwxkBcFhIHNcvMqtsoku2v8PzYNsQjDyGw,10269
|
|
2
|
-
pearmut/assignment.py,sha256=_0hNXtA-Mgn6bRyRVjgeGxERKRvBezR3NmEwx2uME38,11685
|
|
3
|
-
pearmut/cli.py,sha256=3qYmsipPMbkE-ZztJC95bJp763fcTMTEPi5nxtg4NUk,22689
|
|
4
|
-
pearmut/utils.py,sha256=Rl_i-WCaJN3p_VG5iVL0fSeI481jcJUUEZO6HKx62PE,4347
|
|
5
|
-
pearmut/static/basic.bundle.js,sha256=MTbuKxI4l9U8K20YksCAIuBMvvZfG0_h_WkJEbL6spg,110843
|
|
6
|
-
pearmut/static/basic.html,sha256=kP6XrrcUadLqg0gggIjp_6Ab7M_qElDYUcPc9pPDMSs,5998
|
|
7
|
-
pearmut/static/dashboard.bundle.js,sha256=6L45OLVmRLZd0LgzoZJp_pwWEoymavwYl7pcqm4sv6w,100838
|
|
8
|
-
pearmut/static/dashboard.html,sha256=wW1QWEvFqTCrnNIodtepiZLoeZelN1n-gwvfUNZYeD4,2722
|
|
9
|
-
pearmut/static/favicon.svg,sha256=gVPxdBlyfyJVkiMfh8WLaiSyH4lpwmKZs8UiOeX8YW4,7347
|
|
10
|
-
pearmut/static/index.bundle.js,sha256=5EaRCIWX2BcQaOBku7_Ya3n6YBuBruIZ5EVA_Sw3PLI,349
|
|
11
|
-
pearmut/static/index.html,sha256=VKCuvqnMHNdd0w-ey5yOaHBmiqobZ5y5PhVXNU-gupI,930
|
|
12
|
-
pearmut/static/style.css,sha256=NKdwsugsS946w3pREfLab2Rf3Av9hNk0fvkTxmhyGrQ,4102
|
|
13
|
-
pearmut-0.3.3.dist-info/licenses/LICENSE,sha256=GtR6RcTdRn-P23h5pKFuWSLZrLPD0ytHAwSOBt7aLpI,1071
|
|
14
|
-
pearmut-0.3.3.dist-info/METADATA,sha256=GE4dQ5Mfork6YQSjHHzL81qItJku9odiv1iSsfpUcd0,15761
|
|
15
|
-
pearmut-0.3.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
16
|
-
pearmut-0.3.3.dist-info/entry_points.txt,sha256=eEA9LVWsS3neQbMvL_nMvEw8I0oFudw8nQa1iqxOiWM,45
|
|
17
|
-
pearmut-0.3.3.dist-info/top_level.txt,sha256=CdgtUM-SKQDt6o5g0QreO-_7XTBP9_wnHMS1P-Rl5Go,8
|
|
18
|
-
pearmut-0.3.3.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|