pearmut 0.1.1__py3-none-any.whl → 0.1.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pearmut/app.py +55 -25
- pearmut/assignment.py +312 -0
- pearmut/cli.py +37 -19
- pearmut/static/assets/style.css +168 -0
- pearmut/static/dashboard.bundle.js +1 -1
- pearmut/static/listwise.bundle.js +1 -0
- pearmut/static/listwise.html +77 -0
- pearmut/static/pointwise.bundle.js +1 -1
- pearmut/static/pointwise.html +1 -167
- pearmut/utils.py +55 -2
- {pearmut-0.1.1.dist-info → pearmut-0.1.3.dist-info}/METADATA +64 -13
- pearmut-0.1.3.dist-info/RECORD +19 -0
- pearmut/protocols.py +0 -122
- pearmut-0.1.1.dist-info/RECORD +0 -17
- {pearmut-0.1.1.dist-info → pearmut-0.1.3.dist-info}/WHEEL +0 -0
- {pearmut-0.1.1.dist-info → pearmut-0.1.3.dist-info}/entry_points.txt +0 -0
- {pearmut-0.1.1.dist-info → pearmut-0.1.3.dist-info}/licenses/LICENSE +0 -0
- {pearmut-0.1.1.dist-info → pearmut-0.1.3.dist-info}/top_level.txt +0 -0
pearmut/static/pointwise.html
CHANGED
|
@@ -28,10 +28,6 @@
|
|
|
28
28
|
text-align: center;
|
|
29
29
|
}
|
|
30
30
|
|
|
31
|
-
.output_block {
|
|
32
|
-
margin-bottom: 30pt;
|
|
33
|
-
}
|
|
34
|
-
|
|
35
31
|
.output_labels {
|
|
36
32
|
vertical-align: top;
|
|
37
33
|
display: inline-block;
|
|
@@ -48,7 +44,7 @@
|
|
|
48
44
|
width: 20px;
|
|
49
45
|
}
|
|
50
46
|
|
|
51
|
-
/* on displays with less than
|
|
47
|
+
/* on displays with less than 1000px */
|
|
52
48
|
@media (max-width: 1000px) {
|
|
53
49
|
.output_srctgt {
|
|
54
50
|
width: 80%;
|
|
@@ -63,109 +59,6 @@
|
|
|
63
59
|
}
|
|
64
60
|
}
|
|
65
61
|
|
|
66
|
-
.tgt_char:hover {
|
|
67
|
-
background-color: #ccc;
|
|
68
|
-
cursor: pointer;
|
|
69
|
-
;
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
.src_char:hover {
|
|
73
|
-
background-color: #ccc;
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
.error_unknown {
|
|
77
|
-
background-color: #ddf;
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
.error_neutral {
|
|
81
|
-
background-color: #edd;
|
|
82
|
-
}
|
|
83
|
-
|
|
84
|
-
.error_minor {
|
|
85
|
-
background-color: #fcc;
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
.error_major {
|
|
89
|
-
background-color: #e88;
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
.src_char.highlighted,
|
|
93
|
-
.tgt_char.highlighted {
|
|
94
|
-
background-color: #ccc;
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
#button_error_minor:hover,
|
|
98
|
-
#button_error_major:hover {
|
|
99
|
-
opacity: 0.8;
|
|
100
|
-
cursor: pointer;
|
|
101
|
-
}
|
|
102
|
-
|
|
103
|
-
.highlighted_active {
|
|
104
|
-
background-color: #aaf !important;
|
|
105
|
-
}
|
|
106
|
-
|
|
107
|
-
/* span toolbox section */
|
|
108
|
-
.span_toolbox {
|
|
109
|
-
background-color: white;
|
|
110
|
-
padding: 5px;
|
|
111
|
-
border-radius: 8px;
|
|
112
|
-
box-shadow: 0 4px 6px #0005;
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
.span_toolbox_parent {
|
|
116
|
-
position: absolute;
|
|
117
|
-
padding-left: 20px;
|
|
118
|
-
padding-right: 20px;
|
|
119
|
-
padding-top: 10px;
|
|
120
|
-
/* always maximally spread out children */
|
|
121
|
-
min-width: max-content;
|
|
122
|
-
}
|
|
123
|
-
|
|
124
|
-
input[type="button"].error_neutral {
|
|
125
|
-
background-color: #ecc9 !important;
|
|
126
|
-
width: 100%;
|
|
127
|
-
text-align: center;
|
|
128
|
-
border-radius: 8px;
|
|
129
|
-
}
|
|
130
|
-
|
|
131
|
-
input[type="button"].error_neutral:hover {
|
|
132
|
-
background-color: #ecc !important;
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
input[type="button"].error_minor {
|
|
136
|
-
background-color: #fcc !important;
|
|
137
|
-
width: 100%;
|
|
138
|
-
text-align: center;
|
|
139
|
-
border-radius: 8px;
|
|
140
|
-
}
|
|
141
|
-
|
|
142
|
-
input[type="button"].error_minor:hover {
|
|
143
|
-
background-color: #daa !important;
|
|
144
|
-
}
|
|
145
|
-
|
|
146
|
-
input[type="button"].error_major {
|
|
147
|
-
background-color: #e88 !important;
|
|
148
|
-
width: 100%;
|
|
149
|
-
text-align: center;
|
|
150
|
-
border-radius: 8px;
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
input[type="button"].error_major:hover {
|
|
154
|
-
background-color: #c66 !important;
|
|
155
|
-
}
|
|
156
|
-
|
|
157
|
-
input[type="button"].error_delete {
|
|
158
|
-
background-color: #ddd !important;
|
|
159
|
-
font-size: 10pt;
|
|
160
|
-
width: 100%;
|
|
161
|
-
text-align: center;
|
|
162
|
-
border-radius: 8px;
|
|
163
|
-
}
|
|
164
|
-
|
|
165
|
-
input[type="button"].error_delete:hover {
|
|
166
|
-
background-color: #ccc !important;
|
|
167
|
-
}
|
|
168
|
-
|
|
169
62
|
input[type=range][orient=vertical] {
|
|
170
63
|
position: relative;
|
|
171
64
|
top: -5px;
|
|
@@ -173,63 +66,4 @@
|
|
|
173
66
|
direction: rtl;
|
|
174
67
|
width: 16px;
|
|
175
68
|
height: 200px;
|
|
176
|
-
}
|
|
177
|
-
|
|
178
|
-
/* progress bar */
|
|
179
|
-
#progress span {
|
|
180
|
-
font-size: 0pt;
|
|
181
|
-
display: inline-block;
|
|
182
|
-
border-radius: 50%;
|
|
183
|
-
text-align: center;
|
|
184
|
-
line-height: 2em;
|
|
185
|
-
margin-left: 2px;
|
|
186
|
-
box-shadow: 0 1px 1px #0002;
|
|
187
|
-
width: 10px;
|
|
188
|
-
height: 10px;
|
|
189
|
-
}
|
|
190
|
-
|
|
191
|
-
#progress span:hover,
|
|
192
|
-
#progress span.progress_current,
|
|
193
|
-
#progress span.progress_incomplete:last-child {
|
|
194
|
-
cursor: pointer;
|
|
195
|
-
user-select: none;
|
|
196
|
-
position: relative;
|
|
197
|
-
top: 7.5px;
|
|
198
|
-
width: 22px;
|
|
199
|
-
height: 22px;
|
|
200
|
-
font-size: 8pt;
|
|
201
|
-
margin-left: -5px;
|
|
202
|
-
margin-right: -5px;
|
|
203
|
-
box-shadow: 0 1px 3px #0002;
|
|
204
|
-
}
|
|
205
|
-
|
|
206
|
-
#progress span:hover {
|
|
207
|
-
z-index: 100;
|
|
208
|
-
}
|
|
209
|
-
|
|
210
|
-
#progress span.progress_complete {
|
|
211
|
-
color: white;
|
|
212
|
-
background: #3b5238;
|
|
213
|
-
}
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
#progress span.progress_complete:hover {
|
|
217
|
-
background: #2e3e2b;
|
|
218
|
-
}
|
|
219
|
-
|
|
220
|
-
#progress span.progress_current {
|
|
221
|
-
background: #91b08d;
|
|
222
|
-
}
|
|
223
|
-
|
|
224
|
-
#progress span.progress_current:hover {
|
|
225
|
-
background: #739c6f;
|
|
226
|
-
}
|
|
227
|
-
|
|
228
|
-
#progress span.progress_incomplete {
|
|
229
|
-
background: #bbb;
|
|
230
|
-
color: #555;
|
|
231
|
-
}
|
|
232
|
-
|
|
233
|
-
#progress span.progress_incomplete:hover {
|
|
234
|
-
background: #aaa;
|
|
235
69
|
}</style><script defer="defer" src="pointwise.bundle.js"></script></head><body><div style="max-width: 1600px; min-width: 900px; margin-left: auto; margin-right: auto; margin-top: 20px; padding-left: 10px;"><div class="white-box" style="margin-right: 30px; background-color: #e7e2cf; padding: 5px 15px 5px 5px;"><span id="instructions_global" style="display: inline-block; font-size: 11pt; width: calc(100% - 170px);"><ul id="instructions_spans"><li>Error spans:<ul><li><strong>Select</strong> the part of translation where you have identified a <strong>translation error</strong> (drag or click start & end).</li><li><strong>Click</strong> on the highlight to change error severity (minor/major) or remove the highlight.</li></ul>Choose error severity:<ul><li><span class="instruction_sev" id="instruction_sev_minor">Minor errors:</span> Style, grammar, word choice could be better or more natural.</li><li><span class="instruction_sev" id="instruction_sev_major">Major errors:</span>: The meaning is changed significantly and/or the part is really hard to understand.</li></ul><strong>Tip</strong>: Highlight the word or general area of the error (it doesn't need to be exact). Use separate highlights for different errors.<br></li><li id="instructions_score">Score the translation: Please use the slider and set an overall score based on meaning preservation and general quality:</li><ul><li>0: <strong>No meaning preserved</strong>: most information is lost.</li><li>33%: <strong>Some meaning preserved</strong>: major gaps and narrative issues.</li><li>66%: <strong>Most meaning preserved</strong>: minor issues with grammar or consistency.</li><li>100%: <strong>Perfect</strong>: meaning and grammar align completely with the source.</li></ul><li id="instructions_categories">Error types: After highlighting an error fragment, you will be asked to select the specific error type (main category and subcategory). If you are unsure about which errors fall under which categories, please consult the <a href="https://themqm.org/the-mqm-typology/" style="font-weight: bold; text-decoration: none; color: black;">typology definitions</a>.</li></ul></span><div style="width: 170px; display: inline-block; vertical-align: top; text-align: right; padding-top: 5px;"><span id="time" style="width: 135px; text-align: left; display: inline-block; font-size: 11pt;" title="Approximation of total annotation time.">Time: 0m</span> <input type="button" value="⚙️" id="button_settings" style="height: 1.5em; width: 30px;"><br><br><div id="progress" style="text-align: center;"></div><br><br><input type="button" value="Next 🛠️" id="button_next" disabled="disabled" style="width: 170px; height: 2.5em;" title="Finish annotating all examples first."> <input type="button" value="skip tutorial" id="button_skip_tutorial" style="width: 170px; font-size: 11pt; height: 30px; margin-top: 10px; display: none;" title="Skip tutorial only if you completed it already."></div></div><div id="settings_div" class="white-box" style="margin-right: 20px; margin-top: 10px; display: none; background-color: #e7e2cf; font-size: 11pt;"><input type="checkbox" id="settings_approximate_alignment"> <label for="settings_approximate_alignment">Show approximate alignment</label></div><div id="output_div" style="margin-top: 100px;"></div><br><br><br></div></body></html>
|
pearmut/utils.py
CHANGED
|
@@ -3,6 +3,7 @@ import os
|
|
|
3
3
|
|
|
4
4
|
ROOT = "."
|
|
5
5
|
|
|
6
|
+
|
|
6
7
|
def highlight_differences(a, b):
|
|
7
8
|
"""
|
|
8
9
|
Compares two strings and wraps their differences in HTML span tags.
|
|
@@ -30,7 +31,7 @@ def highlight_differences(a, b):
|
|
|
30
31
|
res_a.append(f"{span_open}{a[i1:i2]}{span_close}")
|
|
31
32
|
if tag in ('replace', 'insert'):
|
|
32
33
|
res_b.append(f"{span_open}{b[j1:j2]}{span_close}")
|
|
33
|
-
|
|
34
|
+
|
|
34
35
|
return "".join(res_a), "".join(res_b)
|
|
35
36
|
|
|
36
37
|
|
|
@@ -43,6 +44,58 @@ def load_progress_data(warn: str | None = None):
|
|
|
43
44
|
with open(f"{ROOT}/data/progress.json", "r") as f:
|
|
44
45
|
return json.load(f)
|
|
45
46
|
|
|
47
|
+
|
|
46
48
|
def save_progress_data(data):
|
|
47
49
|
with open(f"{ROOT}/data/progress.json", "w") as f:
|
|
48
|
-
json.dump(data, f, indent=2)
|
|
50
|
+
json.dump(data, f, indent=2)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
_logs = {}
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def get_db_log(campaign_id: str) -> list[dict]:
|
|
57
|
+
"""
|
|
58
|
+
Returns up to date log for the given campaign_id.
|
|
59
|
+
"""
|
|
60
|
+
if campaign_id not in _logs:
|
|
61
|
+
# create a new one if it doesn't exist
|
|
62
|
+
log_path = f"{ROOT}/data/outputs/{campaign_id}.jsonl"
|
|
63
|
+
if os.path.exists(log_path):
|
|
64
|
+
with open(log_path, "r") as f:
|
|
65
|
+
_logs[campaign_id] = [
|
|
66
|
+
json.loads(line) for line in f.readlines()
|
|
67
|
+
]
|
|
68
|
+
else:
|
|
69
|
+
_logs[campaign_id] = []
|
|
70
|
+
|
|
71
|
+
return _logs[campaign_id]
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def get_db_log_item(campaign_id: str, user_id: str | None, item_i: int | None) -> list[dict]:
|
|
75
|
+
"""
|
|
76
|
+
Returns the log item for the given campaign_id, user_id and item_i.
|
|
77
|
+
Can be empty.
|
|
78
|
+
"""
|
|
79
|
+
log = get_db_log(campaign_id)
|
|
80
|
+
return [
|
|
81
|
+
entry for entry in log
|
|
82
|
+
if (
|
|
83
|
+
(user_id is None or entry.get("user_id") == user_id) and
|
|
84
|
+
(item_i is None or entry.get("item_i") == item_i)
|
|
85
|
+
)
|
|
86
|
+
]
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def save_db_payload(campaign_id: str, payload: dict):
|
|
90
|
+
"""
|
|
91
|
+
Saves the given payload to the log for the given campaign_id, user_id and item_i.
|
|
92
|
+
Saves both on disk and in-memory.
|
|
93
|
+
"""
|
|
94
|
+
|
|
95
|
+
log_path = f"{ROOT}/data/outputs/{campaign_id}.jsonl"
|
|
96
|
+
with open(log_path, "a") as log_file:
|
|
97
|
+
log_file.write(json.dumps(payload, ensure_ascii=False,) + "\n")
|
|
98
|
+
|
|
99
|
+
log = get_db_log(campaign_id)
|
|
100
|
+
# copy to avoid mutation issues
|
|
101
|
+
log.append(payload)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pearmut
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.3
|
|
4
4
|
Summary: A tool for evaluation of model outputs, primarily MT.
|
|
5
5
|
Author-email: Vilém Zouhar <vilem.zouhar@gmail.com>
|
|
6
6
|
License: apache-2.0
|
|
@@ -23,7 +23,7 @@ Dynamic: license-file
|
|
|
23
23
|
|
|
24
24
|
Pearmut is a **Platform for Evaluation and Reviewing of Multilingual Tasks**.
|
|
25
25
|
It evaluates model outputs, primarily translation but also various other NLP tasks.
|
|
26
|
-
Supports multimodality (text, video, audio, images) and a variety of annotation protocols (DA, ESA, MQM, paired ESA, etc).
|
|
26
|
+
Supports multimodality (text, video, audio, images) and a variety of annotation protocols ([DA](https://aclanthology.org/N15-1124/), [ESA](https://aclanthology.org/2024.wmt-1.131/), [ESA<sup>AI</sup>](https://aclanthology.org/2025.naacl-long.255/), [MQM](https://doi.org/10.1162/tacl_a_00437), paired ESA, etc).
|
|
27
27
|
|
|
28
28
|
[](https://pypi.org/project/pearmut)
|
|
29
29
|
|
|
@@ -31,7 +31,7 @@ Supports multimodality (text, video, audio, images) and a variety of annotation
|
|
|
31
31
|
|
|
32
32
|
[](https://pypi.org/project/pearmut/)
|
|
33
33
|
|
|
34
|
-
[](https://github.com/zouharvi/pearmut/actions/workflows/test.yml)
|
|
35
35
|
|
|
36
36
|
<img width="1000" alt="Screenshot of ESA/MQM interface" src="https://github.com/user-attachments/assets/f14c91a5-44d7-4248-ada9-387e95ca59d0" />
|
|
37
37
|
|
|
@@ -63,7 +63,7 @@ One of the simplest ones, where each user has a pre-defined list of tasks (`task
|
|
|
63
63
|
```python
|
|
64
64
|
{
|
|
65
65
|
"info": {
|
|
66
|
-
"
|
|
66
|
+
"assignment": "task-based",
|
|
67
67
|
"template": "pointwise",
|
|
68
68
|
"protocol_score": true, # we want scores [0...100] for each segment
|
|
69
69
|
"protocol_error_spans": true, # we want error spans
|
|
@@ -115,19 +115,51 @@ For the standard ones (ESA, DA, MQM), we expect each item to be a dictionary (co
|
|
|
115
115
|
... # definition of another item (document)
|
|
116
116
|
```
|
|
117
117
|
|
|
118
|
-
|
|
118
|
+
## Pre-filled Error Spans (ESA<sup>AI</sup> Support)
|
|
119
|
+
|
|
120
|
+
For workflows where you want to provide pre-filled error annotations (e.g., ESA<sup>AI</sup>), you can include an `error_spans` key in each item.
|
|
121
|
+
These spans will be loaded into the interface as existing annotations that users can review, modify, or delete.
|
|
122
|
+
|
|
123
|
+
```python
|
|
124
|
+
{
|
|
125
|
+
"src": "The quick brown fox jumps over the lazy dog.",
|
|
126
|
+
"tgt": "Rychlá hnědá liška skáče přes líného psa.",
|
|
127
|
+
"error_spans": [
|
|
128
|
+
{
|
|
129
|
+
"start_i": 0, # character index start (inclusive)
|
|
130
|
+
"end_i": 5, # character index end (inclusive)
|
|
131
|
+
"severity": "minor", # "minor", "major", "neutral", or null
|
|
132
|
+
"category": null # MQM category string or null
|
|
133
|
+
},
|
|
134
|
+
{
|
|
135
|
+
"start_i": 27,
|
|
136
|
+
"end_i": 32,
|
|
137
|
+
"severity": "major",
|
|
138
|
+
"category": null
|
|
139
|
+
}
|
|
140
|
+
]
|
|
141
|
+
}
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
For **listwise** template, `error_spans` is a 2D array where each inner array corresponds to error spans for that candidate.
|
|
145
|
+
|
|
146
|
+
See [examples/esaai_prefilled.json](examples/esaai_prefilled.json) for a complete example.
|
|
147
|
+
|
|
148
|
+
## Single-stream Assignment
|
|
149
|
+
|
|
150
|
+
We also support a simple allocation where all annotators draw from the same pool (`single-stream`). Items are randomly assigned to annotators from the pool of unfinished items:
|
|
119
151
|
```python
|
|
120
152
|
{
|
|
121
153
|
"campaign_id": "my campaign 6",
|
|
122
154
|
"info": {
|
|
123
|
-
"
|
|
155
|
+
"assignment": "single-stream",
|
|
124
156
|
"template": "pointwise",
|
|
125
157
|
"protocol_score": True, # collect scores
|
|
126
158
|
"protocol_error_spans": True, # collect error spans
|
|
127
159
|
"protocol_error_categories": False, # do not collect MQM categories, so ESA
|
|
128
|
-
"
|
|
160
|
+
"num_users": 50, # number of annotators
|
|
129
161
|
},
|
|
130
|
-
"data": [...], # list of all items
|
|
162
|
+
"data": [...], # list of all items (shared among all annotators)
|
|
131
163
|
}
|
|
132
164
|
```
|
|
133
165
|
|
|
@@ -137,10 +169,10 @@ We also support dynamic allocation of annotations (`dynamic`, not yet ⚠️), w
|
|
|
137
169
|
{
|
|
138
170
|
"campaign_id": "my campaign 6",
|
|
139
171
|
"info": {
|
|
140
|
-
"
|
|
141
|
-
"template": "
|
|
172
|
+
"assignment": "dynamic",
|
|
173
|
+
"template": "listwise",
|
|
142
174
|
"protocol_k": 5,
|
|
143
|
-
"
|
|
175
|
+
"num_users": 50,
|
|
144
176
|
},
|
|
145
177
|
"data": [...], # list of all items
|
|
146
178
|
}
|
|
@@ -154,6 +186,25 @@ pearmut add my_campaign_4.json
|
|
|
154
186
|
pearmut run
|
|
155
187
|
```
|
|
156
188
|
|
|
189
|
+
## Campaign options
|
|
190
|
+
|
|
191
|
+
In summary, you can select from the assignment types
|
|
192
|
+
|
|
193
|
+
- `task-based`: each user has a predefined set of items
|
|
194
|
+
- `single-stream`: all users are annotating together the same set of items
|
|
195
|
+
- `dynamic`: WIP ⚠️
|
|
196
|
+
|
|
197
|
+
and independently of that select your protocol template:
|
|
198
|
+
|
|
199
|
+
- `pointwise`: evaluate a single output given a single output
|
|
200
|
+
- `protocol_score`: ask for score 0 to 100
|
|
201
|
+
- `protocol_error_spans`: ask for highlighting error spans
|
|
202
|
+
- `protocol_error_categories`: ask for highlighting error categories
|
|
203
|
+
- `listwise`: evaluate multiple outputs at the same time given a single output ⚠️
|
|
204
|
+
- `protocol_score`: ask for score 0 to 100
|
|
205
|
+
- `protocol_error_spans`: ask for highlighting error spans
|
|
206
|
+
- `protocol_error_categories`: ask for highlighting error categories
|
|
207
|
+
|
|
157
208
|
## Campaign management
|
|
158
209
|
|
|
159
210
|
When adding new campaigns or launching pearmut, a management link is shown that gives an overview of annotator progress but also an easy access to the annotation links or resetting the task progress (no data will be lost).
|
|
@@ -170,7 +221,7 @@ An intentionally incorrect token can be shown if the annotations don't pass qual
|
|
|
170
221
|
|
|
171
222
|
We also support anything HTML-compatible both on the input and on the output.
|
|
172
223
|
This includes embedded YouTube videos, or even simple `<video ` tags that point to some resource somewhere.
|
|
173
|
-
For an example, try [examples/
|
|
224
|
+
For an example, try [examples/multimodal.json](examples/multimodal.json).
|
|
174
225
|
Tip: make sure the elements are already appropriately styled.
|
|
175
226
|
|
|
176
227
|
<img width="800" alt="Preview of multimodal elements in Pearmut" src="https://github.com/user-attachments/assets/f34a1a3e-ad95-4114-95ee-8a49e8003faf" />
|
|
@@ -185,7 +236,7 @@ To make changes locally, clone the repository and run the following, which will
|
|
|
185
236
|
cd pearmut
|
|
186
237
|
# watch the frontend for changes (in a separate terminal)
|
|
187
238
|
npm install web/ --prefix web/
|
|
188
|
-
npm run
|
|
239
|
+
npm run build --prefix web/ # `watch` for rebuild on code change
|
|
189
240
|
|
|
190
241
|
# install local package as editable
|
|
191
242
|
pip3 install -e .
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
pearmut/app.py,sha256=ymRlnpKrWSiwdc51Tw4PBDDFFOY1bmdeU-xJ2VlOl-Q,7393
|
|
2
|
+
pearmut/assignment.py,sha256=aOQNlGYzzPNgunAmIIwlcF4qY-l-w6Wmy7hGquArAsc,10623
|
|
3
|
+
pearmut/cli.py,sha256=mV76uw6BywckbU7QEKIKTboukcALEdZp7l-kskJnBVA,7683
|
|
4
|
+
pearmut/utils.py,sha256=gk8b4biPc9TTvZiQMQ_8xh1_FsWuwrhtPzeK3NpzhZc,2902
|
|
5
|
+
pearmut/static/dashboard.bundle.js,sha256=6389gsHLCFh6JqiKdU3ng-Lm6VICRvfJgCSYM61H75U,91257
|
|
6
|
+
pearmut/static/dashboard.html,sha256=tUP1yYvbKySRz0mxFtGq2Si4hTMhJkUCWeTpnq91Nf4,1789
|
|
7
|
+
pearmut/static/index.html,sha256=ieCRLK83MVe-f-gtjYiOlvE-kKd8VnFF2xgyi6FoZpU,872
|
|
8
|
+
pearmut/static/listwise.bundle.js,sha256=Qcz3TSA8C5QRFI-ui47y99WF87wf_4tMKHZ3TyfiYa8,103790
|
|
9
|
+
pearmut/static/listwise.html,sha256=MNS4gV1Fqx7JXZikLhrWgL0z1OPdqgumlOfTcmGnXEI,5212
|
|
10
|
+
pearmut/static/pointwise.bundle.js,sha256=doa3DC8n9L7IIV2ttWxV-TBKVMQHgjTQgSR3Pjozy3k,106133
|
|
11
|
+
pearmut/static/pointwise.html,sha256=dhmfgpWvCFB833Y4kj08_aBZyCN33SayYcS1ckL2-FU,5009
|
|
12
|
+
pearmut/static/assets/favicon.svg,sha256=gVPxdBlyfyJVkiMfh8WLaiSyH4lpwmKZs8UiOeX8YW4,7347
|
|
13
|
+
pearmut/static/assets/style.css,sha256=-B-RySjt8qccqkwvLT0PDy6IRoE1xytLLKAFtR_S-Tg,3967
|
|
14
|
+
pearmut-0.1.3.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
15
|
+
pearmut-0.1.3.dist-info/METADATA,sha256=XhlUE5eAzWzZ1MQX4RmPQuM5Kijk_LwYahgQvTbmmp4,10990
|
|
16
|
+
pearmut-0.1.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
17
|
+
pearmut-0.1.3.dist-info/entry_points.txt,sha256=eEA9LVWsS3neQbMvL_nMvEw8I0oFudw8nQa1iqxOiWM,45
|
|
18
|
+
pearmut-0.1.3.dist-info/top_level.txt,sha256=CdgtUM-SKQDt6o5g0QreO-_7XTBP9_wnHMS1P-Rl5Go,8
|
|
19
|
+
pearmut-0.1.3.dist-info/RECORD,,
|
pearmut/protocols.py
DELETED
|
@@ -1,122 +0,0 @@
|
|
|
1
|
-
from typing import Any
|
|
2
|
-
|
|
3
|
-
from fastapi.responses import JSONResponse
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
def get_next_item(
|
|
7
|
-
campaign_id: str,
|
|
8
|
-
user_id: str,
|
|
9
|
-
tasks_data: dict,
|
|
10
|
-
progress_data: dict,
|
|
11
|
-
) -> JSONResponse:
|
|
12
|
-
"""
|
|
13
|
-
Get the next item for the user in the specified campaign.
|
|
14
|
-
"""
|
|
15
|
-
if tasks_data[campaign_id]["info"]["type"] == "task-based":
|
|
16
|
-
return get_next_item_taskbased(campaign_id, user_id, tasks_data, progress_data)
|
|
17
|
-
elif tasks_data[campaign_id]["info"]["type"] == "dynamic":
|
|
18
|
-
return get_next_item_dynamic(campaign_id, user_id, tasks_data, progress_data)
|
|
19
|
-
else:
|
|
20
|
-
return JSONResponse(content={"error": "Unknown campaign type"}, status_code=400)
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
def get_next_item_taskbased(
|
|
24
|
-
campaign_id: str,
|
|
25
|
-
user_id: str,
|
|
26
|
-
data_all: dict,
|
|
27
|
-
progress_data: dict,
|
|
28
|
-
) -> JSONResponse:
|
|
29
|
-
"""
|
|
30
|
-
Get the next item for task-based protocol.
|
|
31
|
-
"""
|
|
32
|
-
if all(progress_data[campaign_id][user_id]["progress"]):
|
|
33
|
-
# all items completed
|
|
34
|
-
# TODO: add check for data quality
|
|
35
|
-
is_ok = True
|
|
36
|
-
return JSONResponse(
|
|
37
|
-
content={
|
|
38
|
-
"status": "completed",
|
|
39
|
-
"progress": {
|
|
40
|
-
"completed": sum(progress_data[campaign_id][user_id]["progress"]),
|
|
41
|
-
"time": progress_data[campaign_id][user_id]["time"],
|
|
42
|
-
"total": len(data_all[campaign_id]["data"][user_id]),
|
|
43
|
-
},
|
|
44
|
-
"token": progress_data[campaign_id][user_id]["token_correct" if is_ok else "token_incorrect"],
|
|
45
|
-
},
|
|
46
|
-
status_code=200
|
|
47
|
-
)
|
|
48
|
-
|
|
49
|
-
# find first incomplete item
|
|
50
|
-
item_i = min([i for i, v in enumerate(progress_data[campaign_id][user_id]["progress"]) if not v])
|
|
51
|
-
return JSONResponse(
|
|
52
|
-
content={
|
|
53
|
-
"status": "ok",
|
|
54
|
-
"progress": {
|
|
55
|
-
"completed": sum(progress_data[campaign_id][user_id]["progress"]),
|
|
56
|
-
"time": progress_data[campaign_id][user_id]["time"],
|
|
57
|
-
"total": len(data_all[campaign_id]["data"][user_id]),
|
|
58
|
-
},
|
|
59
|
-
"info": {
|
|
60
|
-
"instructions": data_all[campaign_id]["info"].get("instructions", ""),
|
|
61
|
-
"item_i": item_i,
|
|
62
|
-
} | {
|
|
63
|
-
k: v
|
|
64
|
-
for k, v in data_all[campaign_id]["info"].items()
|
|
65
|
-
if k.startswith("protocol")
|
|
66
|
-
},
|
|
67
|
-
"payload": data_all[campaign_id]["data"][user_id][item_i]},
|
|
68
|
-
status_code=200
|
|
69
|
-
)
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
def get_next_item_dynamic(campaign_data: dict, user_id: str, progress_data: dict, data_all: dict):
|
|
73
|
-
raise NotImplementedError("Dynamic protocol is not implemented yet.")
|
|
74
|
-
pass
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
def reset_task(
|
|
78
|
-
campaign_id: str,
|
|
79
|
-
user_id: str,
|
|
80
|
-
tasks_data: dict,
|
|
81
|
-
progress_data: dict,
|
|
82
|
-
) -> JSONResponse:
|
|
83
|
-
"""
|
|
84
|
-
Reset the task progress for the user in the specified campaign.
|
|
85
|
-
"""
|
|
86
|
-
if tasks_data[campaign_id]["info"]["type"] == "task-based":
|
|
87
|
-
progress_data[campaign_id][user_id]["progress"] = [False]*len(tasks_data[campaign_id]["data"][user_id])
|
|
88
|
-
progress_data[campaign_id][user_id]["time"] = 0.0
|
|
89
|
-
progress_data[campaign_id][user_id]["time_start"] = None
|
|
90
|
-
progress_data[campaign_id][user_id]["time_end"] = None
|
|
91
|
-
return JSONResponse(content={"status": "ok"}, status_code=200)
|
|
92
|
-
else:
|
|
93
|
-
progress_data[campaign_id][user_id]["progress"] = []
|
|
94
|
-
progress_data[campaign_id][user_id]["time"] = 0.0
|
|
95
|
-
progress_data[campaign_id][user_id]["time_start"] = None
|
|
96
|
-
progress_data[campaign_id][user_id]["time_end"] = None
|
|
97
|
-
return JSONResponse(content={"status": "ok"}, status_code=200)
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
def update_progress(
|
|
102
|
-
campaign_id: str,
|
|
103
|
-
user_id: str,
|
|
104
|
-
tasks_data: dict,
|
|
105
|
-
progress_data: dict,
|
|
106
|
-
item_i: int,
|
|
107
|
-
payload: Any,
|
|
108
|
-
) -> JSONResponse:
|
|
109
|
-
"""
|
|
110
|
-
Log the user's response for the specified item in the campaign.
|
|
111
|
-
"""
|
|
112
|
-
if tasks_data[campaign_id]["info"]["type"] == "task-based":
|
|
113
|
-
# even if it's already set it should be fine
|
|
114
|
-
progress_data[campaign_id][user_id]["progress"][item_i] = True
|
|
115
|
-
# TODO: log attention checks/quality?
|
|
116
|
-
return JSONResponse(content={"status": "ok"}, status_code=200)
|
|
117
|
-
elif tasks_data[campaign_id]["info"]["type"] == "dynamic":
|
|
118
|
-
return JSONResponse(content={"status": "error", "message": "Dynamic protocol logging not implemented yet."}, status_code=400)
|
|
119
|
-
elif tasks_data[campaign_id]["info"]["type"] == "task-single":
|
|
120
|
-
return JSONResponse(content={"status": "error", "message": "Task-single protocol logging not implemented yet."}, status_code=400)
|
|
121
|
-
else:
|
|
122
|
-
return JSONResponse(content={"status": "error", "message": "Unknown campaign type"}, status_code=400)
|
pearmut-0.1.1.dist-info/RECORD
DELETED
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
pearmut/app.py,sha256=p438nGXTqdGHTfTA3DHRA7GF71DsBT77D3pcCCOW_Ow,6535
|
|
2
|
-
pearmut/cli.py,sha256=PSF6JxAzGuV2ir6hdL6Q0Enf2G6KUgaf59_uS18LbZk,6929
|
|
3
|
-
pearmut/protocols.py,sha256=qd0X2l9c2_x4YKtK8RswNHdwc7bUb59IP2-GlDA1_R0,4674
|
|
4
|
-
pearmut/utils.py,sha256=6hfVenrVdGm1r-7uJIkWHhX9o0ztWjqPse_j_MqkgBw,1443
|
|
5
|
-
pearmut/static/dashboard.bundle.js,sha256=d_uoUgbBaAzcnT-hTGhxmZ1GxtSvfCI1xw6c8F8AAMY,91230
|
|
6
|
-
pearmut/static/dashboard.html,sha256=tUP1yYvbKySRz0mxFtGq2Si4hTMhJkUCWeTpnq91Nf4,1789
|
|
7
|
-
pearmut/static/index.html,sha256=ieCRLK83MVe-f-gtjYiOlvE-kKd8VnFF2xgyi6FoZpU,872
|
|
8
|
-
pearmut/static/pointwise.bundle.js,sha256=zdVJ2S9vBcd5-BHzbpCM-fCghAmB6pGB21lQE6butOc,100218
|
|
9
|
-
pearmut/static/pointwise.html,sha256=oLedp_oyvddf-uS-wsdbDBKQxrhuGO7OPp1YNXDDOgI,8323
|
|
10
|
-
pearmut/static/assets/favicon.svg,sha256=gVPxdBlyfyJVkiMfh8WLaiSyH4lpwmKZs8UiOeX8YW4,7347
|
|
11
|
-
pearmut/static/assets/style.css,sha256=p1gu_IVTdJMsiW9UtFQwtpiQ-4XillAbU9XLw1KJeGA,964
|
|
12
|
-
pearmut-0.1.1.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
13
|
-
pearmut-0.1.1.dist-info/METADATA,sha256=dHpNQ5RP92d_kx7XaQL8dz5IIJPNgOdAx24NsXXTe6M,8811
|
|
14
|
-
pearmut-0.1.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
15
|
-
pearmut-0.1.1.dist-info/entry_points.txt,sha256=eEA9LVWsS3neQbMvL_nMvEw8I0oFudw8nQa1iqxOiWM,45
|
|
16
|
-
pearmut-0.1.1.dist-info/top_level.txt,sha256=CdgtUM-SKQDt6o5g0QreO-_7XTBP9_wnHMS1P-Rl5Go,8
|
|
17
|
-
pearmut-0.1.1.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|