pearmut 0.0.6__py3-none-any.whl → 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pearmut/app.py +3 -2
- pearmut/cli.py +28 -1
- pearmut/protocols.py +15 -2
- pearmut/static/assets/style.css +12 -8
- pearmut/static/dashboard.bundle.js +1 -1
- pearmut/static/dashboard.html +4 -3
- pearmut/static/pointwise.bundle.js +1 -1
- pearmut/static/pointwise.html +85 -23
- {pearmut-0.0.6.dist-info → pearmut-0.1.1.dist-info}/METADATA +41 -23
- pearmut-0.1.1.dist-info/RECORD +17 -0
- pearmut/model.py +0 -61
- pearmut-0.0.6.dist-info/RECORD +0 -18
- {pearmut-0.0.6.dist-info → pearmut-0.1.1.dist-info}/WHEEL +0 -0
- {pearmut-0.0.6.dist-info → pearmut-0.1.1.dist-info}/entry_points.txt +0 -0
- {pearmut-0.0.6.dist-info → pearmut-0.1.1.dist-info}/licenses/LICENSE +0 -0
- {pearmut-0.0.6.dist-info → pearmut-0.1.1.dist-info}/top_level.txt +0 -0
pearmut/static/pointwise.html
CHANGED
|
@@ -1,19 +1,21 @@
|
|
|
1
1
|
<!doctype html><html lang="en" style="height: 100%;"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>Pearmut Evaluation</title><link rel="icon" type="image/svg+xml" href="assets/favicon.svg"><link rel="stylesheet" href="assets/style.css"><style>.output_srctgt {
|
|
2
2
|
display: inline-block;
|
|
3
|
-
width: calc(100% -
|
|
3
|
+
width: calc(100% - 140px);
|
|
4
4
|
vertical-align: top;
|
|
5
5
|
}
|
|
6
6
|
|
|
7
|
-
.output_src,
|
|
7
|
+
.output_src,
|
|
8
|
+
.output_tgt {
|
|
8
9
|
display: inline-block;
|
|
9
10
|
width: calc(50% - 30px);
|
|
10
11
|
background-color: #fffc;
|
|
11
12
|
padding: 10px;
|
|
12
|
-
border-radius:
|
|
13
|
+
border-radius: 8px;
|
|
13
14
|
vertical-align: top;
|
|
14
15
|
margin-right: 2px;
|
|
15
16
|
margin-left: 2px;
|
|
16
17
|
user-select: none;
|
|
18
|
+
box-shadow: 0 4px 6px #0005;
|
|
17
19
|
}
|
|
18
20
|
|
|
19
21
|
.output_src {
|
|
@@ -30,12 +32,6 @@
|
|
|
30
32
|
margin-bottom: 30pt;
|
|
31
33
|
}
|
|
32
34
|
|
|
33
|
-
input[type="range"] {
|
|
34
|
-
height: 150px;
|
|
35
|
-
position: relative;
|
|
36
|
-
top: -5px;
|
|
37
|
-
}
|
|
38
|
-
|
|
39
35
|
.output_labels {
|
|
40
36
|
vertical-align: top;
|
|
41
37
|
display: inline-block;
|
|
@@ -59,7 +55,8 @@
|
|
|
59
55
|
margin-right: 30px;
|
|
60
56
|
}
|
|
61
57
|
|
|
62
|
-
.output_src,
|
|
58
|
+
.output_src,
|
|
59
|
+
.output_tgt {
|
|
63
60
|
width: 100%;
|
|
64
61
|
margin-bottom: 5px;
|
|
65
62
|
margin-right: 10px;
|
|
@@ -68,9 +65,10 @@
|
|
|
68
65
|
|
|
69
66
|
.tgt_char:hover {
|
|
70
67
|
background-color: #ccc;
|
|
71
|
-
cursor: pointer
|
|
68
|
+
cursor: pointer;
|
|
69
|
+
;
|
|
72
70
|
}
|
|
73
|
-
|
|
71
|
+
|
|
74
72
|
.src_char:hover {
|
|
75
73
|
background-color: #ccc;
|
|
76
74
|
}
|
|
@@ -91,11 +89,13 @@
|
|
|
91
89
|
background-color: #e88;
|
|
92
90
|
}
|
|
93
91
|
|
|
94
|
-
.src_char.highlighted,
|
|
92
|
+
.src_char.highlighted,
|
|
93
|
+
.tgt_char.highlighted {
|
|
95
94
|
background-color: #ccc;
|
|
96
95
|
}
|
|
97
96
|
|
|
98
|
-
#button_error_minor:hover,
|
|
97
|
+
#button_error_minor:hover,
|
|
98
|
+
#button_error_major:hover {
|
|
99
99
|
opacity: 0.8;
|
|
100
100
|
cursor: pointer;
|
|
101
101
|
}
|
|
@@ -106,9 +106,10 @@
|
|
|
106
106
|
|
|
107
107
|
/* span toolbox section */
|
|
108
108
|
.span_toolbox {
|
|
109
|
-
background-color:
|
|
109
|
+
background-color: white;
|
|
110
110
|
padding: 5px;
|
|
111
|
-
border-radius:
|
|
111
|
+
border-radius: 8px;
|
|
112
|
+
box-shadow: 0 4px 6px #0005;
|
|
112
113
|
}
|
|
113
114
|
|
|
114
115
|
.span_toolbox_parent {
|
|
@@ -121,10 +122,10 @@
|
|
|
121
122
|
}
|
|
122
123
|
|
|
123
124
|
input[type="button"].error_neutral {
|
|
124
|
-
background-color: #
|
|
125
|
+
background-color: #ecc9 !important;
|
|
125
126
|
width: 100%;
|
|
126
127
|
text-align: center;
|
|
127
|
-
border-radius:
|
|
128
|
+
border-radius: 8px;
|
|
128
129
|
}
|
|
129
130
|
|
|
130
131
|
input[type="button"].error_neutral:hover {
|
|
@@ -135,7 +136,7 @@
|
|
|
135
136
|
background-color: #fcc !important;
|
|
136
137
|
width: 100%;
|
|
137
138
|
text-align: center;
|
|
138
|
-
border-radius:
|
|
139
|
+
border-radius: 8px;
|
|
139
140
|
}
|
|
140
141
|
|
|
141
142
|
input[type="button"].error_minor:hover {
|
|
@@ -146,7 +147,7 @@
|
|
|
146
147
|
background-color: #e88 !important;
|
|
147
148
|
width: 100%;
|
|
148
149
|
text-align: center;
|
|
149
|
-
border-radius:
|
|
150
|
+
border-radius: 8px;
|
|
150
151
|
}
|
|
151
152
|
|
|
152
153
|
input[type="button"].error_major:hover {
|
|
@@ -154,11 +155,11 @@
|
|
|
154
155
|
}
|
|
155
156
|
|
|
156
157
|
input[type="button"].error_delete {
|
|
157
|
-
background-color: #
|
|
158
|
+
background-color: #ddd !important;
|
|
158
159
|
font-size: 10pt;
|
|
159
160
|
width: 100%;
|
|
160
161
|
text-align: center;
|
|
161
|
-
border-radius:
|
|
162
|
+
border-radius: 8px;
|
|
162
163
|
}
|
|
163
164
|
|
|
164
165
|
input[type="button"].error_delete:hover {
|
|
@@ -166,8 +167,69 @@
|
|
|
166
167
|
}
|
|
167
168
|
|
|
168
169
|
input[type=range][orient=vertical] {
|
|
170
|
+
position: relative;
|
|
171
|
+
top: -5px;
|
|
169
172
|
writing-mode: vertical-lr;
|
|
170
173
|
direction: rtl;
|
|
171
174
|
width: 16px;
|
|
172
175
|
height: 200px;
|
|
173
|
-
}
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/* progress bar */
|
|
179
|
+
#progress span {
|
|
180
|
+
font-size: 0pt;
|
|
181
|
+
display: inline-block;
|
|
182
|
+
border-radius: 50%;
|
|
183
|
+
text-align: center;
|
|
184
|
+
line-height: 2em;
|
|
185
|
+
margin-left: 2px;
|
|
186
|
+
box-shadow: 0 1px 1px #0002;
|
|
187
|
+
width: 10px;
|
|
188
|
+
height: 10px;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
#progress span:hover,
|
|
192
|
+
#progress span.progress_current,
|
|
193
|
+
#progress span.progress_incomplete:last-child {
|
|
194
|
+
cursor: pointer;
|
|
195
|
+
user-select: none;
|
|
196
|
+
position: relative;
|
|
197
|
+
top: 7.5px;
|
|
198
|
+
width: 22px;
|
|
199
|
+
height: 22px;
|
|
200
|
+
font-size: 8pt;
|
|
201
|
+
margin-left: -5px;
|
|
202
|
+
margin-right: -5px;
|
|
203
|
+
box-shadow: 0 1px 3px #0002;
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
#progress span:hover {
|
|
207
|
+
z-index: 100;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
#progress span.progress_complete {
|
|
211
|
+
color: white;
|
|
212
|
+
background: #3b5238;
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
#progress span.progress_complete:hover {
|
|
217
|
+
background: #2e3e2b;
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
#progress span.progress_current {
|
|
221
|
+
background: #91b08d;
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
#progress span.progress_current:hover {
|
|
225
|
+
background: #739c6f;
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
#progress span.progress_incomplete {
|
|
229
|
+
background: #bbb;
|
|
230
|
+
color: #555;
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
#progress span.progress_incomplete:hover {
|
|
234
|
+
background: #aaa;
|
|
235
|
+
}</style><script defer="defer" src="pointwise.bundle.js"></script></head><body><div style="max-width: 1600px; min-width: 900px; margin-left: auto; margin-right: auto; margin-top: 20px; padding-left: 10px;"><div class="white-box" style="margin-right: 30px; background-color: #e7e2cf; padding: 5px 15px 5px 5px;"><span id="instructions_global" style="display: inline-block; font-size: 11pt; width: calc(100% - 170px);"><ul id="instructions_spans"><li>Error spans:<ul><li><strong>Select</strong> the part of translation where you have identified a <strong>translation error</strong> (drag or click start & end).</li><li><strong>Click</strong> on the highlight to change error severity (minor/major) or remove the highlight.</li></ul>Choose error severity:<ul><li><span class="instruction_sev" id="instruction_sev_minor">Minor errors:</span> Style, grammar, word choice could be better or more natural.</li><li><span class="instruction_sev" id="instruction_sev_major">Major errors:</span>: The meaning is changed significantly and/or the part is really hard to understand.</li></ul><strong>Tip</strong>: Highlight the word or general area of the error (it doesn't need to be exact). Use separate highlights for different errors.<br></li><li id="instructions_score">Score the translation: Please use the slider and set an overall score based on meaning preservation and general quality:</li><ul><li>0: <strong>No meaning preserved</strong>: most information is lost.</li><li>33%: <strong>Some meaning preserved</strong>: major gaps and narrative issues.</li><li>66%: <strong>Most meaning preserved</strong>: minor issues with grammar or consistency.</li><li>100%: <strong>Perfect</strong>: meaning and grammar align completely with the source.</li></ul><li id="instructions_categories">Error types: After highlighting an error fragment, you will be asked to select the specific error type (main category and subcategory). If you are unsure about which errors fall under which categories, please consult the <a href="https://themqm.org/the-mqm-typology/" style="font-weight: bold; text-decoration: none; color: black;">typology definitions</a>.</li></ul></span><div style="width: 170px; display: inline-block; vertical-align: top; text-align: right; padding-top: 5px;"><span id="time" style="width: 135px; text-align: left; display: inline-block; font-size: 11pt;" title="Approximation of total annotation time.">Time: 0m</span> <input type="button" value="⚙️" id="button_settings" style="height: 1.5em; width: 30px;"><br><br><div id="progress" style="text-align: center;"></div><br><br><input type="button" value="Next 🛠️" id="button_next" disabled="disabled" style="width: 170px; height: 2.5em;" title="Finish annotating all examples first."> <input type="button" value="skip tutorial" id="button_skip_tutorial" style="width: 170px; font-size: 11pt; height: 30px; margin-top: 10px; display: none;" title="Skip tutorial only if you completed it already."></div></div><div id="settings_div" class="white-box" style="margin-right: 20px; margin-top: 10px; display: none; background-color: #e7e2cf; font-size: 11pt;"><input type="checkbox" id="settings_approximate_alignment"> <label for="settings_approximate_alignment">Show approximate alignment</label></div><div id="output_div" style="margin-top: 100px;"></div><br><br><br></div></body></html>
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pearmut
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.1.1
|
|
4
4
|
Summary: A tool for evaluation of model outputs, primarily MT.
|
|
5
5
|
Author-email: Vilém Zouhar <vilem.zouhar@gmail.com>
|
|
6
6
|
License: apache-2.0
|
|
@@ -13,6 +13,7 @@ License-File: LICENSE
|
|
|
13
13
|
Requires-Dist: fastapi>=0.110.0
|
|
14
14
|
Requires-Dist: uvicorn>=0.29.0
|
|
15
15
|
Requires-Dist: wonderwords>=3.0.0
|
|
16
|
+
Requires-Dist: psutil>=7.1.0
|
|
16
17
|
Provides-Extra: dev
|
|
17
18
|
Requires-Dist: pytest; extra == "dev"
|
|
18
19
|
Requires-Dist: pynpm>=0.3.0; extra == "dev"
|
|
@@ -32,9 +33,11 @@ Supports multimodality (text, video, audio, images) and a variety of annotation
|
|
|
32
33
|
|
|
33
34
|
[](https://github.com/zouharvi/pearmut/actions/workflows/ci.yml)
|
|
34
35
|
|
|
35
|
-
<img width="
|
|
36
|
+
<img width="1000" alt="Screenshot of ESA/MQM interface" src="https://github.com/user-attachments/assets/f14c91a5-44d7-4248-ada9-387e95ca59d0" />
|
|
36
37
|
|
|
37
38
|
## Quick start
|
|
39
|
+
|
|
40
|
+
You do not need to clone this repository. Simply install with pip and run locally:
|
|
38
41
|
```bash
|
|
39
42
|
# install the package
|
|
40
43
|
pip install pearmut
|
|
@@ -55,7 +58,7 @@ First, install the package
|
|
|
55
58
|
pip install pearmut
|
|
56
59
|
```
|
|
57
60
|
|
|
58
|
-
A campaign is described in a single JSON file (see [examples/](examples/)
|
|
61
|
+
A campaign is described in a single JSON file (see [examples/](examples/)).
|
|
59
62
|
One of the simplest ones, where each user has a pre-defined list of tasks (`task-based`), is:
|
|
60
63
|
```python
|
|
61
64
|
{
|
|
@@ -65,24 +68,25 @@ One of the simplest ones, where each user has a pre-defined list of tasks (`task
|
|
|
65
68
|
"protocol_score": true, # we want scores [0...100] for each segment
|
|
66
69
|
"protocol_error_spans": true, # we want error spans
|
|
67
70
|
"protocol_error_categories": false, # we do not want error span categories
|
|
68
|
-
"
|
|
69
|
-
"url": "http://localhost:8001" # where the server will be accessible
|
|
71
|
+
"instructions": "Evaluate translation from en to cs_CZ", # message to show to users
|
|
70
72
|
},
|
|
71
73
|
"campaign_id": "wmt25_#_en-cs_CZ",
|
|
72
74
|
"data": [
|
|
73
75
|
# data for first task/user
|
|
74
76
|
[
|
|
75
|
-
|
|
77
|
+
[
|
|
76
78
|
# each evaluation item is a document
|
|
77
|
-
|
|
78
|
-
"This will be the year that Guinness loses its cool. Cheers to that!",
|
|
79
|
-
"
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
"
|
|
83
|
-
"
|
|
84
|
-
|
|
85
|
-
|
|
79
|
+
{
|
|
80
|
+
"src": "This will be the year that Guinness loses its cool. Cheers to that!",
|
|
81
|
+
"tgt": "Nevím přesně, kdy jsem to poprvé zaznamenal. Možná to bylo ve chvíli, ..."
|
|
82
|
+
},
|
|
83
|
+
{
|
|
84
|
+
"src": "I'm not sure I can remember exactly when I sensed it. Maybe it was when some...",
|
|
85
|
+
"tgt": "Tohle bude rok, kdy Guinness přijde o svůj „cool“ faktor. Na zdraví!"
|
|
86
|
+
}
|
|
87
|
+
...
|
|
88
|
+
],
|
|
89
|
+
# more document
|
|
86
90
|
...
|
|
87
91
|
],
|
|
88
92
|
# data for second task/user
|
|
@@ -96,12 +100,19 @@ One of the simplest ones, where each user has a pre-defined list of tasks (`task
|
|
|
96
100
|
In general, the task item can be anything and is handled by the specific protocol template.
|
|
97
101
|
For the standard ones (ESA, DA, MQM), we expect each item to be a dictionary (corresponding to a single document unit) that looks as follows:
|
|
98
102
|
```python
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
103
|
+
# single document definition
|
|
104
|
+
[
|
|
105
|
+
{
|
|
106
|
+
"src": "A najednou se všechna tato voda naplnila dalšími lidmi a dalšími věcmi.", # mandatory for ESA/MQM/DA
|
|
107
|
+
"tgt": "And suddenly all the water became full of other people and other people." # mandatory for ESA/MQM/DA
|
|
108
|
+
},
|
|
109
|
+
{
|
|
110
|
+
"src": "toto je pokračování stejného dokumentu",
|
|
111
|
+
"tgt": "this is a continuation of the same document",
|
|
102
112
|
... # all other keys that will be stored, useful for your analysis
|
|
103
|
-
}
|
|
104
|
-
|
|
113
|
+
}
|
|
114
|
+
],
|
|
115
|
+
... # definition of another item (document)
|
|
105
116
|
```
|
|
106
117
|
|
|
107
118
|
We also support a super simple allocation of annotations (`task-single`, not yet ⚠️), where you simply pass a list of all examples to be evaluated and they are processed in parallel by all annotators:
|
|
@@ -146,6 +157,7 @@ pearmut run
|
|
|
146
157
|
## Campaign management
|
|
147
158
|
|
|
148
159
|
When adding new campaigns or launching pearmut, a management link is shown that gives an overview of annotator progress but also an easy access to the annotation links or resetting the task progress (no data will be lost).
|
|
160
|
+
This is also the place where you can download all progress and collected annotations (these files exist also locally but this might be more convenient).
|
|
149
161
|
|
|
150
162
|
<img width="800" alt="Management dashboard" src="https://github.com/user-attachments/assets/057899d7-2291-46c7-876f-407c4050a9cb" />
|
|
151
163
|
|
|
@@ -165,9 +177,12 @@ Tip: make sure the elements are already appropriately styled.
|
|
|
165
177
|
|
|
166
178
|
## Development
|
|
167
179
|
|
|
168
|
-
|
|
169
|
-
|
|
180
|
+
Pearmut works by running a server that responds to requests from the frontend.
|
|
181
|
+
These requests are not template-based but rather carry only data (which gives flexibility in designing new protocols and interfaces).
|
|
182
|
+
By default, the frontend is served from `static/` which is pre-built when you `pip install pearmut`.
|
|
183
|
+
To make changes locally, clone the repository and run the following, which will recompile the frontend on changes (server changes need server restart):
|
|
170
184
|
```bash
|
|
185
|
+
cd pearmut
|
|
171
186
|
# watch the frontend for changes (in a separate terminal)
|
|
172
187
|
npm install web/ --prefix web/
|
|
173
188
|
npm run watch --prefix web/
|
|
@@ -185,12 +200,15 @@ pearmut run
|
|
|
185
200
|
Optionally, you can specify `--server` in `pearmut add` and `pearmut run` to show correct URL prefixes.
|
|
186
201
|
The `pearmut run` also accepts `--port` (default 8001).
|
|
187
202
|
|
|
203
|
+
If you wish to create a new protocol (referenceable from `info->template`), simply create a new HTML and TS file in `web/src` and add a rule to `webpack.config.js` so that your template gets built.
|
|
204
|
+
A template can call the server for data etc (see [web/src/pointwise.ts](web/src/pointwise.ts) as an exmple).
|
|
205
|
+
|
|
188
206
|
## Citation
|
|
189
207
|
|
|
190
208
|
If you use this work in your paper, please cite as:
|
|
191
209
|
```bibtex
|
|
192
210
|
@misc{zouhar2025pearmut,
|
|
193
|
-
author={Vilém Zouhar
|
|
211
|
+
author={Vilém Zouhar},
|
|
194
212
|
title={Pearmut🍐 Platform for Evaluation and Reviewing of Multilingual Tasks},
|
|
195
213
|
url={https://github.com/zouharvi/pearmut/},
|
|
196
214
|
year={2025},
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
pearmut/app.py,sha256=p438nGXTqdGHTfTA3DHRA7GF71DsBT77D3pcCCOW_Ow,6535
|
|
2
|
+
pearmut/cli.py,sha256=PSF6JxAzGuV2ir6hdL6Q0Enf2G6KUgaf59_uS18LbZk,6929
|
|
3
|
+
pearmut/protocols.py,sha256=qd0X2l9c2_x4YKtK8RswNHdwc7bUb59IP2-GlDA1_R0,4674
|
|
4
|
+
pearmut/utils.py,sha256=6hfVenrVdGm1r-7uJIkWHhX9o0ztWjqPse_j_MqkgBw,1443
|
|
5
|
+
pearmut/static/dashboard.bundle.js,sha256=d_uoUgbBaAzcnT-hTGhxmZ1GxtSvfCI1xw6c8F8AAMY,91230
|
|
6
|
+
pearmut/static/dashboard.html,sha256=tUP1yYvbKySRz0mxFtGq2Si4hTMhJkUCWeTpnq91Nf4,1789
|
|
7
|
+
pearmut/static/index.html,sha256=ieCRLK83MVe-f-gtjYiOlvE-kKd8VnFF2xgyi6FoZpU,872
|
|
8
|
+
pearmut/static/pointwise.bundle.js,sha256=zdVJ2S9vBcd5-BHzbpCM-fCghAmB6pGB21lQE6butOc,100218
|
|
9
|
+
pearmut/static/pointwise.html,sha256=oLedp_oyvddf-uS-wsdbDBKQxrhuGO7OPp1YNXDDOgI,8323
|
|
10
|
+
pearmut/static/assets/favicon.svg,sha256=gVPxdBlyfyJVkiMfh8WLaiSyH4lpwmKZs8UiOeX8YW4,7347
|
|
11
|
+
pearmut/static/assets/style.css,sha256=p1gu_IVTdJMsiW9UtFQwtpiQ-4XillAbU9XLw1KJeGA,964
|
|
12
|
+
pearmut-0.1.1.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
13
|
+
pearmut-0.1.1.dist-info/METADATA,sha256=dHpNQ5RP92d_kx7XaQL8dz5IIJPNgOdAx24NsXXTe6M,8811
|
|
14
|
+
pearmut-0.1.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
15
|
+
pearmut-0.1.1.dist-info/entry_points.txt,sha256=eEA9LVWsS3neQbMvL_nMvEw8I0oFudw8nQa1iqxOiWM,45
|
|
16
|
+
pearmut-0.1.1.dist-info/top_level.txt,sha256=CdgtUM-SKQDt6o5g0QreO-_7XTBP9_wnHMS1P-Rl5Go,8
|
|
17
|
+
pearmut-0.1.1.dist-info/RECORD,,
|
pearmut/model.py
DELETED
|
@@ -1,61 +0,0 @@
|
|
|
1
|
-
# ruff: noqa
|
|
2
|
-
|
|
3
|
-
raise Exception("Deprecated")
|
|
4
|
-
"""
|
|
5
|
-
See scripts/models.py for a list of possible competition models.
|
|
6
|
-
"""
|
|
7
|
-
|
|
8
|
-
import json
|
|
9
|
-
import os
|
|
10
|
-
import random
|
|
11
|
-
|
|
12
|
-
sys1, sys2 = random.sample(systems, 2)
|
|
13
|
-
segment_registry[(sys1, sys2)] += 1
|
|
14
|
-
# TODO: handle overflow better
|
|
15
|
-
if segment_registry[(sys1, sys2)] >= len(data):
|
|
16
|
-
segment_registry[(sys1, sys2)] = 0
|
|
17
|
-
|
|
18
|
-
line = data[segment_registry[(sys1, sys2)]]
|
|
19
|
-
|
|
20
|
-
texts = [highlight_differences(a, b) for a, b in zip(
|
|
21
|
-
line["tgt_text"][sys1],
|
|
22
|
-
line["tgt_text"][sys2],
|
|
23
|
-
)]
|
|
24
|
-
|
|
25
|
-
return JSONResponse(content={
|
|
26
|
-
"doc_id": line["doc_id"],
|
|
27
|
-
# TODO: this is not good sentence splitting
|
|
28
|
-
"src": [line.replace(". ", ".<br><br>") for line in line["src_text"]],
|
|
29
|
-
"sys_a": sys1,
|
|
30
|
-
"out_a": [line_a.replace(". ", ".<br><br>") for line_a, line_b in texts],
|
|
31
|
-
"sys_b": sys2,
|
|
32
|
-
"out_b": [line_b.replace(". ", ".<br><br>") for line_a, line_b in texts],
|
|
33
|
-
})
|
|
34
|
-
|
|
35
|
-
class CompetitionModel():
|
|
36
|
-
def __init__(self, systems):
|
|
37
|
-
if os.path.exists("data/model_elo.json"):
|
|
38
|
-
with open("data/model_elo.json", "r") as f:
|
|
39
|
-
self.scores = json.load(f)
|
|
40
|
-
else:
|
|
41
|
-
print("Initializing new ELO model")
|
|
42
|
-
self.scores = {sys: [] for sys in systems}
|
|
43
|
-
|
|
44
|
-
def system_score(self, sys):
|
|
45
|
-
out = 1000
|
|
46
|
-
for opponent, result in self.scores[sys]:
|
|
47
|
-
out += opponent + result
|
|
48
|
-
return out/len(self.scores[sys]) if self.scores[sys] else out
|
|
49
|
-
|
|
50
|
-
def future_information(self, sys1, sys2):
|
|
51
|
-
pass
|
|
52
|
-
|
|
53
|
-
def record_result(self, sys1, sys2, result):
|
|
54
|
-
self.scores[sys1].append((self.system_score(sys2), 1600*result - 800))
|
|
55
|
-
self.scores[sys2].append((self.system_score(sys1), 1600*(1-result) - 800))
|
|
56
|
-
|
|
57
|
-
self.save()
|
|
58
|
-
|
|
59
|
-
def save(self):
|
|
60
|
-
with open("data/model_elo.json", "w") as f:
|
|
61
|
-
json.dump(self.scores, f)
|
pearmut-0.0.6.dist-info/RECORD
DELETED
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
pearmut/app.py,sha256=66zfaJiVHVXhtORqmJz4RT5K5jXgtShrDqHDRD-EXFQ,6489
|
|
2
|
-
pearmut/cli.py,sha256=7d2Vrt-kWeh-SYkGWdmHUZxcUxEf9hTKXc8qNp-QsLo,5716
|
|
3
|
-
pearmut/model.py,sha256=icC6uE9iQOtoPBtjlaTjCF9l7fb8qrTIlrGeM7k6m4M,1827
|
|
4
|
-
pearmut/protocols.py,sha256=iVOAJiGCvkZy0NWIG_EhO1mpmMInG5xy5ZO5ihP_9lw,4320
|
|
5
|
-
pearmut/utils.py,sha256=6hfVenrVdGm1r-7uJIkWHhX9o0ztWjqPse_j_MqkgBw,1443
|
|
6
|
-
pearmut/static/dashboard.bundle.js,sha256=bd7L6wiFIHTdCk1bgiDkWNhJ-T9OwI3pq8TsisHaubE,91046
|
|
7
|
-
pearmut/static/dashboard.html,sha256=yXwKubqBYdWZ260xRSgNcfebtDVWPl6J5UAa6sj2NOk,1742
|
|
8
|
-
pearmut/static/index.html,sha256=ieCRLK83MVe-f-gtjYiOlvE-kKd8VnFF2xgyi6FoZpU,872
|
|
9
|
-
pearmut/static/pointwise.bundle.js,sha256=C80AaCvHhka-BvB6E1Ub9T5yGPPjESXja9nIk7V_1HI,99780
|
|
10
|
-
pearmut/static/pointwise.html,sha256=ErmUCMHK9jouzigs4sxXUkOOXNT3aH-4RoB7c3HPew0,4899
|
|
11
|
-
pearmut/static/assets/favicon.svg,sha256=gVPxdBlyfyJVkiMfh8WLaiSyH4lpwmKZs8UiOeX8YW4,7347
|
|
12
|
-
pearmut/static/assets/style.css,sha256=jfETRgVCohe680_30GXxbV4Zq4-B6UlXd5pZXlVLIRs,888
|
|
13
|
-
pearmut-0.0.6.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
|
|
14
|
-
pearmut-0.0.6.dist-info/METADATA,sha256=I_1Bh78kQoiN6ShnrAYD2MYxYiwTNFkNUKKdKoi3q0E,7815
|
|
15
|
-
pearmut-0.0.6.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
16
|
-
pearmut-0.0.6.dist-info/entry_points.txt,sha256=eEA9LVWsS3neQbMvL_nMvEw8I0oFudw8nQa1iqxOiWM,45
|
|
17
|
-
pearmut-0.0.6.dist-info/top_level.txt,sha256=CdgtUM-SKQDt6o5g0QreO-_7XTBP9_wnHMS1P-Rl5Go,8
|
|
18
|
-
pearmut-0.0.6.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|