pearmut 0.0.5__py3-none-any.whl → 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (16) hide show
  1. pearmut/{run.py → app.py} +4 -13
  2. pearmut/cli.py +71 -23
  3. pearmut/protocols.py +15 -2
  4. pearmut/static/assets/style.css +12 -8
  5. pearmut/static/dashboard.bundle.js +1 -1
  6. pearmut/static/dashboard.html +4 -3
  7. pearmut/static/pointwise.bundle.js +1 -1
  8. pearmut/static/pointwise.html +87 -23
  9. {pearmut-0.0.5.dist-info → pearmut-0.1.0.dist-info}/METADATA +56 -26
  10. pearmut-0.1.0.dist-info/RECORD +17 -0
  11. pearmut/model.py +0 -61
  12. pearmut-0.0.5.dist-info/RECORD +0 -18
  13. {pearmut-0.0.5.dist-info → pearmut-0.1.0.dist-info}/WHEEL +0 -0
  14. {pearmut-0.0.5.dist-info → pearmut-0.1.0.dist-info}/entry_points.txt +0 -0
  15. {pearmut-0.0.5.dist-info → pearmut-0.1.0.dist-info}/licenses/LICENSE +0 -0
  16. {pearmut-0.0.5.dist-info → pearmut-0.1.0.dist-info}/top_level.txt +0 -0
pearmut/static/pointwise.html CHANGED
@@ -1,19 +1,21 @@
1
1
  <!doctype html><html lang="en" style="height: 100%;"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>Pearmut Evaluation</title><link rel="icon" type="image/svg+xml" href="assets/favicon.svg"><link rel="stylesheet" href="assets/style.css"><style>.output_srctgt {
2
2
  display: inline-block;
3
- width: calc(100% - 150px);
3
+ width: calc(100% - 140px);
4
4
  vertical-align: top;
5
5
  }
6
6
 
7
- .output_src, .output_tgt {
7
+ .output_src,
8
+ .output_tgt {
8
9
  display: inline-block;
9
10
  width: calc(50% - 30px);
10
11
  background-color: #fffc;
11
12
  padding: 10px;
12
- border-radius: 4px;
13
+ border-radius: 8px;
13
14
  vertical-align: top;
14
15
  margin-right: 2px;
15
16
  margin-left: 2px;
16
17
  user-select: none;
18
+ box-shadow: 0 4px 6px #0005;
17
19
  }
18
20
 
19
21
  .output_src {
@@ -30,12 +32,6 @@
30
32
  margin-bottom: 30pt;
31
33
  }
32
34
 
33
- input[type="range"] {
34
- height: 150px;
35
- position: relative;
36
- top: -5px;
37
- }
38
-
39
35
  .output_labels {
40
36
  vertical-align: top;
41
37
  display: inline-block;
@@ -59,7 +55,8 @@
59
55
  margin-right: 30px;
60
56
  }
61
57
 
62
- .output_src, .output_tgt {
58
+ .output_src,
59
+ .output_tgt {
63
60
  width: 100%;
64
61
  margin-bottom: 5px;
65
62
  margin-right: 10px;
@@ -68,9 +65,10 @@
68
65
 
69
66
  .tgt_char:hover {
70
67
  background-color: #ccc;
71
- cursor: pointer;;
68
+ cursor: pointer;
69
+ ;
72
70
  }
73
-
71
+
74
72
  .src_char:hover {
75
73
  background-color: #ccc;
76
74
  }
@@ -91,11 +89,13 @@
91
89
  background-color: #e88;
92
90
  }
93
91
 
94
- .src_char.highlighted, .tgt_char.highlighted {
92
+ .src_char.highlighted,
93
+ .tgt_char.highlighted {
95
94
  background-color: #ccc;
96
95
  }
97
96
 
98
- #button_error_minor:hover, #button_error_major:hover {
97
+ #button_error_minor:hover,
98
+ #button_error_major:hover {
99
99
  opacity: 0.8;
100
100
  cursor: pointer;
101
101
  }
@@ -106,9 +106,10 @@
106
106
 
107
107
  /* span toolbox section */
108
108
  .span_toolbox {
109
- background-color: #ddf;
109
+ background-color: white;
110
110
  padding: 5px;
111
- border-radius: 5px;
111
+ border-radius: 8px;
112
+ box-shadow: 0 4px 6px #0005;
112
113
  }
113
114
 
114
115
  .span_toolbox_parent {
@@ -116,13 +117,15 @@
116
117
  padding-left: 20px;
117
118
  padding-right: 20px;
118
119
  padding-top: 10px;
120
+ /* always maximally spread out children */
121
+ min-width: max-content;
119
122
  }
120
123
 
121
124
  input[type="button"].error_neutral {
122
- background-color: #fee !important;
125
+ background-color: #ecc9 !important;
123
126
  width: 100%;
124
127
  text-align: center;
125
- border-radius: 5pt;
128
+ border-radius: 8px;
126
129
  }
127
130
 
128
131
  input[type="button"].error_neutral:hover {
@@ -133,7 +136,7 @@
133
136
  background-color: #fcc !important;
134
137
  width: 100%;
135
138
  text-align: center;
136
- border-radius: 5pt;
139
+ border-radius: 8px;
137
140
  }
138
141
 
139
142
  input[type="button"].error_minor:hover {
@@ -144,7 +147,7 @@
144
147
  background-color: #e88 !important;
145
148
  width: 100%;
146
149
  text-align: center;
147
- border-radius: 5pt;
150
+ border-radius: 8px;
148
151
  }
149
152
 
150
153
  input[type="button"].error_major:hover {
@@ -152,11 +155,11 @@
152
155
  }
153
156
 
154
157
  input[type="button"].error_delete {
155
- background-color: #eee !important;
158
+ background-color: #ddd !important;
156
159
  font-size: 10pt;
157
160
  width: 100%;
158
161
  text-align: center;
159
- border-radius: 5pt;
162
+ border-radius: 8px;
160
163
  }
161
164
 
162
165
  input[type="button"].error_delete:hover {
@@ -164,8 +167,69 @@
164
167
  }
165
168
 
166
169
  input[type=range][orient=vertical] {
170
+ position: relative;
171
+ top: -5px;
167
172
  writing-mode: vertical-lr;
168
173
  direction: rtl;
169
174
  width: 16px;
170
175
  height: 200px;
171
- }</style><script defer="defer" src="pointwise.bundle.js"></script></head><body><div style="max-width: 1600px; min-width: 900px; margin-left: auto; margin-right: auto; margin-top: 20px; padding-left: 10px;"><div style="display: flex;"><input type="button" value="⚙️" id="button_settings" style="margin-right: 20px; height: 1.5em; margin-top: -5px;"> <span id="progress" style="flex: 0 0 140px;">Annotated: 0/0</span> <span id="time" style="flex: 0 0 190px;">Annotation time: 0m</span> <span id="status_message" style="margin-left: 20px; flex-grow: 1; vertical-align: top;"></span> <input type="button" value="Next 🛠️" id="button_next" disabled="disabled" style="flex: 0 0 150px; margin-right: 20px; margin-left: 20px; margin-top: -10px; height: 2.5em;" title="Finish annotating all examples first."></div><div id="settings_div" class="white-box" style="margin-right: 20px; margin-top: 10px; display: none;"><input type="checkbox" id="settings_approximate_alignment"> <label for="settings_approximate_alignment">Show approximate alignment</label></div><div id="output_div" style="margin-top: 100px;"></div><br><br><br></div></body></html>
176
+ }
177
+
178
+ /* progress bar */
179
+ #progress span {
180
+ font-size: 0pt;
181
+ display: inline-block;
182
+ border-radius: 50%;
183
+ text-align: center;
184
+ line-height: 2em;
185
+ margin-left: 2px;
186
+ box-shadow: 0 1px 1px #0002;
187
+ width: 10px;
188
+ height: 10px;
189
+ }
190
+
191
+ #progress span:hover,
192
+ #progress span.progress_current,
193
+ #progress span.progress_incomplete:last-child {
194
+ cursor: pointer;
195
+ user-select: none;
196
+ position: relative;
197
+ top: 7.5px;
198
+ width: 22px;
199
+ height: 22px;
200
+ font-size: 8pt;
201
+ margin-left: -5px;
202
+ margin-right: -5px;
203
+ box-shadow: 0 1px 3px #0002;
204
+ }
205
+
206
+ #progress span:hover {
207
+ z-index: 100;
208
+ }
209
+
210
+ #progress span.progress_complete {
211
+ color: white;
212
+ background: #3b5238;
213
+ }
214
+
215
+
216
+ #progress span.progress_complete:hover {
217
+ background: #2e3e2b;
218
+ }
219
+
220
+ #progress span.progress_current {
221
+ background: #91b08d;
222
+ }
223
+
224
+ #progress span.progress_current:hover {
225
+ background: #739c6f;
226
+ }
227
+
228
+ #progress span.progress_incomplete {
229
+ background: #bbb;
230
+ color: #555;
231
+ }
232
+
233
+ #progress span.progress_incomplete:hover {
234
+ background: #aaa;
235
+ }</style><script defer="defer" src="pointwise.bundle.js"></script></head><body><div style="max-width: 1600px; min-width: 900px; margin-left: auto; margin-right: auto; margin-top: 20px; padding-left: 10px;"><div class="white-box" style="margin-right: 30px; background-color: #e7e2cf; padding: 5px 15px 5px 5px;"><span id="instructions_global" style="display: inline-block; font-size: 11pt; width: calc(100% - 170px);"><ul id="instructions_spans"><li>Error spans:<ul><li><strong>Select</strong> the part of translation where you have identified a <strong>translation error</strong> (drag or click start & end).</li><li><strong>Click</strong> on the highlight to change error severity (minor/major) or remove the highlight.</li></ul>Choose error severity:<ul><li><span class="instruction_sev" id="instruction_sev_minor">Minor errors:</span> Style, grammar, word choice could be better or more natural.</li><li><span class="instruction_sev" id="instruction_sev_major">Major errors:</span>: The meaning is changed significantly and/or the part is really hard to understand.</li></ul><strong>Tip</strong>: Highlight the word or general area of the error (it doesn't need to be exact). Use separate highlights for different errors.<br></li><li id="instructions_score">Score the translation: Please use the slider and set an overall score based on meaning preservation and general quality:</li><ul><li>0: <strong>No meaning preserved</strong>: most information is lost.</li><li>33%: <strong>Some meaning preserved</strong>: major gaps and narrative issues.</li><li>66%: <strong>Most meaning preserved</strong>: minor issues with grammar or consistency.</li><li>100%: <strong>Perfect</strong>: meaning and grammar align completely with the source.</li></ul><li id="instructions_categories">Error types: After highlighting an error fragment, you will be asked to select the specific error type (main category and subcategory). If you are unsure about which errors fall under which categories, please consult the <a href="https://themqm.org/the-mqm-typology/" style="font-weight: bold; text-decoration: underline;">typology definitions</a>.</li></ul></span><div style="width: 170px; display: inline-block; vertical-align: top; text-align: right; padding-top: 5px;"><span id="time" style="width: 135px; text-align: left; display: inline-block; font-size: 11pt;" title="Approximation of total annotation time.">Time: 0m</span> <input type="button" value="⚙️" id="button_settings" style="height: 1.5em; width: 30px;"><br><br><div id="progress" style="text-align: center;"></div><br><br><input type="button" value="Next 🛠️" id="button_next" disabled="disabled" style="width: 170px; height: 2.5em;" title="Finish annotating all examples first."> <input type="button" value="skip tutorial" id="button_skip_tutorial" style="width: 170px; font-size: 11pt; height: 30px; margin-top: 10px; display: none;" title="Skip tutorial only if you completed it already."></div></div><div id="settings_div" class="white-box" style="margin-right: 20px; margin-top: 10px; display: none; background-color: #e7e2cf; font-size: 11pt;"><input type="checkbox" id="settings_approximate_alignment"> <label for="settings_approximate_alignment">Show approximate alignment</label></div><div id="output_div" style="margin-top: 100px;"></div><br><br><br></div></body></html>
{pearmut-0.0.5.dist-info → pearmut-0.1.0.dist-info}/METADATA RENAMED
@@ -1,9 +1,9 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: pearmut
3
- Version: 0.0.5
3
+ Version: 0.1.0
4
4
  Summary: A tool for evaluation of model outputs, primarily MT.
5
5
  Author-email: Vilém Zouhar <vilem.zouhar@gmail.com>
6
- License: MIT
6
+ License: apache-2.0
7
7
  Project-URL: Repository, https://github.com/zouharvi/pearmut
8
8
  Project-URL: Issues, https://github.com/zouharvi/pearmut/issues
9
9
  Keywords: evaluation,machine translation,human evaluation,annotation
@@ -13,6 +13,7 @@ License-File: LICENSE
13
13
  Requires-Dist: fastapi>=0.110.0
14
14
  Requires-Dist: uvicorn>=0.29.0
15
15
  Requires-Dist: wonderwords>=3.0.0
16
+ Requires-Dist: psutil>=7.1.0
16
17
  Provides-Extra: dev
17
18
  Requires-Dist: pytest; extra == "dev"
18
19
  Requires-Dist: pynpm>=0.3.0; extra == "dev"
@@ -32,9 +33,11 @@ Supports multimodality (text, video, audio, images) and a variety of annotation
32
33
  &nbsp;
33
34
  [![build status](https://github.com/zouharvi/pearmut/actions/workflows/ci.yml/badge.svg)](https://github.com/zouharvi/pearmut/actions/workflows/ci.yml)
34
35
 
35
- <img width="1334" alt="Screenshot of ESA/MQM interface" src="https://github.com/user-attachments/assets/dde04b98-c724-4226-b926-011a89e9ce31" />
36
+ <img width="1000" alt="Screenshot of ESA/MQM interface" src="https://github.com/user-attachments/assets/f14c91a5-44d7-4248-ada9-387e95ca59d0" />
36
37
 
37
- ## Getting started fast
38
+ ## Quick start
39
+
40
+ You do not need to clone this repository. Simply install with pip and run locally:
38
41
  ```bash
39
42
  # install the package
40
43
  pip install pearmut
@@ -55,7 +58,7 @@ First, install the package
55
58
  pip install pearmut
56
59
  ```
57
60
 
58
- A campaign is described in a single JSON file (see [examples/](examples/)!).
61
+ A campaign is described in a single JSON file (see [examples/](examples/)).
59
62
  One of the simplest ones, where each user has a pre-defined list of tasks (`task-based`), is:
60
63
  ```python
61
64
  {
@@ -65,24 +68,25 @@ One of the simplest ones, where each user has a pre-defined list of tasks (`task
65
68
  "protocol_score": true, # we want scores [0...100] for each segment
66
69
  "protocol_error_spans": true, # we want error spans
67
70
  "protocol_error_categories": false, # we do not want error span categories
68
- "status_message": "Evaluate translation from en to cs_CZ", # message to show to users
69
- "url": "http://localhost:8001" # where the server will be accessible
71
+ "instructions": "Evaluate translation from en to cs_CZ", # message to show to users
70
72
  },
71
73
  "campaign_id": "wmt25_#_en-cs_CZ",
72
74
  "data": [
73
75
  # data for first task/user
74
76
  [
75
- {
77
+ [
76
78
  # each evaluation item is a document
77
- "src": [
78
- "This will be the year that Guinness loses its cool. Cheers to that!",
79
- "I'm not sure I can remember exactly when I sensed it. Maybe it was when some...",
80
- ],
81
- "tgt": [
82
- "Tohle bude rok, kdy Guinness přijde o svůj „cool“ faktor. Na zdraví!",
83
- "Nevím přesně, kdy jsem to poprvé zaznamenal. Možná to bylo ve chvíli, ...",
84
- ]
85
- },
79
+ {
80
+ "src": "This will be the year that Guinness loses its cool. Cheers to that!",
81
+ "tgt": "Nevím přesně, kdy jsem to poprvé zaznamenal. Možná to bylo ve chvíli, ..."
82
+ },
83
+ {
84
+ "src": "I'm not sure I can remember exactly when I sensed it. Maybe it was when some...",
85
+ "tgt": "Tohle bude rok, kdy Guinness přijde o svůj „cool“ faktor. Na zdraví!"
86
+ }
87
+ ...
88
+ ],
89
+ # more document
86
90
  ...
87
91
  ],
88
92
  # data for second task/user
@@ -96,12 +100,19 @@ One of the simplest ones, where each user has a pre-defined list of tasks (`task
96
100
  In general, the task item can be anything and is handled by the specific protocol template.
97
101
  For the standard ones (ESA, DA, MQM), we expect each item to be a dictionary (corresponding to a single document unit) that looks as follows:
98
102
  ```python
99
- { # single document definition
100
- "src": ["A najednou se všechna tato voda naplnila dalšími lidmi a dalšími věcmi.", "toto je pokračování stejného dokumentu"], # mandatory for ESA/MQM/DA
101
- "tgt": ["And suddenly all the water became full of other people and other people.", "this is a continuation of the same document"], # mandatory for ESA/MQM/DA
103
+ # single document definition
104
+ [
105
+ {
106
+ "src": "A najednou se všechna tato voda naplnila dalšími lidmi a dalšími věcmi.", # mandatory for ESA/MQM/DA
107
+ "tgt": "And suddenly all the water became full of other people and other people." # mandatory for ESA/MQM/DA
108
+ },
109
+ {
110
+ "src": "toto je pokračování stejného dokumentu",
111
+ "tgt": "this is a continuation of the same document",
102
112
  ... # all other keys that will be stored, useful for your analysis
103
- },
104
- ... # definition of another item
113
+ }
114
+ ],
115
+ ... # definition of another item (document)
105
116
  ```
106
117
 
107
118
  We also support a super simple allocation of annotations (`task-single`, not yet ⚠️), where you simply pass a list of all examples to be evaluated and they are processed in parallel by all annotators:
@@ -146,6 +157,7 @@ pearmut run
146
157
  ## Campaign management
147
158
 
148
159
  When adding new campaigns or launching pearmut, a management link is shown that gives an overview of annotator progress but also an easy access to the annotation links or resetting the task progress (no data will be lost).
160
+ This is also the place where you can download all progress and collected annotations (these files exist also locally but this might be more convenient).
149
161
 
150
162
  <img width="800" alt="Management dashboard" src="https://github.com/user-attachments/assets/057899d7-2291-46c7-876f-407c4050a9cb" />
151
163
 
@@ -154,11 +166,23 @@ An intentionally incorrect token can be shown if the annotations don't pass qual
154
166
 
155
167
  <img width="500" alt="Token on completion" src="https://github.com/user-attachments/assets/4b4d2aa9-7bab-44d6-894b-6c789cd3bc6e" />
156
168
 
157
- ## Development
169
+ ## Multimodal Annotations
170
+
171
+ We also support anything HTML-compatible both on the input and on the output.
172
+ This includes embedded YouTube videos, or even simple `<video ` tags that point to some resource somewhere.
173
+ For an example, try [examples/mock_multimodal.json](examples/mock_multimodal.json).
174
+ Tip: make sure the elements are already appropriately styled.
158
175
 
159
- For the server and frontend locally run:
176
+ <img width="800" alt="Preview of multimodal elements in Pearmut" src="https://github.com/user-attachments/assets/f34a1a3e-ad95-4114-95ee-8a49e8003faf" />
160
177
 
178
+ ## Development
179
+
180
+ Pearmut works by running a server that responds to requests from the frontend.
181
+ These requests are not template-based but rather carry only data (which gives flexibility in designing new protocols and interfaces).
182
+ By default, the frontend is served from `static/` which is pre-built when you `pip install pearmut`.
183
+ To make changes locally, clone the repository and run the following, which will recompile the frontend on changes (server changes need server restart):
161
184
  ```bash
185
+ cd pearmut
162
186
  # watch the frontend for changes (in a separate terminal)
163
187
  npm install web/ --prefix web/
164
188
  npm run watch --prefix web/
@@ -169,16 +193,22 @@ pip3 install -e .
169
193
  # sets up progress/log files in current working folder
170
194
  pearmut add examples/wmt25_#_en-cs_CZ.json
171
195
  pearmut add examples/wmt25_#_cs-de_DE.json
172
- # shows a management link for all loaded campaigns
196
+ # shows a management link for all loaded campaigns and reload on change
173
197
  pearmut run
174
198
  ```
175
199
 
200
+ Optionally, you can specify `--server` in `pearmut add` and `pearmut run` to show correct URL prefixes.
201
+ The `pearmut run` also accepts `--port` (default 8001).
202
+
203
+ If you wish to create a new protocol (referenceable from `info->template`), simply create a new HTML and TS file in `web/src` and add a rule to `webpack.config.js` so that your template gets built.
204
+ A template can call the server for data etc (see [web/src/pointwise.ts](web/src/pointwise.ts) as an exmple).
205
+
176
206
  ## Citation
177
207
 
178
208
  If you use this work in your paper, please cite as:
179
209
  ```bibtex
180
210
  @misc{zouhar2025pearmut,
181
- author={Vilém Zouhar and others},
211
+ author={Vilém Zouhar},
182
212
  title={Pearmut🍐 Platform for Evaluation and Reviewing of Multilingual Tasks},
183
213
  url={https://github.com/zouharvi/pearmut/},
184
214
  year={2025},
pearmut-0.1.0.dist-info/RECORD ADDED
@@ -0,0 +1,17 @@
1
+ pearmut/app.py,sha256=p438nGXTqdGHTfTA3DHRA7GF71DsBT77D3pcCCOW_Ow,6535
2
+ pearmut/cli.py,sha256=PSF6JxAzGuV2ir6hdL6Q0Enf2G6KUgaf59_uS18LbZk,6929
3
+ pearmut/protocols.py,sha256=qd0X2l9c2_x4YKtK8RswNHdwc7bUb59IP2-GlDA1_R0,4674
4
+ pearmut/utils.py,sha256=6hfVenrVdGm1r-7uJIkWHhX9o0ztWjqPse_j_MqkgBw,1443
5
+ pearmut/static/dashboard.bundle.js,sha256=d_uoUgbBaAzcnT-hTGhxmZ1GxtSvfCI1xw6c8F8AAMY,91230
6
+ pearmut/static/dashboard.html,sha256=tUP1yYvbKySRz0mxFtGq2Si4hTMhJkUCWeTpnq91Nf4,1789
7
+ pearmut/static/index.html,sha256=ieCRLK83MVe-f-gtjYiOlvE-kKd8VnFF2xgyi6FoZpU,872
8
+ pearmut/static/pointwise.bundle.js,sha256=zdVJ2S9vBcd5-BHzbpCM-fCghAmB6pGB21lQE6butOc,100218
9
+ pearmut/static/pointwise.html,sha256=F4mWSnUf9DPf7LGyzQTvI1MgKb84ov9NItWzoFn1eIU,8314
10
+ pearmut/static/assets/favicon.svg,sha256=gVPxdBlyfyJVkiMfh8WLaiSyH4lpwmKZs8UiOeX8YW4,7347
11
+ pearmut/static/assets/style.css,sha256=p1gu_IVTdJMsiW9UtFQwtpiQ-4XillAbU9XLw1KJeGA,964
12
+ pearmut-0.1.0.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
13
+ pearmut-0.1.0.dist-info/METADATA,sha256=2SCD_wBHfpjAo1gr7Ygsc1ESvVKeZqkB2E_0wsIBGfM,8811
14
+ pearmut-0.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
15
+ pearmut-0.1.0.dist-info/entry_points.txt,sha256=eEA9LVWsS3neQbMvL_nMvEw8I0oFudw8nQa1iqxOiWM,45
16
+ pearmut-0.1.0.dist-info/top_level.txt,sha256=CdgtUM-SKQDt6o5g0QreO-_7XTBP9_wnHMS1P-Rl5Go,8
17
+ pearmut-0.1.0.dist-info/RECORD,,
pearmut/model.py DELETED
@@ -1,61 +0,0 @@
1
- # ruff: noqa
2
-
3
- raise Exception("Deprecated")
4
- """
5
- See scripts/models.py for a list of possible competition models.
6
- """
7
-
8
- import json
9
- import os
10
- import random
11
-
12
- sys1, sys2 = random.sample(systems, 2)
13
- segment_registry[(sys1, sys2)] += 1
14
- # TODO: handle overflow better
15
- if segment_registry[(sys1, sys2)] >= len(data):
16
- segment_registry[(sys1, sys2)] = 0
17
-
18
- line = data[segment_registry[(sys1, sys2)]]
19
-
20
- texts = [highlight_differences(a, b) for a, b in zip(
21
- line["tgt_text"][sys1],
22
- line["tgt_text"][sys2],
23
- )]
24
-
25
- return JSONResponse(content={
26
- "doc_id": line["doc_id"],
27
- # TODO: this is not good sentence splitting
28
- "src": [line.replace(". ", ".<br><br>") for line in line["src_text"]],
29
- "sys_a": sys1,
30
- "out_a": [line_a.replace(". ", ".<br><br>") for line_a, line_b in texts],
31
- "sys_b": sys2,
32
- "out_b": [line_b.replace(". ", ".<br><br>") for line_a, line_b in texts],
33
- })
34
-
35
- class CompetitionModel():
36
- def __init__(self, systems):
37
- if os.path.exists("data/model_elo.json"):
38
- with open("data/model_elo.json", "r") as f:
39
- self.scores = json.load(f)
40
- else:
41
- print("Initializing new ELO model")
42
- self.scores = {sys: [] for sys in systems}
43
-
44
- def system_score(self, sys):
45
- out = 1000
46
- for opponent, result in self.scores[sys]:
47
- out += opponent + result
48
- return out/len(self.scores[sys]) if self.scores[sys] else out
49
-
50
- def future_information(self, sys1, sys2):
51
- pass
52
-
53
- def record_result(self, sys1, sys2, result):
54
- self.scores[sys1].append((self.system_score(sys2), 1600*result - 800))
55
- self.scores[sys2].append((self.system_score(sys1), 1600*(1-result) - 800))
56
-
57
- self.save()
58
-
59
- def save(self):
60
- with open("data/model_elo.json", "w") as f:
61
- json.dump(self.scores, f)
pearmut-0.0.5.dist-info/RECORD DELETED
@@ -1,18 +0,0 @@
1
- pearmut/cli.py,sha256=L6mC4P0LpA8c8FPovOrMHlNHKUbHKKMNVfUu1nTv23I,5202
2
- pearmut/model.py,sha256=icC6uE9iQOtoPBtjlaTjCF9l7fb8qrTIlrGeM7k6m4M,1827
3
- pearmut/protocols.py,sha256=iVOAJiGCvkZy0NWIG_EhO1mpmMInG5xy5ZO5ihP_9lw,4320
4
- pearmut/run.py,sha256=N318GMjvYRVSxSB29JT_27n2AonzrXgHr3T60f54T9o,6847
5
- pearmut/utils.py,sha256=6hfVenrVdGm1r-7uJIkWHhX9o0ztWjqPse_j_MqkgBw,1443
6
- pearmut/static/dashboard.bundle.js,sha256=bd7L6wiFIHTdCk1bgiDkWNhJ-T9OwI3pq8TsisHaubE,91046
7
- pearmut/static/dashboard.html,sha256=yXwKubqBYdWZ260xRSgNcfebtDVWPl6J5UAa6sj2NOk,1742
8
- pearmut/static/index.html,sha256=ieCRLK83MVe-f-gtjYiOlvE-kKd8VnFF2xgyi6FoZpU,872
9
- pearmut/static/pointwise.bundle.js,sha256=_VP8LWVISWwkINPZ0he5jrdQCBcy_8novl6Ix47tBIU,99378
10
- pearmut/static/pointwise.html,sha256=jVALsvYpgqfV1S3bw0Rf245oW7E9LHh-Zu4ctGDgFp4,4820
11
- pearmut/static/assets/favicon.svg,sha256=gVPxdBlyfyJVkiMfh8WLaiSyH4lpwmKZs8UiOeX8YW4,7347
12
- pearmut/static/assets/style.css,sha256=jfETRgVCohe680_30GXxbV4Zq4-B6UlXd5pZXlVLIRs,888
13
- pearmut-0.0.5.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
14
- pearmut-0.0.5.dist-info/METADATA,sha256=xQXEXn-BclFnsL84dRamFtkF55AEkMvwlQFGsukEn7U,7119
15
- pearmut-0.0.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
16
- pearmut-0.0.5.dist-info/entry_points.txt,sha256=eEA9LVWsS3neQbMvL_nMvEw8I0oFudw8nQa1iqxOiWM,45
17
- pearmut-0.0.5.dist-info/top_level.txt,sha256=CdgtUM-SKQDt6o5g0QreO-_7XTBP9_wnHMS1P-Rl5Go,8
18
- pearmut-0.0.5.dist-info/RECORD,,