potassco-benchmark-tool 2.1.1__py3-none-any.whl → 2.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- benchmarktool/entry_points.py +71 -33
- benchmarktool/init/runscripts/runscript-all.xml +2 -2
- benchmarktool/init/runscripts/runscript-dist.xml +2 -2
- benchmarktool/init/runscripts/runscript-example.xml +1 -1
- benchmarktool/init/templates/seq-generic.sh +20 -5
- benchmarktool/result/ipynb_gen.py +2 -0
- benchmarktool/result/result.py +26 -16
- benchmarktool/result/xlsx_gen.py +935 -0
- benchmarktool/resultparser/clasp.py +20 -9
- benchmarktool/runscript/parser.py +235 -134
- benchmarktool/runscript/runscript.py +190 -191
- benchmarktool/tools.py +22 -2
- {potassco_benchmark_tool-2.1.1.dist-info → potassco_benchmark_tool-2.2.1.dist-info}/METADATA +24 -11
- potassco_benchmark_tool-2.2.1.dist-info/RECORD +26 -0
- benchmarktool/init/templates/seq-generic-single.sh +0 -27
- benchmarktool/init/templates/seq-generic-zip.sh +0 -14
- benchmarktool/result/ods_config.py +0 -42
- benchmarktool/result/ods_gen.py +0 -714
- potassco_benchmark_tool-2.1.1.dist-info/RECORD +0 -29
- {potassco_benchmark_tool-2.1.1.dist-info → potassco_benchmark_tool-2.2.1.dist-info}/WHEEL +0 -0
- {potassco_benchmark_tool-2.1.1.dist-info → potassco_benchmark_tool-2.2.1.dist-info}/entry_points.txt +0 -0
- {potassco_benchmark_tool-2.1.1.dist-info → potassco_benchmark_tool-2.2.1.dist-info}/licenses/LICENSE +0 -0
- {potassco_benchmark_tool-2.1.1.dist-info → potassco_benchmark_tool-2.2.1.dist-info}/top_level.txt +0 -0
benchmarktool/result/ods_gen.py
DELETED
|
@@ -1,714 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
Created on Apr 14, 2025
|
|
3
|
-
|
|
4
|
-
@author: Tom Schmidt
|
|
5
|
-
"""
|
|
6
|
-
|
|
7
|
-
import re
|
|
8
|
-
import warnings
|
|
9
|
-
from dataclasses import dataclass, field
|
|
10
|
-
from typing import TYPE_CHECKING, Any, Optional
|
|
11
|
-
from unittest import mock
|
|
12
|
-
|
|
13
|
-
import numpy as np
|
|
14
|
-
import odswriter as ods # type: ignore[import-untyped]
|
|
15
|
-
import pandas as pd # type: ignore[import-untyped]
|
|
16
|
-
|
|
17
|
-
from benchmarktool.result import ods_config
|
|
18
|
-
|
|
19
|
-
if TYPE_CHECKING:
|
|
20
|
-
from benchmarktool.result import result # nocoverage
|
|
21
|
-
|
|
22
|
-
ods.ods_components.styles_xml = ods_config.STYLES_XML
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
# pylint: disable=all
|
|
26
|
-
def write_row(self: ods.Sheet, cells: list[Any]) -> None: # pragma: no cover
|
|
27
|
-
"""
|
|
28
|
-
Method to write ods row.
|
|
29
|
-
Replaces ods.Sheet.writerow
|
|
30
|
-
"""
|
|
31
|
-
import decimal
|
|
32
|
-
|
|
33
|
-
row = self.dom.createElement("table:table-row")
|
|
34
|
-
content_cells = 0
|
|
35
|
-
|
|
36
|
-
for cell_data in cells:
|
|
37
|
-
cell = self.dom.createElement("table:table-cell")
|
|
38
|
-
text = None
|
|
39
|
-
|
|
40
|
-
if isinstance(cell_data, tuple):
|
|
41
|
-
value = cell_data[0]
|
|
42
|
-
style = cell_data[1]
|
|
43
|
-
else:
|
|
44
|
-
value = cell_data
|
|
45
|
-
style = None
|
|
46
|
-
|
|
47
|
-
if isinstance(value, bool):
|
|
48
|
-
# Bool condition must be checked before numeric because:
|
|
49
|
-
# isinstance(True, int): True
|
|
50
|
-
# isinstance(True, bool): True
|
|
51
|
-
cell.setAttribute("office:value-type", "boolean")
|
|
52
|
-
cell.setAttribute("office:boolean-value", "true" if value else "false")
|
|
53
|
-
cell.setAttribute("table:style-name", "cBool")
|
|
54
|
-
text = "TRUE" if value else "FALSE"
|
|
55
|
-
|
|
56
|
-
elif isinstance(value, (float, int, decimal.Decimal, int)):
|
|
57
|
-
cell.setAttribute("office:value-type", "float")
|
|
58
|
-
float_str = str(value)
|
|
59
|
-
cell.setAttribute("office:value", float_str)
|
|
60
|
-
if style is not None:
|
|
61
|
-
cell.setAttribute("table:style-name", style)
|
|
62
|
-
text = float_str
|
|
63
|
-
|
|
64
|
-
elif isinstance(value, Formula):
|
|
65
|
-
cell.setAttribute("table:formula", str(value))
|
|
66
|
-
if style is not None:
|
|
67
|
-
cell.setAttribute("table:style-name", style)
|
|
68
|
-
|
|
69
|
-
elif value is None:
|
|
70
|
-
pass # Empty element
|
|
71
|
-
|
|
72
|
-
else:
|
|
73
|
-
# String and unknown types become string cells
|
|
74
|
-
cell.setAttribute("office:value-type", "string")
|
|
75
|
-
if style is not None:
|
|
76
|
-
cell.setAttribute("table:style-name", style)
|
|
77
|
-
text = str(value)
|
|
78
|
-
|
|
79
|
-
if text:
|
|
80
|
-
p = self.dom.createElement("text:p")
|
|
81
|
-
p.appendChild(self.dom.createTextNode(text))
|
|
82
|
-
cell.appendChild(p)
|
|
83
|
-
|
|
84
|
-
row.appendChild(cell)
|
|
85
|
-
|
|
86
|
-
content_cells += 1
|
|
87
|
-
|
|
88
|
-
if self.cols is not None:
|
|
89
|
-
if content_cells > self.cols:
|
|
90
|
-
raise Exception("More cells than cols.")
|
|
91
|
-
|
|
92
|
-
for _ in range(content_cells, self.cols):
|
|
93
|
-
cell = self.dom.createElement("table:table-cell")
|
|
94
|
-
row.appendChild(cell)
|
|
95
|
-
|
|
96
|
-
self.table.appendChild(row)
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
class Formula(ods.Formula): # type: ignore[misc]
|
|
100
|
-
"""
|
|
101
|
-
Extending odswriter.Formula class with the ability to
|
|
102
|
-
handle sheet references and some minor fixes.
|
|
103
|
-
"""
|
|
104
|
-
|
|
105
|
-
def __str__(self) -> str:
|
|
106
|
-
"""
|
|
107
|
-
Get ods string representation.
|
|
108
|
-
"""
|
|
109
|
-
s = self.formula_string
|
|
110
|
-
# remove leading '='
|
|
111
|
-
if s.startswith("="):
|
|
112
|
-
s = s[1:]
|
|
113
|
-
# wrap references
|
|
114
|
-
s = re.sub(r"([\w\.]*\$?[A-Z]+\$?[0-9]+(:[\w\.]*\$?[A-Z]+\$?[0-9]+)?)", r"[\1]", s)
|
|
115
|
-
# add '.' before references if necessary
|
|
116
|
-
s = re.sub(r"(?<=[^$A-Z0-9\.])(\$?[A-Z]+\$?[0-9]+)(?![\(\.])", r".\1", s)
|
|
117
|
-
return f"of:={s}"
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
def try_float(v: Any) -> Any:
|
|
121
|
-
"""
|
|
122
|
-
Try to cast given value to float.
|
|
123
|
-
Return input if not possible.
|
|
124
|
-
|
|
125
|
-
Attributes:
|
|
126
|
-
v (Any): Value tried to be cast to float.
|
|
127
|
-
"""
|
|
128
|
-
try:
|
|
129
|
-
return float(v)
|
|
130
|
-
except (ValueError, TypeError):
|
|
131
|
-
return v
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
def get_cell_index(col: int, row: int, abs_col: bool = False, abs_row: bool = False) -> str:
|
|
135
|
-
"""
|
|
136
|
-
Calculate ODS cell index.
|
|
137
|
-
|
|
138
|
-
Attributes:
|
|
139
|
-
col (int): Column index.
|
|
140
|
-
row (int): Row index.
|
|
141
|
-
abs_col (bool): Set '$' for column.
|
|
142
|
-
abs_row (bool): Set '$' for row.
|
|
143
|
-
"""
|
|
144
|
-
radix = ord("Z") - ord("A") + 1
|
|
145
|
-
ret = ""
|
|
146
|
-
while col >= 0:
|
|
147
|
-
rem = col % radix
|
|
148
|
-
ret = chr(rem + ord("A")) + ret
|
|
149
|
-
col = col // radix - 1
|
|
150
|
-
if abs_col:
|
|
151
|
-
pre_col = "$"
|
|
152
|
-
else:
|
|
153
|
-
pre_col = ""
|
|
154
|
-
if abs_row:
|
|
155
|
-
pre_row = "$"
|
|
156
|
-
else:
|
|
157
|
-
pre_row = ""
|
|
158
|
-
return pre_col + ret + pre_row + str(row + 1)
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
class ODSDoc:
|
|
162
|
-
"""
|
|
163
|
-
Class representing ODS document.
|
|
164
|
-
"""
|
|
165
|
-
|
|
166
|
-
def __init__(self, benchmark: "result.BenchmarkMerge", measures: list[tuple[str, Any]]):
|
|
167
|
-
"""
|
|
168
|
-
Setup Instance and Class sheet.
|
|
169
|
-
|
|
170
|
-
Attributes:
|
|
171
|
-
benchmark (BenchmarkMerge): BenchmarkMerge object.
|
|
172
|
-
measures (list[tuple[str, Any]]): Measures to be displayed.
|
|
173
|
-
"""
|
|
174
|
-
self.inst_sheet = Sheet(benchmark, measures, "Instances")
|
|
175
|
-
self.class_sheet = Sheet(benchmark, measures, "Classes", self.inst_sheet)
|
|
176
|
-
|
|
177
|
-
def add_runspec(self, runspec: "result.Runspec") -> None:
|
|
178
|
-
"""
|
|
179
|
-
Attributes:
|
|
180
|
-
runspec (Runspec): Run specification.
|
|
181
|
-
"""
|
|
182
|
-
self.inst_sheet.add_runspec(runspec)
|
|
183
|
-
self.class_sheet.add_runspec(runspec)
|
|
184
|
-
|
|
185
|
-
def finish(self) -> None:
|
|
186
|
-
"""
|
|
187
|
-
Complete sheets by adding formulas and summaries.
|
|
188
|
-
"""
|
|
189
|
-
self.inst_sheet.finish()
|
|
190
|
-
self.class_sheet.finish()
|
|
191
|
-
|
|
192
|
-
def make_ods(self, out: str) -> None:
|
|
193
|
-
"""
|
|
194
|
-
Write ODS file.
|
|
195
|
-
|
|
196
|
-
Attributes:
|
|
197
|
-
out (str): Name of the generated ODS file.
|
|
198
|
-
"""
|
|
199
|
-
|
|
200
|
-
with mock.patch.object(ods.Sheet, "writerow", write_row):
|
|
201
|
-
with ods.writer(open(out, "wb")) as odsfile:
|
|
202
|
-
inst_sheet = odsfile.new_sheet("Instances")
|
|
203
|
-
for line in range(len(self.inst_sheet.content.index)):
|
|
204
|
-
inst_sheet.writerow(list(self.inst_sheet.content.iloc[line]))
|
|
205
|
-
class_sheet = odsfile.new_sheet("Classes")
|
|
206
|
-
for line in range(len(self.class_sheet.content.index)):
|
|
207
|
-
class_sheet.writerow(list(self.class_sheet.content.iloc[line]))
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
# pylint: disable=too-many-instance-attributes
|
|
211
|
-
class Sheet:
|
|
212
|
-
"""
|
|
213
|
-
Class representing an ODS sheet.
|
|
214
|
-
"""
|
|
215
|
-
|
|
216
|
-
def __init__(
|
|
217
|
-
self,
|
|
218
|
-
benchmark: "result.BenchmarkMerge",
|
|
219
|
-
measures: list[tuple[str, Any]],
|
|
220
|
-
name: str,
|
|
221
|
-
ref_sheet: Optional["Sheet"] = None,
|
|
222
|
-
):
|
|
223
|
-
"""
|
|
224
|
-
Initialize sheet.
|
|
225
|
-
|
|
226
|
-
Attributes:
|
|
227
|
-
benchmark (BenchmarkMerge): Benchmark.
|
|
228
|
-
measures (list[tuple[str, Any]]): Measures to be displayed.
|
|
229
|
-
name (str): Name of the sheet.
|
|
230
|
-
refSheet (Optional[Sheet]): Reference sheet.
|
|
231
|
-
"""
|
|
232
|
-
# dataframe resembling almost final ods form
|
|
233
|
-
self.content = pd.DataFrame()
|
|
234
|
-
# name of the sheet
|
|
235
|
-
self.name = name
|
|
236
|
-
# evaluated benchmarks
|
|
237
|
-
self.benchmark = benchmark
|
|
238
|
-
# dataframes containing result data, use these for calculations
|
|
239
|
-
self.system_blocks: dict[tuple[Any, Any], SystemBlock] = {}
|
|
240
|
-
# types of measures
|
|
241
|
-
self.types: dict[str, str] = {}
|
|
242
|
-
# measures to be displayed
|
|
243
|
-
self.measures = measures
|
|
244
|
-
# machines
|
|
245
|
-
self.machines: set["result.Machine"] = set()
|
|
246
|
-
# sheet for references
|
|
247
|
-
self.ref_sheet = ref_sheet
|
|
248
|
-
# references for summary generation
|
|
249
|
-
self.summary_refs: dict[str, Any] = {}
|
|
250
|
-
# dataframe containing all results and stats
|
|
251
|
-
self.values = pd.DataFrame()
|
|
252
|
-
# columns containing floats
|
|
253
|
-
self.float_occur: dict[str, set[Any]] = {}
|
|
254
|
-
|
|
255
|
-
# first column
|
|
256
|
-
self.content[0] = None
|
|
257
|
-
# setup rows for instances/benchmark classes
|
|
258
|
-
if self.ref_sheet is None:
|
|
259
|
-
row = 2
|
|
260
|
-
for benchclass in benchmark:
|
|
261
|
-
for instance in benchclass:
|
|
262
|
-
self.content.loc[row] = instance.benchclass.name + "/" + instance.name
|
|
263
|
-
row += instance.values["max_runs"]
|
|
264
|
-
else:
|
|
265
|
-
row = 2
|
|
266
|
-
for benchclass in benchmark:
|
|
267
|
-
self.content.loc[row] = benchclass.name
|
|
268
|
-
row += 1
|
|
269
|
-
|
|
270
|
-
self.result_offset = row
|
|
271
|
-
self.content.loc[self.result_offset + 1] = "SUM"
|
|
272
|
-
self.content.loc[self.result_offset + 2] = "AVG"
|
|
273
|
-
self.content.loc[self.result_offset + 3] = "DEV"
|
|
274
|
-
self.content.loc[self.result_offset + 4] = "DST"
|
|
275
|
-
self.content.loc[self.result_offset + 5] = "BEST"
|
|
276
|
-
self.content.loc[self.result_offset + 6] = "BETTER"
|
|
277
|
-
self.content.loc[self.result_offset + 7] = "WORSE"
|
|
278
|
-
self.content.loc[self.result_offset + 8] = "WORST"
|
|
279
|
-
# fill missing rows
|
|
280
|
-
self.content = self.content.reindex(list(range(self.content.index.max() + 1)))
|
|
281
|
-
|
|
282
|
-
def add_runspec(self, runspec: "result.Runspec") -> None:
|
|
283
|
-
"""
|
|
284
|
-
Add results to the their respective blocks.
|
|
285
|
-
|
|
286
|
-
Attributes:
|
|
287
|
-
runspec (Runspec): Run specification
|
|
288
|
-
"""
|
|
289
|
-
key = (runspec.setting, runspec.machine)
|
|
290
|
-
if key not in self.system_blocks:
|
|
291
|
-
self.system_blocks[key] = SystemBlock(runspec.setting, runspec.machine)
|
|
292
|
-
block = self.system_blocks[key]
|
|
293
|
-
if block.machine:
|
|
294
|
-
self.machines.add(block.machine)
|
|
295
|
-
|
|
296
|
-
for benchclass_result in runspec:
|
|
297
|
-
benchclass_summary: dict[str, Any] = {}
|
|
298
|
-
for instance_result in benchclass_result:
|
|
299
|
-
self.add_instance_results(block, instance_result, benchclass_summary)
|
|
300
|
-
for m in block.columns:
|
|
301
|
-
if m not in self.types or self.types[m] in ["None", "empty"]:
|
|
302
|
-
self.types[m] = block.columns[m]
|
|
303
|
-
# mixed measure
|
|
304
|
-
elif block.columns[m] not in [self.types[m], "None", "empty"]:
|
|
305
|
-
self.types[m] = "string"
|
|
306
|
-
# classSheet
|
|
307
|
-
if self.ref_sheet:
|
|
308
|
-
self.add_benchclass_summary(block, benchclass_result, benchclass_summary)
|
|
309
|
-
|
|
310
|
-
def add_instance_results(
|
|
311
|
-
self, block: "SystemBlock", instance_result: "result.InstanceResult", benchclass_summary: dict[str, Any]
|
|
312
|
-
) -> None:
|
|
313
|
-
"""
|
|
314
|
-
Add instance results to SystemBlock and add values to summary if necessary.
|
|
315
|
-
|
|
316
|
-
Attributes:
|
|
317
|
-
block (SystemBlock): SystemBlock to which results are added.
|
|
318
|
-
instance_result (InstanceResult): InstanceResult.
|
|
319
|
-
benchclass_summary (dict[str, Any]): Summary of benchmark class.
|
|
320
|
-
"""
|
|
321
|
-
for run in instance_result:
|
|
322
|
-
for name, value_type, value in run.iter(self.measures):
|
|
323
|
-
if value_type == "int":
|
|
324
|
-
value_type = "float"
|
|
325
|
-
elif value_type != "float" and value_type != "None":
|
|
326
|
-
value_type = "string"
|
|
327
|
-
if self.ref_sheet is None:
|
|
328
|
-
if value_type == "float":
|
|
329
|
-
block.add_cell(
|
|
330
|
-
instance_result.instance.values["row"] + run.number - 1, name, value_type, float(value)
|
|
331
|
-
)
|
|
332
|
-
elif value_type == "None":
|
|
333
|
-
block.add_cell(
|
|
334
|
-
instance_result.instance.values["row"] + run.number - 1, name, value_type, np.nan
|
|
335
|
-
)
|
|
336
|
-
else:
|
|
337
|
-
block.add_cell(instance_result.instance.values["row"] + run.number - 1, name, value_type, value)
|
|
338
|
-
elif value_type == "float" and self.ref_sheet.types.get(name, "") == "float":
|
|
339
|
-
if benchclass_summary.get(name, None) is None:
|
|
340
|
-
benchclass_summary[name] = (0.0, 0)
|
|
341
|
-
benchclass_summary[name] = (
|
|
342
|
-
float(value) + benchclass_summary[name][0],
|
|
343
|
-
1 + benchclass_summary[name][1],
|
|
344
|
-
)
|
|
345
|
-
else:
|
|
346
|
-
if not name in benchclass_summary:
|
|
347
|
-
benchclass_summary[name] = None
|
|
348
|
-
|
|
349
|
-
def add_benchclass_summary(
|
|
350
|
-
self, block: "SystemBlock", benchclass_result: "result.ClassResult", benchclass_summary: dict[str, Any]
|
|
351
|
-
) -> None:
|
|
352
|
-
"""
|
|
353
|
-
Add benchmark class summary to SystemBlock.
|
|
354
|
-
|
|
355
|
-
Attributes:
|
|
356
|
-
block (SystemBlock): SystemBlock to which summary is added.
|
|
357
|
-
benchclass_result (ClassResult): ClassResult.
|
|
358
|
-
benchclass_summary (dict[str, Any]): Summary of benchmark class.
|
|
359
|
-
"""
|
|
360
|
-
for name, value in benchclass_summary.items():
|
|
361
|
-
if value is not None:
|
|
362
|
-
temp_res = value[0] / value[1]
|
|
363
|
-
if name == "timeout":
|
|
364
|
-
temp_res = value[0]
|
|
365
|
-
block.add_cell(
|
|
366
|
-
benchclass_result.benchclass.values["row"],
|
|
367
|
-
name,
|
|
368
|
-
"classresult",
|
|
369
|
-
(
|
|
370
|
-
{
|
|
371
|
-
"inst_start": benchclass_result.benchclass.values["inst_start"],
|
|
372
|
-
"inst_end": benchclass_result.benchclass.values["inst_end"],
|
|
373
|
-
"value": temp_res,
|
|
374
|
-
}
|
|
375
|
-
),
|
|
376
|
-
)
|
|
377
|
-
else:
|
|
378
|
-
block.add_cell(benchclass_result.benchclass.values["row"], name, "empty", np.nan)
|
|
379
|
-
|
|
380
|
-
def finish(self) -> None:
|
|
381
|
-
"""
|
|
382
|
-
Finish ODS content.
|
|
383
|
-
"""
|
|
384
|
-
col = 1
|
|
385
|
-
# join results of different blocks
|
|
386
|
-
for block in sorted(self.system_blocks.values()):
|
|
387
|
-
self.content = self.content.join(block.content)
|
|
388
|
-
self.content = self.content.set_axis(list(range(len(self.content.columns))), axis=1)
|
|
389
|
-
self.content.at[0, col] = block.gen_name(len(self.machines) > 1)
|
|
390
|
-
col += len(block.columns)
|
|
391
|
-
|
|
392
|
-
# get columns used for summary calculations
|
|
393
|
-
# add formulas for results of classSheet
|
|
394
|
-
for column in self.content:
|
|
395
|
-
name = self.content.at[1, column]
|
|
396
|
-
if self.types.get(name, "") == "classresult":
|
|
397
|
-
for row in range(2, self.result_offset):
|
|
398
|
-
op = "AVERAGE"
|
|
399
|
-
if name == "timeout":
|
|
400
|
-
op = "SUM"
|
|
401
|
-
|
|
402
|
-
# avoid missing measures
|
|
403
|
-
if isinstance(self.content.at[row, column], dict):
|
|
404
|
-
self.values.at[row, column] = self.content.at[row, column]["value"]
|
|
405
|
-
self.content.at[row, column] = Formula(
|
|
406
|
-
""
|
|
407
|
-
+ op
|
|
408
|
-
+ "(Instances.{0}:Instances.{1})".format(
|
|
409
|
-
get_cell_index(column, self.content.at[row, column]["inst_start"] + 2),
|
|
410
|
-
get_cell_index(column, self.content.at[row, column]["inst_end"] + 2),
|
|
411
|
-
)
|
|
412
|
-
)
|
|
413
|
-
if self.types.get(name, "") in ["float", "classresult"]:
|
|
414
|
-
if not name in self.float_occur:
|
|
415
|
-
self.float_occur[name] = set()
|
|
416
|
-
self.float_occur[name].add(column)
|
|
417
|
-
# defragmentation (temporary workaround)
|
|
418
|
-
self.content = self.content.copy()
|
|
419
|
-
self.values = self.values.copy()
|
|
420
|
-
|
|
421
|
-
if self.ref_sheet is not None:
|
|
422
|
-
self.values = self.values.reindex(index=self.content.index, columns=self.content.columns)
|
|
423
|
-
self.values = self.values.combine_first(self.content)
|
|
424
|
-
else:
|
|
425
|
-
self.values = (
|
|
426
|
-
self.content.iloc[2 : self.result_offset - 1, 1:].combine_first(self.values).combine_first(self.content)
|
|
427
|
-
)
|
|
428
|
-
|
|
429
|
-
# defragmentation (temporary workaround)
|
|
430
|
-
self.content = self.content.copy()
|
|
431
|
-
self.values = self.values.copy()
|
|
432
|
-
|
|
433
|
-
# add summaries
|
|
434
|
-
self.add_row_summary(col)
|
|
435
|
-
self.add_col_summary()
|
|
436
|
-
|
|
437
|
-
# color cells
|
|
438
|
-
self.add_styles()
|
|
439
|
-
|
|
440
|
-
# replace all undefined cells with None (empty cell)
|
|
441
|
-
self.content = self.content.fillna(np.nan).replace([np.nan], [None])
|
|
442
|
-
|
|
443
|
-
def add_row_summary(self, offset: int) -> None:
|
|
444
|
-
"""
|
|
445
|
-
Add row summary (min, max, median).
|
|
446
|
-
|
|
447
|
-
Attributes:
|
|
448
|
-
offset (int): Column offset.
|
|
449
|
-
"""
|
|
450
|
-
col = offset
|
|
451
|
-
for col_name in ["min", "median", "max"]:
|
|
452
|
-
block = SystemBlock(None, None)
|
|
453
|
-
block.offset = col
|
|
454
|
-
self.summary_refs[col_name] = {"col": col}
|
|
455
|
-
measures: list[str]
|
|
456
|
-
if len(self.measures) == 0:
|
|
457
|
-
measures = sorted(self.float_occur.keys())
|
|
458
|
-
else:
|
|
459
|
-
measures = list(map(lambda x: x[0], self.measures))
|
|
460
|
-
for measure in measures:
|
|
461
|
-
if measure in self.float_occur:
|
|
462
|
-
self.values.at[1, col] = measure
|
|
463
|
-
self._add_summary_formula(block, col_name, measure, self.float_occur, col)
|
|
464
|
-
self.summary_refs[col_name][measure] = (
|
|
465
|
-
col,
|
|
466
|
-
"{0}:{1}".format(
|
|
467
|
-
get_cell_index(col, 2, True), get_cell_index(col, self.result_offset - 1, True)
|
|
468
|
-
),
|
|
469
|
-
)
|
|
470
|
-
col += 1
|
|
471
|
-
self.content = self.content.join(block.content)
|
|
472
|
-
self.content = self.content.set_axis(list(range(len(self.content.columns))), axis=1)
|
|
473
|
-
self.content.at[0, block.offset] = col_name
|
|
474
|
-
self.values.at[0, block.offset] = col_name
|
|
475
|
-
|
|
476
|
-
def _add_summary_formula(
|
|
477
|
-
self, block: "SystemBlock", operator: str, measure: str, float_occur: dict[str, Any], col: int
|
|
478
|
-
) -> None:
|
|
479
|
-
"""
|
|
480
|
-
Add row summary formula.
|
|
481
|
-
|
|
482
|
-
Attributes:
|
|
483
|
-
block (SystemBlock): SystemBlock to which summary is added.
|
|
484
|
-
operator (str): Summary operator.
|
|
485
|
-
measure (str): Name of the measure to be summarized.
|
|
486
|
-
float_occur (dict[str, Any]): Dict containing column references of float columns.
|
|
487
|
-
col (int): Current column index.
|
|
488
|
-
"""
|
|
489
|
-
for row in range(self.result_offset - 2):
|
|
490
|
-
ref_range = ""
|
|
491
|
-
for col_ref in sorted(float_occur[measure]):
|
|
492
|
-
if ref_range != "":
|
|
493
|
-
ref_range += ";"
|
|
494
|
-
ref_range += get_cell_index(col_ref, row + 2, True)
|
|
495
|
-
values = np.array(self.values.loc[2 + row, sorted(float_occur[measure])], float)
|
|
496
|
-
if np.isnan(values).all():
|
|
497
|
-
self.values.at[2 + row, col] = np.nan
|
|
498
|
-
else:
|
|
499
|
-
# don't write formula if full row is nan
|
|
500
|
-
block.add_cell(row, measure, "formular", Formula("{1}({0})".format(ref_range, operator.upper())))
|
|
501
|
-
self.values.at[2 + row, col] = getattr(np, "nan" + operator)(values)
|
|
502
|
-
|
|
503
|
-
def add_col_summary(self) -> None:
|
|
504
|
-
"""
|
|
505
|
-
Add column summary if applicable to column type.
|
|
506
|
-
"""
|
|
507
|
-
for col in self.content:
|
|
508
|
-
name = self.content.at[1, col]
|
|
509
|
-
if self.types.get(name, "") in ["float", "classresult"]:
|
|
510
|
-
ref_value = "{0}:{1}".format(
|
|
511
|
-
get_cell_index(col, 2, True), get_cell_index(col, self.result_offset - 1, True)
|
|
512
|
-
)
|
|
513
|
-
values = np.array(self.values.loc[2 : self.result_offset - 1, col], dtype=float)
|
|
514
|
-
if np.isnan(values).all():
|
|
515
|
-
continue
|
|
516
|
-
# SUM
|
|
517
|
-
self.content.at[self.result_offset + 1, col] = Formula("SUM({0})".format(ref_value))
|
|
518
|
-
self.values.at[self.result_offset + 1, col] = np.nansum(values)
|
|
519
|
-
# AVG
|
|
520
|
-
self.content.at[self.result_offset + 2, col] = Formula("AVERAGE({0})".format(ref_value))
|
|
521
|
-
self.values.at[self.result_offset + 2, col] = np.nanmean(values)
|
|
522
|
-
# DEV
|
|
523
|
-
self.content.at[self.result_offset + 3, col] = Formula("STDEV({0})".format(ref_value))
|
|
524
|
-
# catch warnings caused by missing values (nan)
|
|
525
|
-
with warnings.catch_warnings():
|
|
526
|
-
warnings.filterwarnings("ignore", "Degrees of freedom <= 0 for slice", RuntimeWarning)
|
|
527
|
-
if len(values) != 1:
|
|
528
|
-
self.values.at[self.result_offset + 3, col] = np.nanstd(values, ddof=1)
|
|
529
|
-
else: # only one value
|
|
530
|
-
self.values.at[self.result_offset + 3, col] = np.nan # nocoverage
|
|
531
|
-
if col < self.summary_refs["min"]["col"]:
|
|
532
|
-
with np.errstate(invalid="ignore"):
|
|
533
|
-
# DST
|
|
534
|
-
self.content.at[self.result_offset + 4, col] = Formula(
|
|
535
|
-
"SUMPRODUCT(--({0}-{1})^2)^0.5".format(ref_value, self.summary_refs["min"][name][1])
|
|
536
|
-
)
|
|
537
|
-
self.values.at[self.result_offset + 4, col] = (
|
|
538
|
-
np.nansum(
|
|
539
|
-
(
|
|
540
|
-
values
|
|
541
|
-
- np.array(
|
|
542
|
-
self.values.loc[2 : self.result_offset - 1, self.summary_refs["min"][name][0]]
|
|
543
|
-
)
|
|
544
|
-
)
|
|
545
|
-
** 2
|
|
546
|
-
)
|
|
547
|
-
** 0.5
|
|
548
|
-
)
|
|
549
|
-
# BEST (values * -1, since higher better)
|
|
550
|
-
self.content.at[self.result_offset + 5, col] = Formula(
|
|
551
|
-
"SUMPRODUCT(--({0}={1}))".format(ref_value, self.summary_refs["min"][name][1])
|
|
552
|
-
)
|
|
553
|
-
self.values.at[self.result_offset + 5, col] = -1 * np.nansum(
|
|
554
|
-
values
|
|
555
|
-
== np.array(self.values.loc[2 : self.result_offset - 1, self.summary_refs["min"][name][0]])
|
|
556
|
-
)
|
|
557
|
-
# BETTER (values * -1, since higher better)
|
|
558
|
-
self.content.at[self.result_offset + 6, col] = Formula(
|
|
559
|
-
"SUMPRODUCT(--({0}<{1}))".format(ref_value, self.summary_refs["median"][name][1])
|
|
560
|
-
)
|
|
561
|
-
self.values.at[self.result_offset + 6, col] = -1 * np.nansum(
|
|
562
|
-
values
|
|
563
|
-
< np.array(
|
|
564
|
-
self.values.loc[2 : self.result_offset - 1, self.summary_refs["median"][name][0]]
|
|
565
|
-
)
|
|
566
|
-
)
|
|
567
|
-
# WORSE
|
|
568
|
-
self.content.at[self.result_offset + 7, col] = Formula(
|
|
569
|
-
"SUMPRODUCT(--({0}>{1}))".format(ref_value, self.summary_refs["median"][name][1])
|
|
570
|
-
)
|
|
571
|
-
self.values.at[self.result_offset + 7, col] = np.nansum(
|
|
572
|
-
values
|
|
573
|
-
> np.array(
|
|
574
|
-
self.values.loc[2 : self.result_offset - 1, self.summary_refs["median"][name][0]]
|
|
575
|
-
)
|
|
576
|
-
)
|
|
577
|
-
# WORST
|
|
578
|
-
self.content.at[self.result_offset + 8, col] = Formula(
|
|
579
|
-
"SUMPRODUCT(--({0}={1}))".format(ref_value, self.summary_refs["max"][name][1])
|
|
580
|
-
)
|
|
581
|
-
self.values.at[self.result_offset + 8, col] = np.nansum(
|
|
582
|
-
values
|
|
583
|
-
== np.array(self.values.loc[2 : self.result_offset - 1, self.summary_refs["max"][name][0]])
|
|
584
|
-
)
|
|
585
|
-
|
|
586
|
-
def add_styles(self) -> None:
|
|
587
|
-
"""
|
|
588
|
-
Color float results and their summaries.
|
|
589
|
-
"""
|
|
590
|
-
# remove header
|
|
591
|
-
results = self.values.loc[2:, 1:]
|
|
592
|
-
|
|
593
|
-
for measure in self.measures:
|
|
594
|
-
if measure[0] in self.float_occur:
|
|
595
|
-
func = measure[1]
|
|
596
|
-
if func == "t":
|
|
597
|
-
diff = 2
|
|
598
|
-
elif func == "to":
|
|
599
|
-
diff = 0
|
|
600
|
-
else:
|
|
601
|
-
return
|
|
602
|
-
|
|
603
|
-
cols = sorted(self.float_occur[measure[0]])
|
|
604
|
-
# filter empty rows
|
|
605
|
-
values_df = results.loc[:, cols].dropna(how="all")
|
|
606
|
-
rows = values_df.index
|
|
607
|
-
|
|
608
|
-
values = np.array(values_df.values, dtype=float)
|
|
609
|
-
min_values = np.reshape(np.nanmin(values, axis=1), (-1, 1))
|
|
610
|
-
median_values = np.reshape(np.nanmedian(values, axis=1), (-1, 1))
|
|
611
|
-
max_values = np.reshape(np.nanmax(values, axis=1), (-1, 1))
|
|
612
|
-
max_min_diff = (max_values - min_values) > diff
|
|
613
|
-
max_med_diff = (max_values - median_values) > diff
|
|
614
|
-
|
|
615
|
-
self.content = (
|
|
616
|
-
self.content.loc[rows, cols]
|
|
617
|
-
.mask(
|
|
618
|
-
(values == min_values) & (values < median_values) & max_min_diff,
|
|
619
|
-
self.content.loc[rows].map(lambda x: (x, "cellBest")),
|
|
620
|
-
)
|
|
621
|
-
.combine_first(self.content)
|
|
622
|
-
)
|
|
623
|
-
self.content = (
|
|
624
|
-
self.content.loc[rows, cols]
|
|
625
|
-
.mask(
|
|
626
|
-
(values == max_values) & (values > median_values) & max_med_diff,
|
|
627
|
-
self.content.loc[rows].map(lambda x: (x, "cellWorst")),
|
|
628
|
-
)
|
|
629
|
-
.combine_first(self.content)
|
|
630
|
-
)
|
|
631
|
-
|
|
632
|
-
def export_values(self, file_name: str, metadata: dict[str, list[Any]]) -> None:
|
|
633
|
-
"""
|
|
634
|
-
Export values to parquet file.
|
|
635
|
-
|
|
636
|
-
Attributes:
|
|
637
|
-
file_name (str): Name of the parquet file.
|
|
638
|
-
"""
|
|
639
|
-
# currently only inst sheet exported
|
|
640
|
-
if self.ref_sheet is not None:
|
|
641
|
-
return
|
|
642
|
-
# fill settings
|
|
643
|
-
self.values.iloc[0, :] = self.values.iloc[0, :].ffill()
|
|
644
|
-
# group values by measure
|
|
645
|
-
df = self.values.iloc[2:, [0]].reset_index(drop=True).astype("string")
|
|
646
|
-
df.columns = pd.MultiIndex.from_tuples([("", "instance")], names=["measure", "setting"])
|
|
647
|
-
for m, cols in self.float_occur.items():
|
|
648
|
-
nf = self.values.iloc[2:, sorted(cols)].reset_index(drop=True).astype("float64")
|
|
649
|
-
nf.columns = self.values.iloc[0, sorted(cols)].to_list()
|
|
650
|
-
nf.columns = pd.MultiIndex.from_product([[m], nf.columns], names=["measure", "setting"])
|
|
651
|
-
df = df.join(nf)
|
|
652
|
-
# metadata
|
|
653
|
-
# offset -2 (header) -1 (empty row)
|
|
654
|
-
metadict = {**{"offset": [self.result_offset - 3]}, **metadata}
|
|
655
|
-
metadf = pd.DataFrame({k: pd.Series(v) for k, v in metadict.items()})
|
|
656
|
-
metadf.columns = pd.MultiIndex.from_product([["_metadata"], metadf.columns], names=["measure", "setting"])
|
|
657
|
-
self.values = df.join(metadf)
|
|
658
|
-
#! min,med,max no longer included
|
|
659
|
-
self.values.astype(str).to_parquet(file_name)
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
@dataclass(order=True, unsafe_hash=True)
|
|
663
|
-
class SystemBlock:
|
|
664
|
-
"""
|
|
665
|
-
Dataframe containing results for system.
|
|
666
|
-
|
|
667
|
-
Attributes:
|
|
668
|
-
setting (Optional[Setting]): Benchmark setting.
|
|
669
|
-
machine (Optional[Machine]): Machine.
|
|
670
|
-
content (DataFrame): Results.
|
|
671
|
-
columns (dict[str, Any]): Dictionary of columns and their types.
|
|
672
|
-
offset (Optional[int]): Offset for final block position.
|
|
673
|
-
"""
|
|
674
|
-
|
|
675
|
-
setting: Optional["result.Setting"]
|
|
676
|
-
machine: Optional["result.Machine"]
|
|
677
|
-
content: pd.DataFrame = field(default_factory=pd.DataFrame, compare=False)
|
|
678
|
-
columns: dict[str, Any] = field(default_factory=dict, compare=False)
|
|
679
|
-
offset: Optional[int] = field(default=None, compare=False)
|
|
680
|
-
|
|
681
|
-
def gen_name(self, add_machine: bool) -> str:
|
|
682
|
-
"""
|
|
683
|
-
Generate name of the block.
|
|
684
|
-
|
|
685
|
-
Attributes:
|
|
686
|
-
addMachine (bool): Whether to include the machine name in the name.
|
|
687
|
-
"""
|
|
688
|
-
res: str = ""
|
|
689
|
-
if self.setting:
|
|
690
|
-
res = self.setting.system.name + "-" + self.setting.system.version + "/" + self.setting.name
|
|
691
|
-
if add_machine and self.machine:
|
|
692
|
-
res += " ({0})".format(self.machine.name)
|
|
693
|
-
return res
|
|
694
|
-
|
|
695
|
-
def add_cell(self, row: int, name: str, value_type: str, value: Any) -> None:
|
|
696
|
-
"""
|
|
697
|
-
Add cell to dataframe.
|
|
698
|
-
|
|
699
|
-
Attributes:
|
|
700
|
-
row (int): Row of the new cell.
|
|
701
|
-
name (str): Name of the column of the new cell (in most cases the measure).
|
|
702
|
-
valueType (str): Data type of the new cell.
|
|
703
|
-
value (Any): Value of the new cell.
|
|
704
|
-
"""
|
|
705
|
-
if name not in self.columns:
|
|
706
|
-
self.content.at[1, name] = name
|
|
707
|
-
self.columns[name] = value_type
|
|
708
|
-
# mixed system column
|
|
709
|
-
elif value_type not in [self.columns[name], "None"]:
|
|
710
|
-
self.columns[name] = "string"
|
|
711
|
-
# leave space for header and add new row if necessary
|
|
712
|
-
if row + 2 not in self.content.index:
|
|
713
|
-
self.content = self.content.reindex(self.content.index.tolist() + [row + 2])
|
|
714
|
-
self.content.at[row + 2, name] = value
|