holobench 1.3.5__py3-none-any.whl → 1.23.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bencher/__init__.py +12 -0
- bencher/bench_cfg.py +29 -33
- bencher/bench_plot_server.py +5 -1
- bencher/bench_report.py +14 -14
- bencher/bench_runner.py +3 -2
- bencher/bencher.py +80 -53
- bencher/class_enum.py +52 -0
- bencher/job.py +6 -4
- bencher/optuna_conversions.py +4 -3
- bencher/utils.py +55 -4
- bencher/video_writer.py +48 -11
- holobench-1.23.0.data/data/share/bencher/package.xml +33 -0
- holobench-1.23.0.dist-info/LICENSE +21 -0
- {holobench-1.3.5.dist-info → holobench-1.23.0.dist-info}/METADATA +41 -31
- holobench-1.23.0.dist-info/RECORD +20 -0
- {holobench-1.3.5.dist-info → holobench-1.23.0.dist-info}/WHEEL +2 -1
- holobench-1.23.0.dist-info/top_level.txt +1 -0
- bencher/example/benchmark_data.py +0 -200
- bencher/example/example_all.py +0 -45
- bencher/example/example_categorical.py +0 -99
- bencher/example/example_custom_sweep.py +0 -59
- bencher/example/example_docs.py +0 -34
- bencher/example/example_float3D.py +0 -101
- bencher/example/example_float_cat.py +0 -98
- bencher/example/example_floats.py +0 -89
- bencher/example/example_floats2D.py +0 -93
- bencher/example/example_holosweep.py +0 -104
- bencher/example/example_holosweep_objects.py +0 -111
- bencher/example/example_holosweep_tap.py +0 -144
- bencher/example/example_image.py +0 -82
- bencher/example/example_levels.py +0 -181
- bencher/example/example_pareto.py +0 -53
- bencher/example/example_sample_cache.py +0 -85
- bencher/example/example_sample_cache_context.py +0 -116
- bencher/example/example_simple.py +0 -134
- bencher/example/example_simple_bool.py +0 -34
- bencher/example/example_simple_cat.py +0 -47
- bencher/example/example_simple_float.py +0 -38
- bencher/example/example_strings.py +0 -46
- bencher/example/example_time_event.py +0 -62
- bencher/example/example_video.py +0 -124
- bencher/example/example_workflow.py +0 -189
- bencher/example/experimental/example_bokeh_plotly.py +0 -38
- bencher/example/experimental/example_hover_ex.py +0 -45
- bencher/example/experimental/example_hvplot_explorer.py +0 -39
- bencher/example/experimental/example_interactive.py +0 -75
- bencher/example/experimental/example_streamnd.py +0 -49
- bencher/example/experimental/example_streams.py +0 -36
- bencher/example/experimental/example_template.py +0 -40
- bencher/example/experimental/example_updates.py +0 -84
- bencher/example/experimental/example_vector.py +0 -84
- bencher/example/meta/example_meta.py +0 -171
- bencher/example/meta/example_meta_cat.py +0 -25
- bencher/example/meta/example_meta_float.py +0 -23
- bencher/example/meta/example_meta_levels.py +0 -26
- bencher/example/optuna/example_optuna.py +0 -78
- bencher/example/shelved/example_float2D_scatter.py +0 -109
- bencher/example/shelved/example_float3D_cone.py +0 -96
- bencher/example/shelved/example_kwargs.py +0 -63
- bencher/plotting/__init__.py +0 -0
- bencher/plotting/plot_filter.py +0 -110
- bencher/plotting/plt_cnt_cfg.py +0 -74
- bencher/results/__init__.py +0 -0
- bencher/results/bench_result.py +0 -80
- bencher/results/bench_result_base.py +0 -405
- bencher/results/float_formatter.py +0 -44
- bencher/results/holoview_result.py +0 -592
- bencher/results/optuna_result.py +0 -354
- bencher/results/panel_result.py +0 -113
- bencher/results/plotly_result.py +0 -65
- bencher/variables/inputs.py +0 -193
- bencher/variables/parametrised_sweep.py +0 -206
- bencher/variables/results.py +0 -176
- bencher/variables/sweep_base.py +0 -167
- bencher/variables/time.py +0 -74
- holobench-1.3.5.dist-info/RECORD +0 -74
- /bencher/example/__init__.py → /holobench-1.23.0.data/data/share/ament_index/resource_index/packages/bencher +0 -0
bencher/plotting/plot_filter.py
DELETED
@@ -1,110 +0,0 @@
|
|
1
|
-
from __future__ import annotations
|
2
|
-
from typing import Optional
|
3
|
-
from dataclasses import dataclass
|
4
|
-
from bencher.plotting.plt_cnt_cfg import PltCntCfg
|
5
|
-
import panel as pn
|
6
|
-
|
7
|
-
|
8
|
-
class VarRange:
|
9
|
-
"""A VarRange represents the bounded and unbounded ranges of integers. This class is used to define filters for various variable types. For example by defining cat_var = VarRange(0,0), calling matches(0) will return true, but any other integer will not match. You can also have unbounded ranges for example VarRange(2,None) will match to 2,3,4... up to infinity. for By default the lower and upper bounds are set to -1 so so that no matter what value is passsed to matches() will return false. Matches only takes 0 and positive integers."""
|
10
|
-
|
11
|
-
def __init__(self, lower_bound: int = 0, upper_bound: int = -1) -> None:
|
12
|
-
"""
|
13
|
-
Args:
|
14
|
-
lower_bound (int, optional): The smallest acceptable value to matches(). Passing None will result in a lower bound of 0 (as matches only accepts positive integers). Defaults to 0.
|
15
|
-
upper_bound (int, optional): The largest acceptable value to matches(). Passing None will result in no upper bound. Defaults to -1 which results in a range with no matches.
|
16
|
-
"""
|
17
|
-
self.lower_bound = lower_bound
|
18
|
-
self.upper_bound = upper_bound
|
19
|
-
|
20
|
-
def matches(self, val: int) -> bool:
|
21
|
-
"""Checks that a value is within the variable range. lower_bound and upper_bound are inclusive (lower_bound<=val<=upper_bound )
|
22
|
-
|
23
|
-
Args:
|
24
|
-
val (int): A positive integer representing a number of items
|
25
|
-
|
26
|
-
Returns:
|
27
|
-
bool: True if the items is within the range, False otherwise.
|
28
|
-
|
29
|
-
Raises:
|
30
|
-
ValueError: If val < 0
|
31
|
-
"""
|
32
|
-
if val < 0:
|
33
|
-
raise ValueError("val must be >= 0")
|
34
|
-
if self.lower_bound is not None:
|
35
|
-
lower_match = val >= self.lower_bound
|
36
|
-
else:
|
37
|
-
lower_match = True
|
38
|
-
|
39
|
-
if self.upper_bound is not None:
|
40
|
-
upper_match = val <= self.upper_bound
|
41
|
-
else:
|
42
|
-
upper_match = True
|
43
|
-
|
44
|
-
return lower_match and upper_match
|
45
|
-
|
46
|
-
def matches_info(self, val, name):
|
47
|
-
match = self.matches(val)
|
48
|
-
info = f"{name}\t{match}\t{self.lower_bound}>= {val} <={self.upper_bound}"
|
49
|
-
return match, info
|
50
|
-
|
51
|
-
def __str__(self) -> str:
|
52
|
-
return f"VarRange(lower_bound={self.lower_bound}, upper_bound={self.upper_bound})"
|
53
|
-
|
54
|
-
|
55
|
-
@dataclass
|
56
|
-
class PlotFilter:
|
57
|
-
"""A class for representing the types of results a plot is able to represent."""
|
58
|
-
|
59
|
-
float_range: VarRange = VarRange()
|
60
|
-
cat_range: VarRange = VarRange()
|
61
|
-
vector_len: VarRange = VarRange(1, 1)
|
62
|
-
result_vars: VarRange = VarRange(1, 1)
|
63
|
-
panel_range: VarRange = VarRange(0, 0)
|
64
|
-
repeats_range: VarRange = VarRange(1, None)
|
65
|
-
input_range: VarRange = VarRange(1, None)
|
66
|
-
|
67
|
-
def matches_result(self, plt_cnt_cfg: PltCntCfg, plot_name: str) -> PlotMatchesResult:
|
68
|
-
"""Checks if the result data signature matches the type of data the plot is able to display."""
|
69
|
-
return PlotMatchesResult(self, plt_cnt_cfg, plot_name)
|
70
|
-
|
71
|
-
|
72
|
-
# @dataclass
|
73
|
-
class PlotMatchesResult:
|
74
|
-
"""Stores information about which properites match the requirements of a particular plotter"""
|
75
|
-
|
76
|
-
def __init__(self, plot_filter: PlotFilter, plt_cnt_cfg: PltCntCfg, plot_name: str):
|
77
|
-
match_info = []
|
78
|
-
matches = []
|
79
|
-
|
80
|
-
match_candidates = [
|
81
|
-
(plot_filter.float_range, plt_cnt_cfg.float_cnt, "float"),
|
82
|
-
(plot_filter.cat_range, plt_cnt_cfg.cat_cnt, "cat"),
|
83
|
-
(plot_filter.vector_len, plt_cnt_cfg.vector_len, "vec"),
|
84
|
-
(plot_filter.result_vars, plt_cnt_cfg.result_vars, "results"),
|
85
|
-
(plot_filter.panel_range, plt_cnt_cfg.panel_cnt, "panels"),
|
86
|
-
(plot_filter.repeats_range, plt_cnt_cfg.repeats, "repeats"),
|
87
|
-
(plot_filter.input_range, plt_cnt_cfg.inputs_cnt, "inputs"),
|
88
|
-
]
|
89
|
-
|
90
|
-
for m, cnt, name in match_candidates:
|
91
|
-
match, info = m.matches_info(cnt, name)
|
92
|
-
matches.append(match)
|
93
|
-
if not match:
|
94
|
-
match_info.append(info)
|
95
|
-
|
96
|
-
self.overall = all(matches)
|
97
|
-
|
98
|
-
match_info.insert(0, f"plot {plot_name} matches: {self.overall}")
|
99
|
-
self.matches_info = "\n".join(match_info).strip()
|
100
|
-
self.plt_cnt_cfg = plt_cnt_cfg
|
101
|
-
|
102
|
-
if self.plt_cnt_cfg.print_debug:
|
103
|
-
print(f"checking {plot_name} result: {self.overall}")
|
104
|
-
if not self.overall:
|
105
|
-
print(self.matches_info)
|
106
|
-
|
107
|
-
def to_panel(self, **kwargs) -> Optional[pn.pane.Markdown]:
|
108
|
-
if self.plt_cnt_cfg.print_debug:
|
109
|
-
return pn.pane.Markdown(self.matches_info, **kwargs)
|
110
|
-
return None
|
bencher/plotting/plt_cnt_cfg.py
DELETED
@@ -1,74 +0,0 @@
|
|
1
|
-
from __future__ import annotations
|
2
|
-
import param
|
3
|
-
from bencher.bench_cfg import BenchCfg
|
4
|
-
from bencher.variables.results import PANEL_TYPES
|
5
|
-
|
6
|
-
from bencher.variables.inputs import IntSweep, FloatSweep, BoolSweep, EnumSweep, StringSweep
|
7
|
-
from bencher.variables.time import TimeSnapshot
|
8
|
-
|
9
|
-
|
10
|
-
class PltCntCfg(param.Parameterized):
|
11
|
-
"""Plot Count Config"""
|
12
|
-
|
13
|
-
float_vars = param.List(doc="A list of float vars in order of plotting, x then y")
|
14
|
-
float_cnt = param.Integer(0, doc="The number of float variables to plot")
|
15
|
-
cat_vars = param.List(doc="A list of categorical values to plot in order hue,row,col")
|
16
|
-
cat_cnt = param.Integer(0, doc="The number of cat variables")
|
17
|
-
vector_len = param.Integer(1, doc="The vector length of the return variable , scalars = len 1")
|
18
|
-
result_vars = param.Integer(1, doc="The number result variables to plot") # todo remove
|
19
|
-
panel_vars = param.List(doc="A list of panel results")
|
20
|
-
panel_cnt = param.Integer(0, doc="Number of results reprented as panel panes")
|
21
|
-
repeats = param.Integer(0, doc="The number of repeat samples")
|
22
|
-
inputs_cnt = param.Integer(0, doc="The number of repeat samples")
|
23
|
-
|
24
|
-
print_debug = param.Boolean(
|
25
|
-
True, doc="Print debug information about why a filter matches this config or not"
|
26
|
-
)
|
27
|
-
|
28
|
-
@staticmethod
|
29
|
-
def generate_plt_cnt_cfg(
|
30
|
-
bench_cfg: BenchCfg,
|
31
|
-
) -> PltCntCfg:
|
32
|
-
"""Given a BenchCfg work out how many float and cat variables there are and store in a PltCntCfg class
|
33
|
-
|
34
|
-
Args:
|
35
|
-
bench_cfg (BenchCfg): See BenchCfg definition
|
36
|
-
|
37
|
-
Raises:
|
38
|
-
ValueError: If no plotting procedure could be automatically detected
|
39
|
-
|
40
|
-
Returns:
|
41
|
-
PltCntCfg: see PltCntCfg definition
|
42
|
-
"""
|
43
|
-
plt_cnt_cfg = PltCntCfg()
|
44
|
-
# plt_cnt_cfg.float_vars = deepcopy(bench_cfg.iv_time)
|
45
|
-
|
46
|
-
plt_cnt_cfg.cat_vars = []
|
47
|
-
plt_cnt_cfg.float_vars = []
|
48
|
-
|
49
|
-
for iv in bench_cfg.input_vars:
|
50
|
-
if isinstance(iv, (IntSweep, FloatSweep, TimeSnapshot)):
|
51
|
-
# if "IntSweep" in typestr or "FloatSweep" in typestr:
|
52
|
-
plt_cnt_cfg.float_vars.append(iv)
|
53
|
-
type_allocated = True
|
54
|
-
if isinstance(iv, (EnumSweep, BoolSweep, StringSweep)):
|
55
|
-
# if "EnumSweep" in typestr or "BoolSweep" in typestr or "StringSweep" in typestr:
|
56
|
-
plt_cnt_cfg.cat_vars.append(iv)
|
57
|
-
type_allocated = True
|
58
|
-
|
59
|
-
if not type_allocated:
|
60
|
-
raise ValueError(f"No rule for type {type(iv)}")
|
61
|
-
|
62
|
-
for rv in bench_cfg.result_vars:
|
63
|
-
if isinstance(rv, PANEL_TYPES):
|
64
|
-
plt_cnt_cfg.panel_vars.append(rv)
|
65
|
-
|
66
|
-
plt_cnt_cfg.float_cnt = len(plt_cnt_cfg.float_vars)
|
67
|
-
plt_cnt_cfg.cat_cnt = len(plt_cnt_cfg.cat_vars)
|
68
|
-
plt_cnt_cfg.panel_cnt = len(plt_cnt_cfg.panel_vars)
|
69
|
-
plt_cnt_cfg.repeats = bench_cfg.repeats
|
70
|
-
plt_cnt_cfg.inputs_cnt = len(bench_cfg.input_vars)
|
71
|
-
return plt_cnt_cfg
|
72
|
-
|
73
|
-
def __str__(self):
|
74
|
-
return f"float_cnt: {self.float_cnt}\ncat_cnt: {self.cat_cnt} \npanel_cnt: {self.panel_cnt}\nvector_len: {self.vector_len}"
|
bencher/results/__init__.py
DELETED
File without changes
|
bencher/results/bench_result.py
DELETED
@@ -1,80 +0,0 @@
|
|
1
|
-
from __future__ import annotations
|
2
|
-
from typing import List
|
3
|
-
import panel as pn
|
4
|
-
|
5
|
-
from bencher.results.panel_result import PanelResult
|
6
|
-
from bencher.results.plotly_result import PlotlyResult
|
7
|
-
from bencher.results.holoview_result import HoloviewResult
|
8
|
-
from bencher.results.bench_result_base import EmptyContainer
|
9
|
-
|
10
|
-
|
11
|
-
class BenchResult(PlotlyResult, HoloviewResult):
|
12
|
-
|
13
|
-
"""Contains the results of the benchmark and has methods to cast the results to various datatypes and graphical representations"""
|
14
|
-
|
15
|
-
def __init__(self, bench_cfg) -> None:
|
16
|
-
PlotlyResult.__init__(self, bench_cfg)
|
17
|
-
HoloviewResult.__init__(self, bench_cfg)
|
18
|
-
|
19
|
-
@staticmethod
|
20
|
-
def default_plot_callbacks():
|
21
|
-
return [
|
22
|
-
HoloviewResult.to_bar,
|
23
|
-
HoloviewResult.to_scatter_jitter,
|
24
|
-
HoloviewResult.to_curve,
|
25
|
-
HoloviewResult.to_line,
|
26
|
-
HoloviewResult.to_heatmap,
|
27
|
-
PlotlyResult.to_volume,
|
28
|
-
PanelResult.to_video,
|
29
|
-
PanelResult.to_panes,
|
30
|
-
]
|
31
|
-
|
32
|
-
@staticmethod
|
33
|
-
def plotly_callbacks():
|
34
|
-
return [HoloviewResult.to_surface, PlotlyResult.to_volume]
|
35
|
-
|
36
|
-
def to_auto(
|
37
|
-
self,
|
38
|
-
plot_list: List[callable] = None,
|
39
|
-
remove_plots: List[callable] = None,
|
40
|
-
**kwargs,
|
41
|
-
) -> List[pn.panel]:
|
42
|
-
self.plt_cnt_cfg.print_debug = False
|
43
|
-
|
44
|
-
if plot_list is None:
|
45
|
-
plot_list = BenchResult.default_plot_callbacks()
|
46
|
-
if remove_plots is not None:
|
47
|
-
for p in remove_plots:
|
48
|
-
plot_list.remove(p)
|
49
|
-
|
50
|
-
kwargs = self.set_plot_size(**kwargs)
|
51
|
-
|
52
|
-
row = EmptyContainer(pn.Row())
|
53
|
-
for plot_callback in plot_list:
|
54
|
-
if self.plt_cnt_cfg.print_debug:
|
55
|
-
print(f"checking: {plot_callback.__name__}")
|
56
|
-
# the callbacks are passed from the static class definition, so self needs to be passed before the plotting callback can be called
|
57
|
-
row.append(plot_callback(self, **kwargs))
|
58
|
-
|
59
|
-
self.plt_cnt_cfg.print_debug = True
|
60
|
-
if len(row.pane) == 0:
|
61
|
-
row.append(
|
62
|
-
pn.pane.Markdown("No Plotters are able to represent these results", **kwargs)
|
63
|
-
)
|
64
|
-
return row.pane
|
65
|
-
|
66
|
-
def to_auto_plots(self, **kwargs) -> List[pn.panel]:
|
67
|
-
"""Given the dataset result of a benchmark run, automatically dedeuce how to plot the data based on the types of variables that were sampled
|
68
|
-
|
69
|
-
Args:
|
70
|
-
bench_cfg (BenchCfg): Information on how the benchmark was sampled and the resulting data
|
71
|
-
|
72
|
-
Returns:
|
73
|
-
pn.pane: A panel containing plot results
|
74
|
-
"""
|
75
|
-
|
76
|
-
plot_cols = pn.Column()
|
77
|
-
plot_cols.append(self.to_sweep_summary(name="Plots View"))
|
78
|
-
plot_cols.append(self.to_auto(**kwargs))
|
79
|
-
plot_cols.append(self.bench_cfg.to_post_description())
|
80
|
-
return plot_cols
|
@@ -1,405 +0,0 @@
|
|
1
|
-
import logging
|
2
|
-
from typing import List, Any, Tuple, Optional
|
3
|
-
from enum import Enum, auto
|
4
|
-
import xarray as xr
|
5
|
-
from param import Parameter
|
6
|
-
import holoviews as hv
|
7
|
-
from functools import partial
|
8
|
-
from bencher.utils import int_to_col, color_tuple_to_css, callable_name
|
9
|
-
|
10
|
-
from bencher.variables.parametrised_sweep import ParametrizedSweep
|
11
|
-
from bencher.variables.results import OptDir
|
12
|
-
from copy import deepcopy
|
13
|
-
from bencher.results.optuna_result import OptunaResult
|
14
|
-
from bencher.variables.results import ResultVar
|
15
|
-
from bencher.results.float_formatter import FormatFloat
|
16
|
-
from bencher.plotting.plot_filter import VarRange, PlotFilter
|
17
|
-
import panel as pn
|
18
|
-
|
19
|
-
|
20
|
-
# todo add plugins
|
21
|
-
# https://gist.github.com/dorneanu/cce1cd6711969d581873a88e0257e312
|
22
|
-
# https://kaleidoescape.github.io/decorated-plugins/
|
23
|
-
|
24
|
-
|
25
|
-
class ReduceType(Enum):
|
26
|
-
AUTO = auto() # automatically determine the best way to reduce the dataset
|
27
|
-
SQUEEZE = auto() # remove any dimensions of length 1
|
28
|
-
REDUCE = auto() # get the mean and std dev of the the "repeat" dimension
|
29
|
-
NONE = auto() # don't reduce
|
30
|
-
|
31
|
-
|
32
|
-
class EmptyContainer:
|
33
|
-
"""A wrapper for list like containers that only appends if the item is not None"""
|
34
|
-
|
35
|
-
def __init__(self, pane) -> None:
|
36
|
-
self.pane = pane
|
37
|
-
|
38
|
-
def append(self, child):
|
39
|
-
if child is not None:
|
40
|
-
self.pane.append(child)
|
41
|
-
|
42
|
-
def get(self):
|
43
|
-
return self.pane if len(self.pane) > 0 else None
|
44
|
-
|
45
|
-
|
46
|
-
class BenchResultBase(OptunaResult):
|
47
|
-
def result_samples(self) -> int:
|
48
|
-
"""The number of samples in the results dataframe"""
|
49
|
-
return self.ds.count()
|
50
|
-
|
51
|
-
def to_hv_dataset(
|
52
|
-
self, reduce: ReduceType = ReduceType.AUTO, result_var: ResultVar = None
|
53
|
-
) -> hv.Dataset:
|
54
|
-
"""Generate a holoviews dataset from the xarray dataset.
|
55
|
-
|
56
|
-
Args:
|
57
|
-
reduce (ReduceType, optional): Optionally perform reduce options on the dataset. By default the returned dataset will calculate the mean and standard devation over the "repeat" dimension so that the dataset plays nicely with most of the holoviews plot types. Reduce.Sqeeze is used if there is only 1 repeat and you want the "reduce" variable removed from the dataset. ReduceType.None returns an unaltered dataset. Defaults to ReduceType.AUTO.
|
58
|
-
|
59
|
-
Returns:
|
60
|
-
hv.Dataset: results in the form of a holoviews dataset
|
61
|
-
"""
|
62
|
-
|
63
|
-
if reduce == ReduceType.NONE:
|
64
|
-
kdims = [i.name for i in self.bench_cfg.all_vars]
|
65
|
-
return hv.Dataset(self.to_dataset(reduce, result_var), kdims=kdims)
|
66
|
-
return hv.Dataset(self.to_dataset(reduce, result_var))
|
67
|
-
|
68
|
-
def to_dataset(
|
69
|
-
self, reduce: ReduceType = ReduceType.AUTO, result_var: ResultVar = None
|
70
|
-
) -> xr.Dataset:
|
71
|
-
"""Generate a summarised xarray dataset.
|
72
|
-
|
73
|
-
Args:
|
74
|
-
reduce (ReduceType, optional): Optionally perform reduce options on the dataset. By default the returned dataset will calculate the mean and standard devation over the "repeat" dimension so that the dataset plays nicely with most of the holoviews plot types. Reduce.Sqeeze is used if there is only 1 repeat and you want the "reduce" variable removed from the dataset. ReduceType.None returns an unaltered dataset. Defaults to ReduceType.AUTO.
|
75
|
-
|
76
|
-
Returns:
|
77
|
-
xr.Dataset: results in the form of an xarray dataset
|
78
|
-
"""
|
79
|
-
if reduce == ReduceType.AUTO:
|
80
|
-
reduce = ReduceType.REDUCE if self.bench_cfg.repeats > 1 else ReduceType.SQUEEZE
|
81
|
-
|
82
|
-
ds = self.ds if result_var is None else self.ds[result_var.name]
|
83
|
-
|
84
|
-
match (reduce):
|
85
|
-
case ReduceType.REDUCE:
|
86
|
-
ds_reduce_mean = ds.mean(dim="repeat", keep_attrs=True)
|
87
|
-
ds_reduce_std = ds.std(dim="repeat", keep_attrs=True)
|
88
|
-
|
89
|
-
for v in ds_reduce_mean.data_vars:
|
90
|
-
ds_reduce_mean[f"{v}_std"] = ds_reduce_std[v]
|
91
|
-
return ds_reduce_mean
|
92
|
-
case ReduceType.SQUEEZE:
|
93
|
-
return ds.squeeze(drop=True)
|
94
|
-
case _:
|
95
|
-
return ds
|
96
|
-
|
97
|
-
def get_optimal_vec(
|
98
|
-
self,
|
99
|
-
result_var: ParametrizedSweep,
|
100
|
-
input_vars: List[ParametrizedSweep],
|
101
|
-
) -> List[Any]:
|
102
|
-
"""Get the optimal values from the sweep as a vector.
|
103
|
-
|
104
|
-
Args:
|
105
|
-
result_var (bch.ParametrizedSweep): Optimal values of this result variable
|
106
|
-
input_vars (List[bch.ParametrizedSweep]): Define which input vars values are returned in the vector
|
107
|
-
|
108
|
-
Returns:
|
109
|
-
List[Any]: A vector of optimal values for the desired input vector
|
110
|
-
"""
|
111
|
-
|
112
|
-
da = self.get_optimal_value_indices(result_var)
|
113
|
-
output = []
|
114
|
-
for iv in input_vars:
|
115
|
-
if da.coords[iv.name].values.size == 1:
|
116
|
-
# https://stackoverflow.com/questions/773030/why-are-0d-arrays-in-numpy-not-considered-scalar
|
117
|
-
# use [()] to convert from a 0d numpy array to a scalar
|
118
|
-
output.append(da.coords[iv.name].values[()])
|
119
|
-
else:
|
120
|
-
logging.warning(f"values size: {da.coords[iv.name].values.size}")
|
121
|
-
output.append(max(da.coords[iv.name].values[()]))
|
122
|
-
logging.info(f"Maximum value of {iv.name}: {output[-1]}")
|
123
|
-
return output
|
124
|
-
|
125
|
-
def get_optimal_value_indices(self, result_var: ParametrizedSweep) -> xr.DataArray:
|
126
|
-
"""Get an xarray mask of the values with the best values found during a parameter sweep
|
127
|
-
|
128
|
-
Args:
|
129
|
-
result_var (bch.ParametrizedSweep): Optimal value of this result variable
|
130
|
-
|
131
|
-
Returns:
|
132
|
-
xr.DataArray: xarray mask of optimal values
|
133
|
-
"""
|
134
|
-
result_da = self.ds[result_var.name]
|
135
|
-
if result_var.direction == OptDir.maximize:
|
136
|
-
opt_val = result_da.max()
|
137
|
-
else:
|
138
|
-
opt_val = result_da.min()
|
139
|
-
indicies = result_da.where(result_da == opt_val, drop=True).squeeze()
|
140
|
-
logging.info(f"optimal value of {result_var.name}: {opt_val.values}")
|
141
|
-
return indicies
|
142
|
-
|
143
|
-
def get_optimal_inputs(
|
144
|
-
self,
|
145
|
-
result_var: ParametrizedSweep,
|
146
|
-
keep_existing_consts: bool = True,
|
147
|
-
as_dict: bool = False,
|
148
|
-
) -> Tuple[ParametrizedSweep, Any] | dict[ParametrizedSweep, Any]:
|
149
|
-
"""Get a list of tuples of optimal variable names and value pairs, that can be fed in as constant values to subsequent parameter sweeps
|
150
|
-
|
151
|
-
Args:
|
152
|
-
result_var (bch.ParametrizedSweep): Optimal values of this result variable
|
153
|
-
keep_existing_consts (bool): Include any const values that were defined as part of the parameter sweep
|
154
|
-
as_dict (bool): return value as a dictionary
|
155
|
-
|
156
|
-
Returns:
|
157
|
-
Tuple[bch.ParametrizedSweep, Any]|[ParametrizedSweep, Any]: Tuples of variable name and optimal values
|
158
|
-
"""
|
159
|
-
da = self.get_optimal_value_indices(result_var)
|
160
|
-
if keep_existing_consts:
|
161
|
-
output = deepcopy(self.bench_cfg.const_vars)
|
162
|
-
else:
|
163
|
-
output = []
|
164
|
-
|
165
|
-
for iv in self.bench_cfg.input_vars:
|
166
|
-
# assert da.coords[iv.name].values.size == (1,)
|
167
|
-
if da.coords[iv.name].values.size == 1:
|
168
|
-
# https://stackoverflow.com/questions/773030/why-are-0d-arrays-in-numpy-not-considered-scalar
|
169
|
-
# use [()] to convert from a 0d numpy array to a scalar
|
170
|
-
output.append((iv, da.coords[iv.name].values[()]))
|
171
|
-
else:
|
172
|
-
logging.warning(f"values size: {da.coords[iv.name].values.size}")
|
173
|
-
output.append((iv, max(da.coords[iv.name].values[()])))
|
174
|
-
|
175
|
-
logging.info(f"Maximum value of {iv.name}: {output[-1][1]}")
|
176
|
-
if as_dict:
|
177
|
-
return dict(output)
|
178
|
-
return output
|
179
|
-
|
180
|
-
def describe_sweep(self):
|
181
|
-
return self.bench_cfg.describe_sweep()
|
182
|
-
|
183
|
-
def get_best_holomap(self, name: str = None):
|
184
|
-
return self.get_hmap(name)[self.get_best_trial_params(True)]
|
185
|
-
|
186
|
-
def get_hmap(self, name: str = None):
|
187
|
-
try:
|
188
|
-
if name is None:
|
189
|
-
name = self.result_hmaps[0].name
|
190
|
-
if name in self.hmaps:
|
191
|
-
return self.hmaps[name]
|
192
|
-
except Exception as e:
|
193
|
-
raise RuntimeError(
|
194
|
-
"You are trying to plot a holomap result but it is not in the result_vars list. Add the holomap to the result_vars list"
|
195
|
-
) from e
|
196
|
-
return None
|
197
|
-
|
198
|
-
def to_plot_title(self) -> str:
|
199
|
-
if len(self.bench_cfg.input_vars) > 0 and len(self.bench_cfg.result_vars) > 0:
|
200
|
-
return f"{self.bench_cfg.result_vars[0].name} vs {self.bench_cfg.input_vars[0].name}"
|
201
|
-
return ""
|
202
|
-
|
203
|
-
def title_from_ds(self, dataset: xr.Dataset, result_var: Parameter, **kwargs):
|
204
|
-
if "title" in kwargs:
|
205
|
-
return kwargs["title"]
|
206
|
-
|
207
|
-
if isinstance(dataset, xr.DataArray):
|
208
|
-
tit = [dataset.name]
|
209
|
-
for d in dataset.dims:
|
210
|
-
tit.append(d)
|
211
|
-
else:
|
212
|
-
tit = [result_var.name]
|
213
|
-
tit.extend(list(dataset.sizes))
|
214
|
-
|
215
|
-
return " vs ".join(tit)
|
216
|
-
|
217
|
-
def get_results_var_list(self, result_var: ParametrizedSweep = None) -> List[ResultVar]:
|
218
|
-
return self.bench_cfg.result_vars if result_var is None else [result_var]
|
219
|
-
|
220
|
-
def map_plots(
|
221
|
-
self,
|
222
|
-
plot_callback: callable,
|
223
|
-
result_var: ParametrizedSweep = None,
|
224
|
-
row: EmptyContainer = None,
|
225
|
-
) -> Optional[pn.Row]:
|
226
|
-
if row is None:
|
227
|
-
row = EmptyContainer(pn.Row(name=self.to_plot_title()))
|
228
|
-
for rv in self.get_results_var_list(result_var):
|
229
|
-
row.append(plot_callback(rv))
|
230
|
-
return row.get()
|
231
|
-
|
232
|
-
def map_plot_panes(
|
233
|
-
self,
|
234
|
-
plot_callback: callable,
|
235
|
-
hv_dataset: hv.Dataset = None,
|
236
|
-
target_dimension: int = 2,
|
237
|
-
result_var: ResultVar = None,
|
238
|
-
result_types=None,
|
239
|
-
**kwargs,
|
240
|
-
) -> Optional[pn.Row]:
|
241
|
-
if hv_dataset is None:
|
242
|
-
hv_dataset = self.to_hv_dataset()
|
243
|
-
row = EmptyContainer(pn.Row())
|
244
|
-
|
245
|
-
# kwargs= self.set_plot_size(**kwargs)
|
246
|
-
for rv in self.get_results_var_list(result_var):
|
247
|
-
if result_types is None or isinstance(rv, result_types):
|
248
|
-
row.append(
|
249
|
-
self.to_panes_multi_panel(
|
250
|
-
hv_dataset,
|
251
|
-
rv,
|
252
|
-
plot_callback=partial(plot_callback, **kwargs),
|
253
|
-
target_dimension=target_dimension,
|
254
|
-
)
|
255
|
-
)
|
256
|
-
return row.get()
|
257
|
-
|
258
|
-
def filter(
|
259
|
-
self,
|
260
|
-
plot_callback: callable,
|
261
|
-
plot_filter=None,
|
262
|
-
float_range: VarRange = VarRange(0, None),
|
263
|
-
cat_range: VarRange = VarRange(0, None),
|
264
|
-
vector_len: VarRange = VarRange(1, 1),
|
265
|
-
result_vars: VarRange = VarRange(1, 1),
|
266
|
-
panel_range: VarRange = VarRange(0, 0),
|
267
|
-
repeats_range: VarRange = VarRange(1, None),
|
268
|
-
input_range: VarRange = VarRange(1, None),
|
269
|
-
reduce: ReduceType = ReduceType.AUTO,
|
270
|
-
target_dimension: int = 2,
|
271
|
-
result_var: ResultVar = None,
|
272
|
-
result_types=None,
|
273
|
-
**kwargs,
|
274
|
-
):
|
275
|
-
plot_filter = PlotFilter(
|
276
|
-
float_range=float_range,
|
277
|
-
cat_range=cat_range,
|
278
|
-
vector_len=vector_len,
|
279
|
-
result_vars=result_vars,
|
280
|
-
panel_range=panel_range,
|
281
|
-
repeats_range=repeats_range,
|
282
|
-
input_range=input_range,
|
283
|
-
)
|
284
|
-
matches_res = plot_filter.matches_result(self.plt_cnt_cfg, callable_name(plot_callback))
|
285
|
-
if matches_res.overall:
|
286
|
-
return self.map_plot_panes(
|
287
|
-
plot_callback=plot_callback,
|
288
|
-
hv_dataset=self.to_hv_dataset(reduce=reduce),
|
289
|
-
target_dimension=target_dimension,
|
290
|
-
result_var=result_var,
|
291
|
-
result_types=result_types,
|
292
|
-
**kwargs,
|
293
|
-
)
|
294
|
-
return matches_res.to_panel()
|
295
|
-
|
296
|
-
def to_panes_multi_panel(
|
297
|
-
self,
|
298
|
-
hv_dataset: hv.Dataset,
|
299
|
-
result_var: ResultVar,
|
300
|
-
plot_callback: callable = None,
|
301
|
-
target_dimension: int = 1,
|
302
|
-
**kwargs,
|
303
|
-
):
|
304
|
-
dims = len(hv_dataset.dimensions())
|
305
|
-
if target_dimension is None:
|
306
|
-
target_dimension = dims
|
307
|
-
return self._to_panes_da(
|
308
|
-
hv_dataset.data,
|
309
|
-
plot_callback=plot_callback,
|
310
|
-
target_dimension=target_dimension,
|
311
|
-
horizontal=dims <= target_dimension + 1,
|
312
|
-
result_var=result_var,
|
313
|
-
**kwargs,
|
314
|
-
)
|
315
|
-
|
316
|
-
def _to_panes_da(
|
317
|
-
self,
|
318
|
-
dataset: xr.Dataset,
|
319
|
-
plot_callback=pn.pane.panel,
|
320
|
-
target_dimension=1,
|
321
|
-
horizontal=False,
|
322
|
-
result_var=None,
|
323
|
-
**kwargs,
|
324
|
-
) -> pn.panel:
|
325
|
-
# todo, when dealing with time and repeats, add feature to allow custom order of dimension recursion
|
326
|
-
##todo remove recursion
|
327
|
-
num_dims = len(dataset.sizes)
|
328
|
-
# print(f"num_dims: {num_dims}, horizontal: {horizontal}, target: {target_dimension}")
|
329
|
-
dims = list(d for d in dataset.sizes)
|
330
|
-
|
331
|
-
time_dim_delta = 0
|
332
|
-
if self.bench_cfg.over_time:
|
333
|
-
time_dim_delta = 0
|
334
|
-
|
335
|
-
if num_dims > (target_dimension + time_dim_delta) and num_dims != 0:
|
336
|
-
dim_sel = dims[-1]
|
337
|
-
# print(f"selected dim {dim_sel}")
|
338
|
-
|
339
|
-
dim_color = color_tuple_to_css(int_to_col(num_dims - 2, 0.05, 1.0))
|
340
|
-
|
341
|
-
background_col = dim_color
|
342
|
-
name = " vs ".join(dims)
|
343
|
-
|
344
|
-
container_args = {"name": name, "styles": {"background": background_col}}
|
345
|
-
outer_container = (
|
346
|
-
pn.Row(**container_args) if horizontal else pn.Column(**container_args)
|
347
|
-
)
|
348
|
-
|
349
|
-
max_len = 0
|
350
|
-
|
351
|
-
for i in range(dataset.sizes[dim_sel]):
|
352
|
-
sliced = dataset.isel({dim_sel: i})
|
353
|
-
|
354
|
-
lable_val = sliced.coords[dim_sel].values.item()
|
355
|
-
if isinstance(lable_val, (int, float)):
|
356
|
-
lable_val = FormatFloat()(lable_val)
|
357
|
-
|
358
|
-
label = f"{dim_sel}={lable_val}"
|
359
|
-
|
360
|
-
panes = self._to_panes_da(
|
361
|
-
sliced,
|
362
|
-
plot_callback=plot_callback,
|
363
|
-
target_dimension=target_dimension,
|
364
|
-
horizontal=len(sliced.sizes) <= target_dimension + 1,
|
365
|
-
result_var=result_var,
|
366
|
-
)
|
367
|
-
width = num_dims - target_dimension
|
368
|
-
|
369
|
-
container_args = {
|
370
|
-
"name": name,
|
371
|
-
"styles": {"border-bottom": f"{width}px solid grey"},
|
372
|
-
}
|
373
|
-
|
374
|
-
if horizontal:
|
375
|
-
inner_container = pn.Column(**container_args)
|
376
|
-
align = ("center", "center")
|
377
|
-
else:
|
378
|
-
inner_container = pn.Row(**container_args)
|
379
|
-
align = ("end", "center")
|
380
|
-
|
381
|
-
label_len = len(label)
|
382
|
-
if label_len > max_len:
|
383
|
-
max_len = label_len
|
384
|
-
side = pn.pane.Markdown(label, align=align)
|
385
|
-
|
386
|
-
inner_container.append(side)
|
387
|
-
inner_container.append(panes)
|
388
|
-
outer_container.append(inner_container)
|
389
|
-
# outer_container.append(pn.Row(inner_container, align="center"))
|
390
|
-
for c in outer_container:
|
391
|
-
c[0].width = max_len * 7
|
392
|
-
else:
|
393
|
-
return plot_callback(dataset=dataset, result_var=result_var, **kwargs)
|
394
|
-
|
395
|
-
return outer_container
|
396
|
-
|
397
|
-
# MAPPING TO LOWER LEVEL BENCHCFG functions so they are available at a top level.
|
398
|
-
def to_sweep_summary(self, **kwargs):
|
399
|
-
return self.bench_cfg.to_sweep_summary(**kwargs)
|
400
|
-
|
401
|
-
def to_title(self, panel_name: str = None) -> pn.pane.Markdown:
|
402
|
-
return self.bench_cfg.to_title(panel_name)
|
403
|
-
|
404
|
-
def to_description(self, width: int = 800) -> pn.pane.Markdown:
|
405
|
-
return self.bench_cfg.to_description(width)
|