holobench 1.35.0__py3-none-any.whl → 1.36.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bencher/bench_cfg.py +6 -6
- bencher/bench_runner.py +8 -8
- bencher/bencher.py +4 -4
- bencher/example/example_composable_container.py +1 -1
- bencher/example/example_composable_container2.py +2 -2
- bencher/example/example_holosweep.py +1 -1
- bencher/example/example_image.py +2 -2
- bencher/example/example_image1.py +1 -1
- bencher/example/example_sample_cache.py +4 -4
- bencher/example/example_sample_cache_context.py +1 -1
- bencher/example/example_video.py +13 -13
- bencher/example/experimental/example_hvplot_explorer.py +1 -1
- bencher/example/inputs_1D/example_1D.py +0 -9
- bencher/example/shelved/example_float3D_cone.py +1 -1
- bencher/job.py +7 -4
- bencher/results/bench_result_base.py +27 -7
- bencher/results/composable_container/composable_container_video.py +3 -3
- bencher/results/holoview_result.py +1 -1
- bencher/results/video_summary.py +5 -0
- {holobench-1.35.0.dist-info → holobench-1.36.1.dist-info}/METADATA +1 -1
- {holobench-1.35.0.dist-info → holobench-1.36.1.dist-info}/RECORD +23 -23
- {holobench-1.35.0.dist-info → holobench-1.36.1.dist-info}/WHEEL +0 -0
- {holobench-1.35.0.dist-info → holobench-1.36.1.dist-info}/licenses/LICENSE +0 -0
bencher/bench_cfg.py
CHANGED
@@ -83,16 +83,16 @@ class BenchRunCfg(BenchPlotSrvCfg):
|
|
83
83
|
|
84
84
|
raise_duplicate_exception: bool = param.Boolean(False, doc=" Used to debug unique plot names.")
|
85
85
|
|
86
|
-
|
86
|
+
cache_results: bool = param.Boolean(
|
87
87
|
False,
|
88
|
-
doc="This is a benchmark level cache that stores the results of a fully completed benchmark. At the end of a benchmark the values are added to the cache but are not if the benchmark does not complete. If you want to cache values during the benchmark you need to use the
|
88
|
+
doc="This is a benchmark level cache that stores the results of a fully completed benchmark. At the end of a benchmark the values are added to the cache but are not if the benchmark does not complete. If you want to cache values during the benchmark you need to use the cache_samples option. Beware that depending on how you change code in the objective function, the cache could provide values that are not correct.",
|
89
89
|
)
|
90
90
|
|
91
91
|
clear_cache: bool = param.Boolean(
|
92
92
|
False, doc=" Clear the cache of saved input->output mappings."
|
93
93
|
)
|
94
94
|
|
95
|
-
|
95
|
+
cache_samples: bool = param.Boolean(
|
96
96
|
False,
|
97
97
|
doc="If true, every time the benchmark function is called, bencher will check if that value has been calculated before and if so load the from the cache. Note that the sample level cache is different from the benchmark level cache which only caches the aggregate of all the results at the end of the benchmark. This cache lets you stop a benchmark halfway through and continue. However, beware that depending on how you change code in the objective function, the cache could provide values that are not correct.",
|
98
98
|
)
|
@@ -182,7 +182,7 @@ class BenchRunCfg(BenchPlotSrvCfg):
|
|
182
182
|
parser.add_argument(
|
183
183
|
"--use-cache",
|
184
184
|
action="store_true",
|
185
|
-
help=BenchRunCfg.param.
|
185
|
+
help=BenchRunCfg.param.cache_results.doc,
|
186
186
|
)
|
187
187
|
|
188
188
|
parser.add_argument(
|
@@ -380,8 +380,8 @@ class BenchCfg(BenchRunCfg):
|
|
380
380
|
benchmark_sampling_str.append(f" run tag: {self.run_tag}")
|
381
381
|
if self.level is not None:
|
382
382
|
benchmark_sampling_str.append(f" bench level: {self.level}")
|
383
|
-
benchmark_sampling_str.append(f"
|
384
|
-
benchmark_sampling_str.append(f"
|
383
|
+
benchmark_sampling_str.append(f" cache_results: {self.cache_results}")
|
384
|
+
benchmark_sampling_str.append(f" cache_samples {self.cache_samples}")
|
385
385
|
benchmark_sampling_str.append(f" only_hash_tag: {self.only_hash_tag}")
|
386
386
|
benchmark_sampling_str.append(f" executor: {self.executor}")
|
387
387
|
|
bencher/bench_runner.py
CHANGED
@@ -33,11 +33,11 @@ class BenchRunner:
|
|
33
33
|
|
34
34
|
@staticmethod
|
35
35
|
def setup_run_cfg(
|
36
|
-
run_cfg: BenchRunCfg = BenchRunCfg(), level: int = 2,
|
36
|
+
run_cfg: BenchRunCfg = BenchRunCfg(), level: int = 2, cache_results=True
|
37
37
|
) -> BenchRunCfg:
|
38
38
|
run_cfg_out = deepcopy(run_cfg)
|
39
|
-
run_cfg_out.
|
40
|
-
run_cfg_out.only_hash_tag =
|
39
|
+
run_cfg_out.cache_samples = cache_results
|
40
|
+
run_cfg_out.only_hash_tag = cache_results
|
41
41
|
run_cfg_out.level = level
|
42
42
|
return run_cfg_out
|
43
43
|
|
@@ -78,9 +78,9 @@ class BenchRunner:
|
|
78
78
|
show: bool = False,
|
79
79
|
save: bool = False,
|
80
80
|
grouped: bool = True,
|
81
|
-
|
81
|
+
cache_results: bool = True,
|
82
82
|
) -> List[Bench]:
|
83
|
-
"""This function controls how a benchmark or a set of benchmarks are run. If you are only running a single benchmark it can be simpler to just run it directly, but if you are running several benchmarks together and want them to be sampled at different levels of fidelity or published together in a single report this function enables that workflow. If you have an expensive function, it can be useful to view low fidelity results as they are computed but also continue to compute higher fidelity results while reusing previously computed values. The parameters min_level and max_level let you specify how to progressivly increase the sampling resolution of the benchmark sweep. By default
|
83
|
+
"""This function controls how a benchmark or a set of benchmarks are run. If you are only running a single benchmark it can be simpler to just run it directly, but if you are running several benchmarks together and want them to be sampled at different levels of fidelity or published together in a single report this function enables that workflow. If you have an expensive function, it can be useful to view low fidelity results as they are computed but also continue to compute higher fidelity results while reusing previously computed values. The parameters min_level and max_level let you specify how to progressivly increase the sampling resolution of the benchmark sweep. By default cache_results=True so that previous values are reused.
|
84
84
|
|
85
85
|
Args:
|
86
86
|
min_level (int, optional): The minimum level to start sampling at. Defaults to 2.
|
@@ -93,14 +93,14 @@ class BenchRunner:
|
|
93
93
|
show (bool, optional): show the results in the local web browser. Defaults to False.
|
94
94
|
save (bool, optional): save the results to disk in index.html. Defaults to False.
|
95
95
|
grouped (bool, optional): Produce a single html page with all the benchmarks included. Defaults to True.
|
96
|
-
|
96
|
+
cache_results (bool, optional): Use the sample cache to reused previous results. Defaults to True.
|
97
97
|
|
98
98
|
Returns:
|
99
99
|
List[BenchCfg]: A list of bencher instances
|
100
100
|
"""
|
101
101
|
if run_cfg is None:
|
102
102
|
run_cfg = deepcopy(self.run_cfg)
|
103
|
-
run_cfg = BenchRunner.setup_run_cfg(run_cfg,
|
103
|
+
run_cfg = BenchRunner.setup_run_cfg(run_cfg, cache_results=cache_results)
|
104
104
|
|
105
105
|
if level is not None:
|
106
106
|
min_level = level
|
@@ -136,7 +136,7 @@ class BenchRunner:
|
|
136
136
|
else:
|
137
137
|
report.publish(remote_callback=self.publisher, debug=debug)
|
138
138
|
if show:
|
139
|
-
self.servers.append(report.show())
|
139
|
+
self.servers.append(report.show(self.run_cfg))
|
140
140
|
|
141
141
|
def shutdown(self):
|
142
142
|
while self.servers:
|
bencher/bencher.py
CHANGED
@@ -327,7 +327,7 @@ class Bench(BenchPlotServer):
|
|
327
327
|
logging.info("Copy run cfg from bench class")
|
328
328
|
|
329
329
|
if run_cfg.only_plot:
|
330
|
-
run_cfg.
|
330
|
+
run_cfg.cache_results = True
|
331
331
|
|
332
332
|
self.last_run_cfg = run_cfg
|
333
333
|
|
@@ -371,7 +371,7 @@ class Bench(BenchPlotServer):
|
|
371
371
|
title += "s"
|
372
372
|
title += ": " + ", ".join([f"{c[0].name}={c[1]}" for c in const_vars_in])
|
373
373
|
else:
|
374
|
-
title = " ".join([i.name for i in result_vars_in])
|
374
|
+
title = "Recording: " + ", ".join([i.name for i in result_vars_in])
|
375
375
|
|
376
376
|
if run_cfg.level > 0:
|
377
377
|
inputs = []
|
@@ -448,7 +448,7 @@ class Bench(BenchPlotServer):
|
|
448
448
|
if run_cfg.clear_cache:
|
449
449
|
c.delete(bench_cfg_hash)
|
450
450
|
logging.info("cleared cache")
|
451
|
-
elif run_cfg.
|
451
|
+
elif run_cfg.cache_results:
|
452
452
|
logging.info(
|
453
453
|
f"checking for previously calculated results with key: {bench_cfg_hash}"
|
454
454
|
)
|
@@ -813,7 +813,7 @@ class Bench(BenchPlotServer):
|
|
813
813
|
cache_name="sample_cache",
|
814
814
|
tag_index=True,
|
815
815
|
size_limit=self.cache_size,
|
816
|
-
|
816
|
+
cache_results=run_cfg.cache_samples,
|
817
817
|
)
|
818
818
|
|
819
819
|
def clear_tag_from_sample_cache(self, tag: str, run_cfg):
|
@@ -98,7 +98,7 @@ def example_composable_container_video(
|
|
98
98
|
|
99
99
|
if __name__ == "__main__":
|
100
100
|
ex_run_cfg = bch.BenchRunCfg()
|
101
|
-
ex_run_cfg.
|
101
|
+
ex_run_cfg.cache_samples = False
|
102
102
|
# ex_run_cfg.level = 2
|
103
103
|
ex_report = bch.BenchReport()
|
104
104
|
example_composable_container_image(ex_run_cfg, report=ex_report)
|
@@ -144,7 +144,7 @@ def example_composable_container_image(
|
|
144
144
|
|
145
145
|
# if __name__ == "__main__":
|
146
146
|
# ex_run_cfg = bch.BenchRunCfg()
|
147
|
-
# ex_run_cfg.
|
147
|
+
# ex_run_cfg.cache_samples = False
|
148
148
|
# # ex_run_cfg.level = 2
|
149
149
|
# ex_report = bch.BenchReport()
|
150
150
|
# example_composable_container_image(ex_run_cfg, report=ex_report)
|
@@ -157,4 +157,4 @@ if __name__ == "__main__":
|
|
157
157
|
# bench_runner.add_run(bench_image)
|
158
158
|
bench_runner.add_run(example_composable_container_image)
|
159
159
|
|
160
|
-
bench_runner.run(level=6, show=True,
|
160
|
+
bench_runner.run(level=6, show=True, cache_results=False)
|
bencher/example/example_image.py
CHANGED
@@ -58,7 +58,7 @@ class BenchPolygons(bch.ParametrizedSweep):
|
|
58
58
|
def example_image(
|
59
59
|
run_cfg: bch.BenchRunCfg = bch.BenchRunCfg(), report: bch.BenchReport = bch.BenchReport()
|
60
60
|
) -> bch.Bench:
|
61
|
-
run_cfg.
|
61
|
+
run_cfg.cache_results = False
|
62
62
|
bench = bch.Bench("polygons", BenchPolygons(), run_cfg=run_cfg, report=report)
|
63
63
|
|
64
64
|
bench.result_vars = ["polygon", "area"]
|
@@ -142,7 +142,7 @@ if __name__ == "__main__":
|
|
142
142
|
# def example_image_pairs()
|
143
143
|
|
144
144
|
ex_run_cfg = bch.BenchRunCfg()
|
145
|
-
ex_run_cfg.
|
145
|
+
ex_run_cfg.cache_samples = True
|
146
146
|
# ex_run_cfg.debug = True
|
147
147
|
# ex_run_cfg.repeats = 2
|
148
148
|
ex_run_cfg.level = 4
|
@@ -2,7 +2,7 @@ import bencher as bch
|
|
2
2
|
|
3
3
|
|
4
4
|
class UnreliableClass(bch.ParametrizedSweep):
|
5
|
-
"""This class helps demonstrate benchmarking a function that sometimes crashes during sampling. By using BenchRunCfg.
|
5
|
+
"""This class helps demonstrate benchmarking a function that sometimes crashes during sampling. By using BenchRunCfg.cache_samples you can store the results of every call to the benchmark function so data is not lost in the event of a crash. However, because cache invalidation is hard (https://martinfowler.com/bliki/TwoHardThings.html) you need to be mindful of how you could get bad results due to incorrect cache data. For example if you change your benchmark function and use the sample cache you will not get correct values; you will need to use BenchRunCfg.clear_sample_cache to purge any out of date results."""
|
6
6
|
|
7
7
|
input_val = bch.IntSweep(
|
8
8
|
default=0,
|
@@ -31,7 +31,7 @@ def example_sample_cache(
|
|
31
31
|
report: bch.BenchReport = bch.BenchReport(),
|
32
32
|
trigger_crash: bool = False,
|
33
33
|
) -> bch.Bench:
|
34
|
-
"""This example shows how to use the
|
34
|
+
"""This example shows how to use the cache_samples option to deal with unreliable functions and to continue benchmarking using previously calculated results even if the code crashed during the run
|
35
35
|
|
36
36
|
Args:
|
37
37
|
run_cfg (BenchRunCfg): configuration of how to perform the param sweep
|
@@ -50,7 +50,7 @@ def example_sample_cache(
|
|
50
50
|
title="Example Crashy Function with the sample_cache",
|
51
51
|
input_vars=[UnreliableClass.param.input_val],
|
52
52
|
result_vars=[UnreliableClass.param.return_value, UnreliableClass.param.trigger_crash],
|
53
|
-
description="""This example shows how to use the
|
53
|
+
description="""This example shows how to use the cache_samples option to deal with unreliable functions and to continue benchmarking using previously calculated results even if the code crashed during the run""",
|
54
54
|
run_cfg=run_cfg,
|
55
55
|
post_description="The input_val vs return value graph is a straight line as expected and there is no record of the fact the benchmark crashed halfway through. The second graph shows that for values >1 the trigger_crash value had to be 0 in order to proceed",
|
56
56
|
)
|
@@ -63,7 +63,7 @@ if __name__ == "__main__":
|
|
63
63
|
ex_run_cfg.executor = bch.Executors.SCOOP
|
64
64
|
|
65
65
|
# this will store the result of of every call to crashy_fn
|
66
|
-
ex_run_cfg.
|
66
|
+
ex_run_cfg.cache_samples = True
|
67
67
|
ex_run_cfg.clear_sample_cache = True
|
68
68
|
|
69
69
|
try:
|
@@ -51,7 +51,7 @@ def assert_call_counts(bencher, run_cfg, wrapper_calls=-1, fn_calls=-1, cache_ca
|
|
51
51
|
|
52
52
|
def example_cache_context() -> bch.Bench:
|
53
53
|
run_cfg = bch.BenchRunCfg()
|
54
|
-
run_cfg.
|
54
|
+
run_cfg.cache_samples = True
|
55
55
|
run_cfg.only_hash_tag = True
|
56
56
|
run_cfg.repeats = 2
|
57
57
|
run_cfg.parallel = False
|
bencher/example/example_video.py
CHANGED
@@ -78,14 +78,11 @@ class TuringPattern(bch.ParametrizedSweep):
|
|
78
78
|
def example_video(
|
79
79
|
run_cfg: bch.BenchRunCfg = bch.BenchRunCfg(), report: bch.BenchReport = bch.BenchReport()
|
80
80
|
) -> bch.Bench:
|
81
|
-
|
82
|
-
# run_cfg.use_sample_cache = True
|
83
|
-
bench = bch.Bench("example_video", TuringPattern(), run_cfg=run_cfg, report=report)
|
81
|
+
bench = TuringPattern().to_bench(run_cfg, report)
|
84
82
|
|
85
83
|
bench.plot_sweep(
|
86
84
|
"Turing patterns with different parameters",
|
87
|
-
input_vars=[
|
88
|
-
# input_vars=[TuringPattern.param.alpha],
|
85
|
+
input_vars=["alpha", "beta"],
|
89
86
|
result_vars=[TuringPattern.param.video],
|
90
87
|
)
|
91
88
|
|
@@ -96,14 +93,17 @@ def example_video_tap(
|
|
96
93
|
run_cfg: bch.BenchRunCfg = bch.BenchRunCfg(), report: bch.BenchReport = bch.BenchReport()
|
97
94
|
) -> bch.Bench: # pragma: no cover
|
98
95
|
bench = TuringPattern().to_bench(run_cfg=run_cfg, report=report)
|
99
|
-
res = bench.plot_sweep(
|
100
|
-
input_vars=["alpha", "beta"],
|
101
|
-
# result_vars=["video","score"],
|
102
|
-
# result_vars=["score"],
|
103
|
-
run_cfg=run_cfg,
|
104
|
-
)
|
96
|
+
res = bench.plot_sweep(input_vars=["alpha", "beta"])
|
105
97
|
|
106
|
-
bench.report.append(res.to_video_grid())
|
98
|
+
bench.report.append(res.to_video_grid(result_types=(bch.ResultVideo)))
|
99
|
+
|
100
|
+
res = bench.plot_sweep(input_vars=["alpha"])
|
101
|
+
bench.report.append(
|
102
|
+
res.to_video_grid(
|
103
|
+
result_types=(bch.ResultVideo),
|
104
|
+
compose_method_list=[bch.ComposeType.right],
|
105
|
+
)
|
106
|
+
)
|
107
107
|
|
108
108
|
return bench
|
109
109
|
|
@@ -111,7 +111,7 @@ def example_video_tap(
|
|
111
111
|
if __name__ == "__main__":
|
112
112
|
run_cfg_ex = bch.BenchRunCfg()
|
113
113
|
run_cfg_ex.level = 2
|
114
|
-
run_cfg_ex.
|
114
|
+
run_cfg_ex.cache_samples = True
|
115
115
|
run_cfg_ex.only_hash_tag = True
|
116
116
|
|
117
117
|
# example_video(run_cfg_ex).report.show()
|
@@ -30,7 +30,7 @@ if __name__ == "__main__":
|
|
30
30
|
post_description="Here you can see the output plot of sin theta between 0 and pi. In the tabs at the top you can also view 3 tabular representations of the data",
|
31
31
|
run_cfg=bch.BenchRunCfg(
|
32
32
|
auto_plot=True,
|
33
|
-
|
33
|
+
cache_results=False,
|
34
34
|
repeats=2,
|
35
35
|
),
|
36
36
|
)
|
@@ -34,18 +34,9 @@ def example_1D_float_repeats(
|
|
34
34
|
"""This example shows how to sample a 1 dimensional float variable and plot the result of passing that parameter sweep to the benchmarking function"""
|
35
35
|
|
36
36
|
bench = Example1D().to_bench(run_cfg, report)
|
37
|
-
# bench.plot_sweep(pass_repeat=True,plot_callbacks=False)
|
38
|
-
|
39
|
-
# res = bench.get_result()
|
40
37
|
bench.run_cfg = bch.BenchRunCfg(repeats=4)
|
41
38
|
bench.plot_sweep(pass_repeat=True)
|
42
39
|
|
43
|
-
res = bench.get_result()
|
44
|
-
bench.report.append(res.to_auto())
|
45
|
-
bench.report.append(res.to_scatter())
|
46
|
-
bench.report.append(res.to_scatter_jitter(override=True))
|
47
|
-
|
48
|
-
# bench.report.append()
|
49
40
|
return bench
|
50
41
|
|
51
42
|
|
bencher/job.py
CHANGED
@@ -36,7 +36,10 @@ class JobFuture:
|
|
36
36
|
self.res = res
|
37
37
|
self.future = future
|
38
38
|
# either a result or a future needs to be passed
|
39
|
-
assert
|
39
|
+
assert (
|
40
|
+
self.res is not None or self.future is not None
|
41
|
+
), "make sure you are returning a dict or super().__call__(**kwargs) from your __call__ function"
|
42
|
+
|
40
43
|
self.cache = cache
|
41
44
|
|
42
45
|
def result(self):
|
@@ -59,7 +62,7 @@ class Executors(StrEnum):
|
|
59
62
|
# THREADS=auto() #not that useful as most bench code is cpu bound
|
60
63
|
|
61
64
|
@staticmethod
|
62
|
-
def factory(provider: Executors) -> Future
|
65
|
+
def factory(provider: Executors) -> Future:
|
63
66
|
providers = {
|
64
67
|
Executors.SERIAL: None,
|
65
68
|
Executors.MULTIPROCESSING: ProcessPoolExecutor(),
|
@@ -78,11 +81,11 @@ class FutureCache:
|
|
78
81
|
cache_name: str = "fcache",
|
79
82
|
tag_index: bool = True,
|
80
83
|
size_limit: int = int(20e9), # 20 GB
|
81
|
-
|
84
|
+
cache_results=True,
|
82
85
|
):
|
83
86
|
self.executor_type = executor
|
84
87
|
self.executor = None
|
85
|
-
if
|
88
|
+
if cache_results:
|
86
89
|
self.cache = Cache(f"cachedir/{cache_name}", tag_index=tag_index, size_limit=size_limit)
|
87
90
|
logging.info(f"cache dir: {self.cache.directory}")
|
88
91
|
else:
|
@@ -33,7 +33,8 @@ from bencher.results.composable_container.composable_container_panel import (
|
|
33
33
|
class ReduceType(Enum):
|
34
34
|
AUTO = auto() # automatically determine the best way to reduce the dataset
|
35
35
|
SQUEEZE = auto() # remove any dimensions of length 1
|
36
|
-
REDUCE = auto() # get the mean and std dev of the the "repeat" dimension
|
36
|
+
REDUCE = auto() # get the mean and std dev of the data along the "repeat" dimension
|
37
|
+
MINMAX = auto() # get the minimum and maximum of data along the "repeat" dimension
|
37
38
|
NONE = auto() # don't reduce
|
38
39
|
|
39
40
|
|
@@ -93,16 +94,35 @@ class BenchResultBase(OptunaResult):
|
|
93
94
|
if reduce == ReduceType.AUTO:
|
94
95
|
reduce = ReduceType.REDUCE if self.bench_cfg.repeats > 1 else ReduceType.SQUEEZE
|
95
96
|
|
96
|
-
ds_out = self.ds
|
97
|
+
ds_out = self.ds.copy()
|
98
|
+
|
99
|
+
if result_var is not None:
|
100
|
+
ds_out = ds_out[result_var.name]
|
101
|
+
|
102
|
+
def rename_ds(dataset: xr.Dataset, suffix: str):
|
103
|
+
# var_name =
|
104
|
+
rename_dict = {var: f"{var}_{suffix}" for var in dataset.data_vars}
|
105
|
+
ds = dataset.rename_vars(rename_dict)
|
106
|
+
return ds
|
97
107
|
|
98
108
|
match reduce:
|
99
109
|
case ReduceType.REDUCE:
|
100
110
|
ds_reduce_mean = ds_out.mean(dim="repeat", keep_attrs=True)
|
101
|
-
ds_reduce_std = ds_out.std(dim="repeat", keep_attrs=
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
111
|
+
ds_reduce_std = ds_out.std(dim="repeat", keep_attrs=False)
|
112
|
+
ds_reduce_std = rename_ds(ds_reduce_std, "std")
|
113
|
+
ds_out = xr.merge([ds_reduce_mean, ds_reduce_std])
|
114
|
+
ds_out = xr.merge(
|
115
|
+
[
|
116
|
+
ds_reduce_mean,
|
117
|
+
ds_reduce_std,
|
118
|
+
]
|
119
|
+
)
|
120
|
+
case ReduceType.MINMAX: # TODO, need to pass mean, center of minmax, and minmax
|
121
|
+
ds_reduce_mean = ds_out.mean(dim="repeat", keep_attrs=True)
|
122
|
+
ds_reduce_min = ds_out.min(dim="repeat")
|
123
|
+
ds_reduce_max = ds_out.max(dim="repeat")
|
124
|
+
ds_reduce_range = rename_ds(ds_reduce_max - ds_reduce_min, "range")
|
125
|
+
ds_out = xr.merge([ds_reduce_mean, ds_reduce_range])
|
106
126
|
case ReduceType.SQUEEZE:
|
107
127
|
ds_out = ds_out.squeeze(drop=True)
|
108
128
|
if level is not None:
|
@@ -100,7 +100,7 @@ class ComposableContainerVideo(ComposableContainerBase):
|
|
100
100
|
print("rc", render_cfg)
|
101
101
|
_, frame_duration = self.calculate_duration(float(len(self.container)), render_cfg)
|
102
102
|
out = None
|
103
|
-
print(f"using compose type{render_cfg.compose_method}")
|
103
|
+
print(f"using compose type: {render_cfg.compose_method}")
|
104
104
|
max_duration = 0.0
|
105
105
|
|
106
106
|
for i in range(len(self.container)):
|
@@ -129,8 +129,8 @@ class ComposableContainerVideo(ComposableContainerBase):
|
|
129
129
|
# case ComposeType.overlay:
|
130
130
|
# for i in range(len(self.container)):
|
131
131
|
# self.container[i].alpha = 1./len(self.container)
|
132
|
-
# out = CompositeVideoClip(self.container, bg_color=
|
133
|
-
# out.duration = fps
|
132
|
+
# out = CompositeVideoClip(self.container, bg_color=render_cfg.background_col)
|
133
|
+
# # out.duration = fps
|
134
134
|
case _:
|
135
135
|
raise RuntimeError("This compose type is not supported")
|
136
136
|
|
@@ -190,6 +190,7 @@ class HoloviewResult(PanelResult):
|
|
190
190
|
cat_range=VarRange(0, None),
|
191
191
|
repeats_range=VarRange(2, None),
|
192
192
|
reduce=ReduceType.REDUCE,
|
193
|
+
# reduce=ReduceType.MINMAX,
|
193
194
|
target_dimension=2,
|
194
195
|
result_var=result_var,
|
195
196
|
result_types=(ResultVar),
|
@@ -200,7 +201,6 @@ class HoloviewResult(PanelResult):
|
|
200
201
|
self, dataset: xr.Dataset, result_var: Parameter, **kwargs
|
201
202
|
) -> Optional[hv.Curve]:
|
202
203
|
hvds = hv.Dataset(dataset)
|
203
|
-
# result_var = self.get_results_var_list(result_var)[0]
|
204
204
|
title = self.title_from_ds(dataset, result_var, **kwargs)
|
205
205
|
pt = hvds.to(hv.Curve).opts(title=title, **kwargs)
|
206
206
|
pt *= hvds.to(hv.Spread).opts(alpha=0.2)
|
bencher/results/video_summary.py
CHANGED
@@ -39,6 +39,7 @@ class VideoSummaryResult(BenchResultBase):
|
|
39
39
|
pane_collection: pn.pane = None,
|
40
40
|
time_sequence_dimension=0,
|
41
41
|
target_duration: float = None,
|
42
|
+
compose_method_list: List = None,
|
42
43
|
**kwargs,
|
43
44
|
) -> Optional[pn.panel]:
|
44
45
|
"""Returns the results compiled into a video
|
@@ -47,6 +48,7 @@ class VideoSummaryResult(BenchResultBase):
|
|
47
48
|
result_var (Parameter, optional): The result var to plot. Defaults to None.
|
48
49
|
result_types (tuple, optional): The types of result var to convert to video. Defaults to (ResultImage,).
|
49
50
|
collection (pn.pane, optional): If there are multiple results, use this collection to stack them. Defaults to pn.Row().
|
51
|
+
compose_method_list (List: optional): Defines how each of the dimensions is composed in the video. ie, concatenate the videos horizontally, vertically, sequentially or alpha overlay. Seee bch.ComposeType for the options.
|
50
52
|
|
51
53
|
Returns:
|
52
54
|
Optional[pn.panel]: a panel pane with a video of all results concatenated together
|
@@ -74,6 +76,7 @@ class VideoSummaryResult(BenchResultBase):
|
|
74
76
|
rv,
|
75
77
|
time_sequence_dimension=time_sequence_dimension,
|
76
78
|
target_duration=target_duration,
|
79
|
+
compose_method_list=compose_method_list,
|
77
80
|
**kwargs,
|
78
81
|
)
|
79
82
|
)
|
@@ -88,6 +91,7 @@ class VideoSummaryResult(BenchResultBase):
|
|
88
91
|
time_sequence_dimension=0,
|
89
92
|
video_controls: VideoControls = None,
|
90
93
|
target_duration: float = None,
|
94
|
+
compose_method_list: List = None,
|
91
95
|
**kwargs,
|
92
96
|
):
|
93
97
|
cvc = self._to_video_panes_ds(
|
@@ -100,6 +104,7 @@ class VideoSummaryResult(BenchResultBase):
|
|
100
104
|
result_var=result_var,
|
101
105
|
final=True,
|
102
106
|
reverse=reverse,
|
107
|
+
compose_method_list=compose_method_list,
|
103
108
|
target_duration=target_duration,
|
104
109
|
**kwargs,
|
105
110
|
)
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: holobench
|
3
|
-
Version: 1.
|
3
|
+
Version: 1.36.1
|
4
4
|
Summary: A package for benchmarking the performance of arbitrary functions
|
5
5
|
Project-URL: Repository, https://github.com/dyson-ai/bencher
|
6
6
|
Project-URL: Home, https://github.com/dyson-ai/bencher
|
@@ -1,13 +1,13 @@
|
|
1
1
|
bencher/__init__.py,sha256=hWfQxlvuHRsFK4ZPCpRXo3nDzQB52JOUoi67wcnhopE,1890
|
2
|
-
bencher/bench_cfg.py,sha256=
|
2
|
+
bencher/bench_cfg.py,sha256=gEpF1J4RAxCQPSkI8npKfhw-o3-8cw80TjiWK7As5WE,18417
|
3
3
|
bencher/bench_plot_server.py,sha256=nvGTr981XgWELqV7yID91j6V1UIPGtKilzxHcNWaZ6Q,4196
|
4
4
|
bencher/bench_report.py,sha256=ikMSHceyc8cYFH-sIza167DH-H-_iiTYDm2TmusUHDc,7515
|
5
|
-
bencher/bench_runner.py,sha256=
|
6
|
-
bencher/bencher.py,sha256
|
5
|
+
bencher/bench_runner.py,sha256=xoeXt8gcEhx5nAFsKBs8RmdkEyXUOyFd8mrEXyIxj3U,6531
|
6
|
+
bencher/bencher.py,sha256=-vbZIzBr2IYYG7be5Hh8IZgIGUysTxoxQUV6xUToH14,35437
|
7
7
|
bencher/caching.py,sha256=AusaNrzGGlj5m6zcwcqnTn55Mam2mQdF--oqelO806M,1627
|
8
8
|
bencher/class_enum.py,sha256=kYHW9qKkKcNdwaXizZL-fTptS_DUEGv4c88yCehk3gc,1492
|
9
9
|
bencher/flask_server.py,sha256=uMhMaySUki5StC-r_TXb4KTVqAiffyqfH7UzQidFqSw,831
|
10
|
-
bencher/job.py,sha256=
|
10
|
+
bencher/job.py,sha256=smSRLLZNxFSuAo7mA3m_lzHc2opjNf-DjzszsCn4ENU,6216
|
11
11
|
bencher/optuna_conversions.py,sha256=an-LfPsQXyyvhIZnG8Wl1RQVYMvJj7WOi3YNqoUnuxQ,5356
|
12
12
|
bencher/utils.py,sha256=DP2GJP28nSEihvZwiV1Rl7YJ5NTrRt2zBLs46eQ37hQ,9887
|
13
13
|
bencher/utils_rerun.py,sha256=E1itolYJMjmtBE5qcSotiS20I-dobVnjznsTRvZaV0s,1212
|
@@ -17,8 +17,8 @@ bencher/example/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
17
17
|
bencher/example/benchmark_data.py,sha256=DL5Grj7UwnKZz2BHfGNKv35Ln0y2ntFwvZdkchOMHVU,6985
|
18
18
|
bencher/example/example_all.py,sha256=iiKV2poYWu4SUIQkpoX4qT1zTm574QfuNHpYww3meFA,1952
|
19
19
|
bencher/example/example_categorical.py,sha256=ydJa-76aZLxxiqzVFkCY3SZNon0yscYhgX41yonoCnU,3685
|
20
|
-
bencher/example/example_composable_container.py,sha256=
|
21
|
-
bencher/example/example_composable_container2.py,sha256=
|
20
|
+
bencher/example/example_composable_container.py,sha256=URN_2Wv98_JpjWJUhbvK2YejpqumUJMTbQ9ok07TNZw,3753
|
21
|
+
bencher/example/example_composable_container2.py,sha256=tVcSggLAzuXsiHM00qSJk9fKOgqXbSzzyjs7IayLZCw,5667
|
22
22
|
bencher/example/example_consts.py,sha256=upKkrMNYUCS38IA4duuyJHERwdZIMB4FA60Gytu_BzU,1475
|
23
23
|
bencher/example/example_custom_sweep.py,sha256=Tl3SJXXlx2_Jko-q8nkeKLfVL21rITI3j4bTrPJVJiA,1917
|
24
24
|
bencher/example/example_custom_sweep2.py,sha256=6RBiyVVaxAh5Aul85DfYA5P-lwhwW3Bb54o0CBgTB6Q,1225
|
@@ -29,19 +29,19 @@ bencher/example/example_float3D.py,sha256=Ya9zIfUZmbqV6GZrRjcV44uMnKeKgG6RrLBsky
|
|
29
29
|
bencher/example/example_float_cat.py,sha256=nQDBWYRVZrJW5ABIizqcD6mXswHWSdEDzM-FeYFqYqY,3821
|
30
30
|
bencher/example/example_floats.py,sha256=HcQgfwldTVeFBmBTMtZ0yRy17ZJ4cfJeI_t8TxY2iOI,4269
|
31
31
|
bencher/example/example_floats2D.py,sha256=D0kljoUCinMKCEW-Zg-cQ8sYu_yPCZqzKJ9tRtt-Ono,3697
|
32
|
-
bencher/example/example_holosweep.py,sha256=
|
32
|
+
bencher/example/example_holosweep.py,sha256=u3w2ixvpx96nNS49sE5j704ZSFwdJVS5HVRsz19kIXc,3021
|
33
33
|
bencher/example/example_holosweep_objects.py,sha256=vHuAtkM1VrJelHOazn_SJfzxNywKyaMzN-DE8W7Ricc,3228
|
34
34
|
bencher/example/example_holosweep_tap.py,sha256=NYXofWGV9GaBN72Q3kKPT5lKJ-slYZH5VzTAavUu23w,4527
|
35
|
-
bencher/example/example_image.py,sha256=
|
36
|
-
bencher/example/example_image1.py,sha256=
|
35
|
+
bencher/example/example_image.py,sha256=UiRzV1cKdtBLxlD16xswro61memxxcy25wteHyMjSkc,5548
|
36
|
+
bencher/example/example_image1.py,sha256=GO-PuiEKcFBma5fVNDi-203ZhYJvP4hojjxsDa9B9HM,2678
|
37
37
|
bencher/example/example_levels.py,sha256=gcNqjdsC0Bi0dX9oakssPEmuh_zFqpJhPYPPoDN3uw8,6890
|
38
38
|
bencher/example/example_levels2.py,sha256=tMhA6dYYQskzMeAZdaz6jRmQTe-c-jLXfkaUqWT30-I,1117
|
39
39
|
bencher/example/example_pareto.py,sha256=sLh37iZSQitW7DE7ktbyaOgHr1ZWxG7pXxCMH00KKqQ,2688
|
40
40
|
bencher/example/example_publish.py,sha256=qIilHdsjqcUQ5u_IuGjjhJCLRkcQvhVuhI3efDrFTww,1005
|
41
41
|
bencher/example/example_rerun.py,sha256=gruldZ6tmW56Mc6bPrNc7ChZEGHhYXPE4VVeG-L8w7M,981
|
42
42
|
bencher/example/example_rerun2.py,sha256=TV8e2UMwLFowhiXb_sQG2IS-W5jXF67VCitVjD8u1o8,660
|
43
|
-
bencher/example/example_sample_cache.py,sha256=
|
44
|
-
bencher/example/example_sample_cache_context.py,sha256=
|
43
|
+
bencher/example/example_sample_cache.py,sha256=sN5AcbX0Ycw1EFVRHWT0k6P3I1yKKDU-cPkZsqW2kGA,4188
|
44
|
+
bencher/example/example_sample_cache_context.py,sha256=v_4hfmwZK38ZqV6QyEfs_I4W3BJX7RO_J-wDaS_DTT0,4062
|
45
45
|
bencher/example/example_simple.py,sha256=E1-D10N-O50S33UQ9iLIlq09-x7BohbjYaR_lzLjQjc,11706
|
46
46
|
bencher/example/example_simple_bool.py,sha256=GZ6pyj8FaQV9gNxaqAmX6c5XWtMvKosezAbSADEl0G0,1248
|
47
47
|
bencher/example/example_simple_cat.py,sha256=XsV_75Jk3phVPI4om3q0vn1POfREb3CGRm9Kq1tL-OA,1760
|
@@ -49,11 +49,11 @@ bencher/example/example_simple_float.py,sha256=5l3O6DzS1AV9LdpN_HfYxR-rvheXoRtWQ
|
|
49
49
|
bencher/example/example_simple_float2d.py,sha256=xsVOLO6AtMi9_fybpS_JZnhev5f11YuYWHrAOzJw2dI,1033
|
50
50
|
bencher/example/example_strings.py,sha256=vStjrvfezNz7115iRtuwy0i7Gbu6w8mu-oHNfKNLNog,1570
|
51
51
|
bencher/example/example_time_event.py,sha256=e6R-a6ZPe-ePiWoNvN3YuSQK-Y2HOGntsjCm_SPon28,2159
|
52
|
-
bencher/example/example_video.py,sha256=
|
52
|
+
bencher/example/example_video.py,sha256=QnEaMOt7taHVKD_t3EZ60aCTkVdt2hhJ2bDH2fNparg,3897
|
53
53
|
bencher/example/example_workflow.py,sha256=00QnUuViMfX_PqzqkXmg1wPX6yAq7IS7mCL_RFKwrMM,6806
|
54
54
|
bencher/example/experimental/example_bokeh_plotly.py,sha256=3jUKh8eKIAlpklKnp8UopIHhUDw1A0_5CwjeyTzbi7o,846
|
55
55
|
bencher/example/experimental/example_hover_ex.py,sha256=qszw4FkIfqQkVviPSpmUoFOoi6PGotGbsc7Ojyx8EtU,1052
|
56
|
-
bencher/example/experimental/example_hvplot_explorer.py,sha256=
|
56
|
+
bencher/example/experimental/example_hvplot_explorer.py,sha256=LqTcIHwEj5DPDgvrHw0nKg_JErlraOrWqR0_6hZXvnI,1819
|
57
57
|
bencher/example/experimental/example_interactive.py,sha256=MM1A2EVsKTy95RERTNnld0tUmZmCy8N41_jGm2wlG7U,2619
|
58
58
|
bencher/example/experimental/example_streamnd.py,sha256=LqkTtdY4NhnP5dEB1Ifv7RQ5Vq4dLkp5E3aWnWuzniA,1414
|
59
59
|
bencher/example/experimental/example_streams.py,sha256=rrTmcmxDlirGoyTbJ4LT4fBIAc1k28qjnjy5JxGKyhg,1030
|
@@ -61,35 +61,35 @@ bencher/example/experimental/example_template.py,sha256=XdIVS9RtLdE5GNnerWiZMXvP
|
|
61
61
|
bencher/example/experimental/example_updates.py,sha256=rF4UgWY-CW6ohNtOpQklTuwbwVRvEM5j6edZOiMkspQ,1835
|
62
62
|
bencher/example/experimental/example_vector.py,sha256=3o_1dA4dc2HL6uIEvDAcvLPVJB8jgkq1QZ3BQIL-LEo,3118
|
63
63
|
bencher/example/inputs_0D/example_0D.py,sha256=WgelKMVahagrLpCByosqh4bHC58BXJ4-AO7zqgjNjH8,1225
|
64
|
-
bencher/example/inputs_1D/example_1D.py,sha256=
|
64
|
+
bencher/example/inputs_1D/example_1D.py,sha256=aT_4mtYNSN49qMG51jgW9X3NSGGC_GrGpgUNG6i0NZ8,1315
|
65
65
|
bencher/example/meta/example_meta.py,sha256=XBpfu47z86vz8QSZjLA1uWltoRZSej7W48nveFKqj5w,5564
|
66
66
|
bencher/example/meta/example_meta_cat.py,sha256=FMBT0yMPJJo0pmUYVtlq64R6qn_EXEt74xYAsK6HQag,641
|
67
67
|
bencher/example/meta/example_meta_float.py,sha256=D71oiFqGauLvqTxv2BC4CJOwHIdpvq8FdCBVejwZ4Do,624
|
68
68
|
bencher/example/meta/example_meta_levels.py,sha256=MkVL8pAIogn8ObKdSn8BC_DKk6PSVvvPU7_KUCgP5vQ,1436
|
69
69
|
bencher/example/optuna/example_optuna.py,sha256=-RIuDrdPjfXz1c1hOAmWeJNdmGICiWnyJfAavRsiMuk,2370
|
70
70
|
bencher/example/shelved/example_float2D_scatter.py,sha256=z8ranMq8IcJ1yoVSFDncp3gw-yWG7X9lXLimXKpy5Ks,3372
|
71
|
-
bencher/example/shelved/example_float3D_cone.py,sha256=
|
71
|
+
bencher/example/shelved/example_float3D_cone.py,sha256=T3dkiEhjm6z3-Vs2SjCNWPKeHk8Bp4FbANE6yXYc_YM,2970
|
72
72
|
bencher/example/shelved/example_kwargs.py,sha256=ahWC3d1vQMS0bdYtZGEILyDGnT5ixR5nVHSnvqMgFL0,2462
|
73
73
|
bencher/plotting/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
74
74
|
bencher/plotting/plot_filter.py,sha256=hWRjZa9zTncVJiF6r_DI4Ce1xcU49PxJw4gXk7AzsnA,4931
|
75
75
|
bencher/plotting/plt_cnt_cfg.py,sha256=0P9KjVQSUfPY7Kh7UGAbTqihaTgnmLm3oZ5Nvf-pcjM,3193
|
76
76
|
bencher/results/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
77
77
|
bencher/results/bench_result.py,sha256=j-Al54h26Qypma0dYbx0hs8lBUUX46xXov7DQAZsG7A,3763
|
78
|
-
bencher/results/bench_result_base.py,sha256=
|
78
|
+
bencher/results/bench_result_base.py,sha256=9woMlOI2elc14SA8XGWOiOEEg7C3hzACelp6NogLCGM,21897
|
79
79
|
bencher/results/dataset_result.py,sha256=qXmFMrVAo_1qM6hhV4XpQqmCz9RiVkQo6ICYmbT-Kvk,8680
|
80
80
|
bencher/results/float_formatter.py,sha256=sX6HNCyaXdHDxC8ybVUHwCJ3qOKbPUkBOplVIHtKWjM,1746
|
81
|
-
bencher/results/holoview_result.py,sha256=
|
81
|
+
bencher/results/holoview_result.py,sha256=9tmVUrP7qJD_5lIURm0QJFRDvFfnZKr_0UGvktx5u5g,29112
|
82
82
|
bencher/results/hvplot_result.py,sha256=bYSewYhPLVzW6HF_WPjAhS1ZiRp9FJHs008UEBXgH4Y,1993
|
83
83
|
bencher/results/optuna_result.py,sha256=QtZ4TGRun7gJoFVUjEyXKPF5yakwOSXrqEXQVJdJmm4,13587
|
84
84
|
bencher/results/panel_result.py,sha256=lXOtfhWKSspf53Wgm94DTiVD3rliieHQW96sOdu5UYk,1336
|
85
85
|
bencher/results/plotly_result.py,sha256=wkgfL38qJp6RviekXBYpNPeU4HCf0nbtKDAhu5QZhUg,2132
|
86
86
|
bencher/results/video_result.py,sha256=E3fAxXctRVxiRyamadpKCMXanM5TTqw1tEYICS2LDLs,1146
|
87
|
-
bencher/results/video_summary.py,sha256
|
87
|
+
bencher/results/video_summary.py,sha256=-YUAkSZhS65a5ULGqyi9vPP-3nhIv9miogM2XLKr0iI,8782
|
88
88
|
bencher/results/composable_container/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
89
89
|
bencher/results/composable_container/composable_container_base.py,sha256=gmlQl3NQ3LWIfH15neLoZMNos6hbu3SklslfcFDAacA,2778
|
90
90
|
bencher/results/composable_container/composable_container_dataframe.py,sha256=ZbFaQSo4UsRxY8NUdJPjNFW3_kzlm8jtWuoLf8y_t8U,1789
|
91
91
|
bencher/results/composable_container/composable_container_panel.py,sha256=HrOoeGB0y0jGQcxcci_M82ftsvklLkJgo-4SjDBJCks,1232
|
92
|
-
bencher/results/composable_container/composable_container_video.py,sha256=
|
92
|
+
bencher/results/composable_container/composable_container_video.py,sha256=X6XxBNDglnNLjQ4QrhxJ8W3Re_aLeTZKKVsbjjJ3av8,7107
|
93
93
|
bencher/variables/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
94
94
|
bencher/variables/inputs.py,sha256=vxpVKhbM8inYiiHcatJLGl0zg9boMYKJRtLVc32YGpY,6730
|
95
95
|
bencher/variables/parametrised_sweep.py,sha256=fxjKOQ2x5xuCyi0kO1_XS9bXiib1bjThhvpulZPeyv8,7802
|
@@ -97,7 +97,7 @@ bencher/variables/results.py,sha256=Wq14e8rAj5mcK22325wcaeTMjgZ6JuduqceAHItHFY8,
|
|
97
97
|
bencher/variables/sweep_base.py,sha256=gfEhKvsb16ZLbe38JewZqu0AMOHpsqwRbZbt-aCg9Bc,6258
|
98
98
|
bencher/variables/time.py,sha256=zcRS5p4ZkFjMta9nZMEuWv86rLnPkUSqyO69QwI5q3E,3142
|
99
99
|
resource/bencher,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
100
|
-
holobench-1.
|
101
|
-
holobench-1.
|
102
|
-
holobench-1.
|
103
|
-
holobench-1.
|
100
|
+
holobench-1.36.1.dist-info/METADATA,sha256=tVg9fYHfBnnb7xlVW_UEikyH2hSOtf2pbrUT1i_K9e4,6600
|
101
|
+
holobench-1.36.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
102
|
+
holobench-1.36.1.dist-info/licenses/LICENSE,sha256=dSHXTdRY4Y7qGFMv63UksV700iff7iE-p7GGs6Sbnvo,1065
|
103
|
+
holobench-1.36.1.dist-info/RECORD,,
|
File without changes
|
File without changes
|