holobench 1.18.0__py2.py3-none-any.whl → 1.30.0__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bencher/__init__.py +13 -1
- bencher/bench_cfg.py +1 -1
- bencher/bench_report.py +6 -109
- bencher/bench_runner.py +1 -1
- bencher/bencher.py +117 -62
- bencher/example/benchmark_data.py +0 -4
- bencher/example/example_composable_container.py +106 -0
- bencher/example/example_composable_container2.py +160 -0
- bencher/example/example_consts.py +39 -0
- bencher/example/example_custom_sweep2.py +42 -0
- bencher/example/example_dataframe.py +48 -0
- bencher/example/example_filepath.py +27 -0
- bencher/example/example_image.py +31 -16
- bencher/example/example_image1.py +81 -0
- bencher/example/example_levels2.py +37 -0
- bencher/example/example_simple_float.py +15 -25
- bencher/example/example_simple_float2d.py +29 -0
- bencher/example/example_strings.py +3 -2
- bencher/example/example_video.py +2 -11
- bencher/example/meta/example_meta.py +2 -2
- bencher/example/meta/example_meta_cat.py +2 -2
- bencher/example/meta/example_meta_float.py +1 -1
- bencher/example/meta/example_meta_levels.py +2 -2
- bencher/optuna_conversions.py +3 -2
- bencher/plotting/plt_cnt_cfg.py +1 -0
- bencher/results/bench_result.py +3 -1
- bencher/results/bench_result_base.py +65 -8
- bencher/results/composable_container/composable_container_base.py +25 -12
- bencher/results/composable_container/composable_container_dataframe.py +52 -0
- bencher/results/composable_container/composable_container_panel.py +17 -18
- bencher/results/composable_container/composable_container_video.py +163 -55
- bencher/results/dataset_result.py +227 -0
- bencher/results/holoview_result.py +15 -7
- bencher/results/optuna_result.py +4 -3
- bencher/results/panel_result.py +1 -3
- bencher/results/video_summary.py +104 -99
- bencher/utils.py +29 -3
- bencher/variables/__init__.py +0 -0
- bencher/variables/inputs.py +24 -1
- bencher/variables/parametrised_sweep.py +8 -24
- bencher/variables/results.py +67 -9
- bencher/variables/time.py +22 -0
- bencher/video_writer.py +20 -74
- {holobench-1.18.0.dist-info → holobench-1.30.0.dist-info}/METADATA +77 -35
- {holobench-1.18.0.dist-info → holobench-1.30.0.dist-info}/RECORD +48 -34
- {holobench-1.18.0.dist-info → holobench-1.30.0.dist-info}/WHEEL +1 -1
- holobench-1.30.0.dist-info/licenses/LICENSE +21 -0
- resource/bencher +0 -0
bencher/__init__.py
CHANGED
@@ -6,11 +6,12 @@ from .variables.sweep_base import hash_sha1
|
|
6
6
|
from .variables.inputs import IntSweep, FloatSweep, StringSweep, EnumSweep, BoolSweep, SweepBase
|
7
7
|
from .variables.time import TimeSnapshot
|
8
8
|
|
9
|
-
from .variables.inputs import box
|
9
|
+
from .variables.inputs import box, p
|
10
10
|
from .variables.results import (
|
11
11
|
ResultVar,
|
12
12
|
ResultVec,
|
13
13
|
ResultHmap,
|
14
|
+
ResultPath,
|
14
15
|
ResultVideo,
|
15
16
|
ResultImage,
|
16
17
|
ResultString,
|
@@ -18,9 +19,19 @@ from .variables.results import (
|
|
18
19
|
ResultReference,
|
19
20
|
ResultVolume,
|
20
21
|
OptDir,
|
22
|
+
ResultDataSet,
|
21
23
|
curve,
|
22
24
|
)
|
23
25
|
|
26
|
+
from .results.composable_container.composable_container_base import (
|
27
|
+
ComposeType,
|
28
|
+
ComposableContainerBase,
|
29
|
+
)
|
30
|
+
from .results.composable_container.composable_container_video import (
|
31
|
+
ComposableContainerVideo,
|
32
|
+
RenderCfg,
|
33
|
+
)
|
34
|
+
|
24
35
|
from .plotting.plot_filter import VarRange, PlotFilter
|
25
36
|
from .utils import (
|
26
37
|
hmap_canonical_input,
|
@@ -30,6 +41,7 @@ from .utils import (
|
|
30
41
|
gen_image_path,
|
31
42
|
gen_video_path,
|
32
43
|
lerp,
|
44
|
+
tabs_in_markdown,
|
33
45
|
)
|
34
46
|
from .variables.parametrised_sweep import ParametrizedSweep
|
35
47
|
from .caching import CachedParams
|
bencher/bench_cfg.py
CHANGED
@@ -383,7 +383,7 @@ class BenchCfg(BenchRunCfg):
|
|
383
383
|
benchmark_sampling_str.append(f" use_cache: {self.use_cache}")
|
384
384
|
benchmark_sampling_str.append(f" use_sample_cache: {self.use_sample_cache}")
|
385
385
|
benchmark_sampling_str.append(f" only_hash_tag: {self.only_hash_tag}")
|
386
|
-
benchmark_sampling_str.append(f"
|
386
|
+
benchmark_sampling_str.append(f" executor: {self.executor}")
|
387
387
|
|
388
388
|
for mv in self.meta_vars:
|
389
389
|
benchmark_sampling_str.extend(describe_variable(mv, True))
|
bencher/bench_report.py
CHANGED
@@ -157,112 +157,9 @@ class BenchReport(BenchPlotServer):
|
|
157
157
|
|
158
158
|
return publish_url
|
159
159
|
|
160
|
-
|
161
|
-
# def
|
162
|
-
#
|
163
|
-
|
164
|
-
#
|
165
|
-
#
|
166
|
-
# # Don't pickle baz
|
167
|
-
# del state["pane"]
|
168
|
-
# return state
|
169
|
-
|
170
|
-
# def __setstate__(self, state):
|
171
|
-
# self.__dict__.update(state)
|
172
|
-
# # Add baz back since it doesn't exist in the pickle
|
173
|
-
# self.report = []
|
174
|
-
|
175
|
-
# def publish_old(
|
176
|
-
# self,
|
177
|
-
# directory: str = "bench_results",
|
178
|
-
# branch_name: str = "bench_results",
|
179
|
-
# url_postprocess: Callable = None,
|
180
|
-
# **kwargs,
|
181
|
-
# ) -> str:
|
182
|
-
# """Publish the results as an html file by committing it to the bench_results branch in the current repo. If you have set up your repo with github pages or equivalent then the html file will be served as a viewable webpage.
|
183
|
-
|
184
|
-
# Args:
|
185
|
-
# directory (str, optional): Directory to save the results. Defaults to "bench_results".
|
186
|
-
# branch_name (str, optional): Branch to publish on. Defaults to "bench_results".
|
187
|
-
# url_postprocess (Callable, optional): A function that maps the origin url to a github pages url. Pass your own function if you are using another git providers. Defaults to None.
|
188
|
-
|
189
|
-
# Returns:
|
190
|
-
# str: _description_
|
191
|
-
# """
|
192
|
-
|
193
|
-
# def get_output(cmd: str) -> str:
|
194
|
-
# return (
|
195
|
-
# subprocess.run(cmd.split(" "), stdout=subprocess.PIPE, check=False)
|
196
|
-
# .stdout.decode("utf=8")
|
197
|
-
# .strip()
|
198
|
-
# )
|
199
|
-
|
200
|
-
# def postprocess_url(publish_url: str, branch_name: str, report_path: str, **kwargs) -> str:
|
201
|
-
# # import re
|
202
|
-
|
203
|
-
# # return re.sub(
|
204
|
-
# # """((git|ssh|http(s)?)|(git@[\w\.-]+))(:(//)?)([\w\.@\:/\-~]+)(\.git)(/)?""",
|
205
|
-
# # """https://$7/""",
|
206
|
-
# # publish_url,
|
207
|
-
# # )
|
208
|
-
# # git@github.com:user/project.git
|
209
|
-
# # https://github.com/user/project.git
|
210
|
-
# # http://github.com/user/project.git
|
211
|
-
# # git@192.168.101.127:user/project.git
|
212
|
-
# # https://192.168.101.127/user/project.git
|
213
|
-
# # http://192.168.101.127/user/project.git
|
214
|
-
# # ssh://user@host.xz:port/path/to/repo.git/
|
215
|
-
# # ssh://user@host.xz/path/to/repo.git/
|
216
|
-
# # ssh://host.xz:port/path/to/repo.git/
|
217
|
-
# # ssh://host.xz/path/to/repo.git/
|
218
|
-
# # ssh://user@host.xz/path/to/repo.git/
|
219
|
-
# # ssh://host.xz/path/to/repo.git/
|
220
|
-
# # ssh://user@host.xz/~user/path/to/repo.git/
|
221
|
-
# # ssh://host.xz/~user/path/to/repo.git/
|
222
|
-
# # ssh://user@host.xz/~/path/to/repo.git
|
223
|
-
# # ssh://host.xz/~/path/to/repo.git
|
224
|
-
# # git://host.xz/path/to/repo.git/
|
225
|
-
# # git://host.xz/~user/path/to/repo.git/
|
226
|
-
# # http://host.xz/path/to/repo.git/
|
227
|
-
# # https://host.xz/path/to/repo.git/
|
228
|
-
# # https://regex101.com/r/qT7NP0/3
|
229
|
-
|
230
|
-
# return publish_url.replace(".git", f"/blob/{directory}/{report_path}")
|
231
|
-
|
232
|
-
# if url_postprocess is None:
|
233
|
-
# url_postprocess = postprocess_url
|
234
|
-
# current_branch = get_output("git symbolic-ref --short HEAD")
|
235
|
-
# logging.info(f"on branch: {current_branch}")
|
236
|
-
# stash_msg = get_output("git stash")
|
237
|
-
# logging.info(f"stashing current work :{stash_msg}")
|
238
|
-
# checkout_msg = get_output(f"git checkout -b {branch_name}")
|
239
|
-
# checkout_msg = get_output(f"git checkout {branch_name}")
|
240
|
-
# get_output("git pull")
|
241
|
-
|
242
|
-
# logging.info(f"checking out branch: {checkout_msg}")
|
243
|
-
# report_path = self.save(directory, in_html_folder=False)
|
244
|
-
# logging.info(f"created report at: {report_path.absolute()}")
|
245
|
-
# # commit_msg = f""
|
246
|
-
# logging.info("adding report to git")
|
247
|
-
# get_output(f"git add {report_path.absolute()}")
|
248
|
-
# get_output("git status")
|
249
|
-
# logging.info("committing report")
|
250
|
-
# cmd = f'git commit -m "generate_report:{self.bench_name}"'
|
251
|
-
# logging.info(cmd)
|
252
|
-
# get_output(cmd)
|
253
|
-
# logging.info("pushing report to origin")
|
254
|
-
# get_output(f"git push --set-upstream origin {branch_name}")
|
255
|
-
# logging.info("checking out original branch")
|
256
|
-
# get_output(f"git checkout {current_branch}")
|
257
|
-
# if "No local changes" not in stash_msg:
|
258
|
-
# logging.info("restoring work with git stash pop")
|
259
|
-
# get_output("git stash pop")
|
260
|
-
|
261
|
-
# publish_url = get_output("git remote get-url --push origin")
|
262
|
-
# logging.info(f"raw url:{publish_url}")
|
263
|
-
# publish_url = url_postprocess(
|
264
|
-
# publish_url, branch_name=branch_name, report_path=report_path, **kwargs
|
265
|
-
# )
|
266
|
-
# logging.info("Published report @")
|
267
|
-
# logging.info(publish_url)
|
268
|
-
# return publish_url
|
160
|
+
# @staticmethod
|
161
|
+
# def publish_github(github_user: str, repo_name: str, branch_name: str) -> Tuple[str, str]:
|
162
|
+
# return (
|
163
|
+
# f"https://github.com/{github_user}/{repo_name}.git",
|
164
|
+
# f"https://github.com/{github_user}/{repo_name}/blob/{branch_name}",
|
165
|
+
# )
|
bencher/bench_runner.py
CHANGED
@@ -121,7 +121,7 @@ class BenchRunner:
|
|
121
121
|
self.show_publish(report_level, show, publish, save, debug)
|
122
122
|
return self.results
|
123
123
|
|
124
|
-
def show_publish(self, report, show, publish, save, debug):
|
124
|
+
def show_publish(self, report: BenchReport, show: bool, publish: bool, save: bool, debug: bool):
|
125
125
|
if save:
|
126
126
|
report.save_index()
|
127
127
|
if publish and self.publisher is not None:
|
bencher/bencher.py
CHANGED
@@ -2,7 +2,7 @@ import logging
|
|
2
2
|
from datetime import datetime
|
3
3
|
from itertools import product, combinations
|
4
4
|
|
5
|
-
from typing import Callable, List
|
5
|
+
from typing import Callable, List, Optional
|
6
6
|
from copy import deepcopy
|
7
7
|
import numpy as np
|
8
8
|
import param
|
@@ -24,11 +24,13 @@ from bencher.variables.results import (
|
|
24
24
|
ResultVar,
|
25
25
|
ResultVec,
|
26
26
|
ResultHmap,
|
27
|
+
ResultPath,
|
27
28
|
ResultVideo,
|
28
29
|
ResultImage,
|
29
30
|
ResultString,
|
30
31
|
ResultContainer,
|
31
32
|
ResultReference,
|
33
|
+
ResultDataSet,
|
32
34
|
)
|
33
35
|
from bencher.results.bench_result import BenchResult
|
34
36
|
from bencher.variables.parametrised_sweep import ParametrizedSweep
|
@@ -219,9 +221,11 @@ class Bench(BenchPlotServer):
|
|
219
221
|
relationship_cb=None,
|
220
222
|
plot_callbacks: List | bool = None,
|
221
223
|
) -> List[BenchResult]:
|
222
|
-
results = []
|
223
224
|
if relationship_cb is None:
|
224
225
|
relationship_cb = combinations
|
226
|
+
if input_vars is None:
|
227
|
+
input_vars = self.worker_class_instance.get_inputs_only()
|
228
|
+
results = []
|
225
229
|
for it in range(iterations):
|
226
230
|
for input_group in relationship_cb(input_vars, group_size):
|
227
231
|
title_gen = title + "Sweeping " + " vs ".join(params_to_str(input_group))
|
@@ -277,57 +281,42 @@ class Bench(BenchPlotServer):
|
|
277
281
|
BenchResult: A class with all the data used to generate the results and the results
|
278
282
|
"""
|
279
283
|
|
284
|
+
input_vars_in = deepcopy(input_vars)
|
285
|
+
result_vars_in = deepcopy(result_vars)
|
286
|
+
const_vars_in = deepcopy(const_vars)
|
287
|
+
|
280
288
|
if self.worker_class_instance is not None:
|
281
|
-
if
|
289
|
+
if input_vars_in is None:
|
282
290
|
logging.info(
|
283
291
|
"No input variables passed, using all param variables in bench class as inputs"
|
284
292
|
)
|
285
293
|
if self.input_vars is None:
|
286
|
-
|
294
|
+
input_vars_in = self.worker_class_instance.get_inputs_only()
|
287
295
|
else:
|
288
|
-
|
289
|
-
for i in
|
296
|
+
input_vars_in = deepcopy(self.input_vars)
|
297
|
+
for i in input_vars_in:
|
290
298
|
logging.info(f"input var: {i.name}")
|
291
|
-
if
|
299
|
+
if result_vars_in is None:
|
292
300
|
logging.info(
|
293
301
|
"No results variables passed, using all result variables in bench class:"
|
294
302
|
)
|
295
303
|
if self.result_vars is None:
|
296
|
-
|
304
|
+
result_vars_in = self.worker_class_instance.get_results_only()
|
297
305
|
else:
|
298
|
-
|
306
|
+
result_vars_in = deepcopy(self.result_vars)
|
299
307
|
|
300
|
-
if
|
308
|
+
if const_vars_in is None:
|
301
309
|
if self.const_vars is None:
|
302
|
-
|
310
|
+
const_vars_in = self.worker_class_instance.get_input_defaults()
|
303
311
|
else:
|
304
|
-
|
312
|
+
const_vars_in = deepcopy(self.const_vars)
|
305
313
|
else:
|
306
|
-
if
|
307
|
-
|
308
|
-
if
|
309
|
-
|
310
|
-
if
|
311
|
-
|
312
|
-
else:
|
313
|
-
const_vars = deepcopy(const_vars)
|
314
|
-
|
315
|
-
for i in range(len(input_vars)):
|
316
|
-
input_vars[i] = self.convert_vars_to_params(input_vars[i], "input")
|
317
|
-
for i in range(len(result_vars)):
|
318
|
-
result_vars[i] = self.convert_vars_to_params(result_vars[i], "result")
|
319
|
-
|
320
|
-
for r in result_vars:
|
321
|
-
logging.info(f"result var: {r.name}")
|
322
|
-
|
323
|
-
if isinstance(const_vars, dict):
|
324
|
-
const_vars = list(const_vars.items())
|
325
|
-
|
326
|
-
for i in range(len(const_vars)):
|
327
|
-
# consts come as tuple pairs
|
328
|
-
cv_list = list(const_vars[i])
|
329
|
-
cv_list[0] = self.convert_vars_to_params(cv_list[0], "const")
|
330
|
-
const_vars[i] = cv_list
|
314
|
+
if input_vars_in is None:
|
315
|
+
input_vars_in = []
|
316
|
+
if result_vars_in is None:
|
317
|
+
result_vars_in = []
|
318
|
+
if const_vars_in is None:
|
319
|
+
const_vars_in = []
|
331
320
|
|
332
321
|
if run_cfg is None:
|
333
322
|
if self.run_cfg is None:
|
@@ -342,37 +331,68 @@ class Bench(BenchPlotServer):
|
|
342
331
|
|
343
332
|
self.last_run_cfg = run_cfg
|
344
333
|
|
334
|
+
if isinstance(input_vars_in, dict):
|
335
|
+
input_lists = []
|
336
|
+
for k, v in input_vars_in.items():
|
337
|
+
param_var = self.convert_vars_to_params(k, "input", run_cfg)
|
338
|
+
if isinstance(v, list):
|
339
|
+
assert len(v) > 0
|
340
|
+
param_var = param_var.with_sample_values(v)
|
341
|
+
|
342
|
+
else:
|
343
|
+
raise RuntimeError("Unsupported type")
|
344
|
+
input_lists.append(param_var)
|
345
|
+
|
346
|
+
input_vars_in = input_lists
|
347
|
+
else:
|
348
|
+
for i in range(len(input_vars_in)):
|
349
|
+
input_vars_in[i] = self.convert_vars_to_params(input_vars_in[i], "input", run_cfg)
|
350
|
+
for i in range(len(result_vars_in)):
|
351
|
+
result_vars_in[i] = self.convert_vars_to_params(result_vars_in[i], "result", run_cfg)
|
352
|
+
|
353
|
+
for r in result_vars_in:
|
354
|
+
logging.info(f"result var: {r.name}")
|
355
|
+
|
356
|
+
if isinstance(const_vars_in, dict):
|
357
|
+
const_vars_in = list(const_vars_in.items())
|
358
|
+
|
359
|
+
for i in range(len(const_vars_in)):
|
360
|
+
# consts come as tuple pairs
|
361
|
+
cv_list = list(const_vars_in[i])
|
362
|
+
cv_list[0] = self.convert_vars_to_params(cv_list[0], "const", run_cfg)
|
363
|
+
const_vars_in[i] = cv_list
|
364
|
+
|
345
365
|
if title is None:
|
346
|
-
if len(
|
347
|
-
title = "Sweeping " + " vs ".join([i.name for i in
|
348
|
-
elif len(
|
366
|
+
if len(input_vars_in) > 0:
|
367
|
+
title = "Sweeping " + " vs ".join([i.name for i in input_vars_in])
|
368
|
+
elif len(const_vars_in) > 0:
|
349
369
|
title = "Constant Value"
|
350
|
-
if len(
|
370
|
+
if len(const_vars_in) > 1:
|
351
371
|
title += "s"
|
352
|
-
title += ": " + ", ".join([f"{c[0].name}={c[1]}" for c in
|
372
|
+
title += ": " + ", ".join([f"{c[0].name}={c[1]}" for c in const_vars_in])
|
353
373
|
else:
|
354
374
|
raise RuntimeError("you must pass a title, or define inputs or consts")
|
355
375
|
|
356
376
|
if run_cfg.level > 0:
|
357
377
|
inputs = []
|
358
|
-
print(
|
359
|
-
if len(
|
360
|
-
for i in
|
378
|
+
print(input_vars_in)
|
379
|
+
if len(input_vars_in) > 0:
|
380
|
+
for i in input_vars_in:
|
361
381
|
inputs.append(i.with_level(run_cfg.level))
|
362
|
-
|
382
|
+
input_vars_in = inputs
|
363
383
|
|
364
384
|
# if any of the inputs have been include as constants, remove those variables from the list of constants
|
365
385
|
with suppress(ValueError, AttributeError):
|
366
|
-
for i in
|
367
|
-
for c in
|
386
|
+
for i in input_vars_in:
|
387
|
+
for c in const_vars_in:
|
368
388
|
# print(i.hash_persistent())
|
369
389
|
if i.name == c[0].name:
|
370
|
-
|
390
|
+
const_vars_in.remove(c)
|
371
391
|
logging.info(f"removing {i.name} from constants")
|
372
392
|
|
373
393
|
result_hmaps = []
|
374
394
|
result_vars_only = []
|
375
|
-
for i in
|
395
|
+
for i in result_vars_in:
|
376
396
|
if isinstance(i, ResultHmap):
|
377
397
|
result_hmaps.append(i)
|
378
398
|
else:
|
@@ -392,10 +412,10 @@ class Bench(BenchPlotServer):
|
|
392
412
|
plot_callbacks = [BenchResult.to_auto_plots] if plot_callbacks else []
|
393
413
|
|
394
414
|
bench_cfg = BenchCfg(
|
395
|
-
input_vars=
|
415
|
+
input_vars=input_vars_in,
|
396
416
|
result_vars=result_vars_only,
|
397
417
|
result_hmaps=result_hmaps,
|
398
|
-
const_vars=
|
418
|
+
const_vars=const_vars_in,
|
399
419
|
bench_name=self.bench_name,
|
400
420
|
description=description,
|
401
421
|
post_description=post_description,
|
@@ -469,7 +489,12 @@ class Bench(BenchPlotServer):
|
|
469
489
|
self.results.append(bench_res)
|
470
490
|
return bench_res
|
471
491
|
|
472
|
-
def convert_vars_to_params(
|
492
|
+
def convert_vars_to_params(
|
493
|
+
self,
|
494
|
+
variable: param.Parameter | str | dict | tuple,
|
495
|
+
var_type: str,
|
496
|
+
run_cfg: Optional[BenchRunCfg],
|
497
|
+
) -> param.Parameter:
|
473
498
|
"""check that a variable is a subclass of param
|
474
499
|
|
475
500
|
Args:
|
@@ -481,6 +506,17 @@ class Bench(BenchPlotServer):
|
|
481
506
|
"""
|
482
507
|
if isinstance(variable, str):
|
483
508
|
variable = self.worker_class_instance.param.objects(instance=False)[variable]
|
509
|
+
if isinstance(variable, dict):
|
510
|
+
param_var = self.worker_class_instance.param.objects(instance=False)[variable["name"]]
|
511
|
+
if variable.get("values"):
|
512
|
+
param_var = param_var.with_sample_values(variable["values"])
|
513
|
+
|
514
|
+
if variable.get("samples"):
|
515
|
+
param_var = param_var.with_samples(variable["samples"])
|
516
|
+
if variable.get("max_level"):
|
517
|
+
if run_cfg is not None:
|
518
|
+
param_var = param_var.with_level(run_cfg.level, variable["max_level"])
|
519
|
+
variable = param_var
|
484
520
|
if not isinstance(variable, param.Parameter):
|
485
521
|
raise TypeError(
|
486
522
|
f"You need to use {var_type}_vars =[{self.worker_input_cfg}.param.your_variable], instead of {var_type}_vars =[{self.worker_input_cfg}.your_variable]"
|
@@ -557,7 +593,7 @@ class Bench(BenchPlotServer):
|
|
557
593
|
time_src (datetime | str): a representation of the sample time
|
558
594
|
|
559
595
|
Returns:
|
560
|
-
|
596
|
+
tuple[BenchResult, List, List]: bench_result, function intputs, dimension names
|
561
597
|
"""
|
562
598
|
|
563
599
|
if time_src is None:
|
@@ -565,9 +601,7 @@ class Bench(BenchPlotServer):
|
|
565
601
|
bench_cfg.meta_vars = self.define_extra_vars(bench_cfg, bench_cfg.repeats, time_src)
|
566
602
|
|
567
603
|
bench_cfg.all_vars = bench_cfg.input_vars + bench_cfg.meta_vars
|
568
|
-
|
569
604
|
# bench_cfg.all_vars = bench_cfg.iv_time + bench_cfg.input_vars +[ bench_cfg.iv_repeat]
|
570
|
-
|
571
605
|
# bench_cfg.all_vars = [ bench_cfg.iv_repeat] +bench_cfg.input_vars + bench_cfg.iv_time
|
572
606
|
|
573
607
|
for i in bench_cfg.all_vars:
|
@@ -579,18 +613,22 @@ class Bench(BenchPlotServer):
|
|
579
613
|
)
|
580
614
|
# xarray stores K N-dimensional arrays of data. Each array is named and in this case we have a nd array for each result variable
|
581
615
|
data_vars = {}
|
616
|
+
dataset_list = []
|
582
617
|
|
583
618
|
for rv in bench_cfg.result_vars:
|
584
619
|
if isinstance(rv, ResultVar):
|
585
620
|
result_data = np.full(dims_cfg.dims_size, np.nan, dtype=float)
|
586
621
|
data_vars[rv.name] = (dims_cfg.dims_name, result_data)
|
587
|
-
if isinstance(rv, ResultReference):
|
622
|
+
if isinstance(rv, (ResultReference, ResultDataSet)):
|
588
623
|
result_data = np.full(dims_cfg.dims_size, -1, dtype=int)
|
589
624
|
data_vars[rv.name] = (dims_cfg.dims_name, result_data)
|
590
|
-
if isinstance(
|
625
|
+
if isinstance(
|
626
|
+
rv, (ResultPath, ResultVideo, ResultImage, ResultString, ResultContainer)
|
627
|
+
):
|
591
628
|
result_data = np.full(dims_cfg.dims_size, "NAN", dtype=object)
|
592
629
|
data_vars[rv.name] = (dims_cfg.dims_name, result_data)
|
593
|
-
|
630
|
+
|
631
|
+
elif type(rv) is ResultVec:
|
594
632
|
for i in range(rv.size):
|
595
633
|
result_data = np.full(dims_cfg.dims_size, np.nan)
|
596
634
|
data_vars[rv.index_name(i)] = (dims_cfg.dims_name, result_data)
|
@@ -598,6 +636,7 @@ class Bench(BenchPlotServer):
|
|
598
636
|
bench_res = BenchResult(bench_cfg)
|
599
637
|
bench_res.ds = xr.Dataset(data_vars=data_vars, coords=dims_cfg.coords)
|
600
638
|
bench_res.ds_dynamic = self.ds_dynamic
|
639
|
+
bench_res.dataset_list = dataset_list
|
601
640
|
bench_res.setup_object_index()
|
602
641
|
|
603
642
|
return bench_res, function_inputs, dims_cfg.dims_name
|
@@ -724,9 +763,24 @@ class Bench(BenchPlotServer):
|
|
724
763
|
logging.info(f"{rv.name}: {result_value}")
|
725
764
|
|
726
765
|
if isinstance(
|
727
|
-
rv,
|
766
|
+
rv,
|
767
|
+
(
|
768
|
+
ResultVar,
|
769
|
+
ResultVideo,
|
770
|
+
ResultImage,
|
771
|
+
ResultString,
|
772
|
+
ResultContainer,
|
773
|
+
ResultPath,
|
774
|
+
),
|
728
775
|
):
|
729
776
|
set_xarray_multidim(bench_res.ds[rv.name], worker_job.index_tuple, result_value)
|
777
|
+
elif isinstance(rv, ResultDataSet):
|
778
|
+
bench_res.dataset_list.append(result_value)
|
779
|
+
set_xarray_multidim(
|
780
|
+
bench_res.ds[rv.name],
|
781
|
+
worker_job.index_tuple,
|
782
|
+
len(bench_res.dataset_list) - 1,
|
783
|
+
)
|
730
784
|
elif isinstance(rv, ResultReference):
|
731
785
|
bench_res.object_index.append(result_value)
|
732
786
|
set_xarray_multidim(
|
@@ -734,6 +788,7 @@ class Bench(BenchPlotServer):
|
|
734
788
|
worker_job.index_tuple,
|
735
789
|
len(bench_res.object_index) - 1,
|
736
790
|
)
|
791
|
+
|
737
792
|
elif isinstance(rv, ResultVec):
|
738
793
|
if isinstance(result_value, (list, np.ndarray)):
|
739
794
|
if len(result_value) == rv.size:
|
@@ -779,10 +834,10 @@ class Bench(BenchPlotServer):
|
|
779
834
|
"""
|
780
835
|
|
781
836
|
for rv in bench_res.bench_cfg.result_vars:
|
782
|
-
if type(rv)
|
837
|
+
if type(rv) is ResultVar:
|
783
838
|
bench_res.ds[rv.name].attrs["units"] = rv.units
|
784
839
|
bench_res.ds[rv.name].attrs["long_name"] = rv.name
|
785
|
-
elif type(rv)
|
840
|
+
elif type(rv) is ResultVec:
|
786
841
|
for i in range(rv.size):
|
787
842
|
bench_res.ds[rv.index_name(i)].attrs["units"] = rv.units
|
788
843
|
bench_res.ds[rv.index_name(i)].attrs["long_name"] = rv.name
|
@@ -6,13 +6,9 @@ You can define a subclass which contains an input configuration which can be pas
|
|
6
6
|
import math
|
7
7
|
import random
|
8
8
|
from enum import auto
|
9
|
-
|
10
9
|
from strenum import StrEnum
|
11
|
-
|
12
|
-
|
13
10
|
from bencher.variables.inputs import IntSweep, FloatSweep, StringSweep, EnumSweep, BoolSweep
|
14
11
|
from bencher.variables.results import ResultVar, OptDir
|
15
|
-
|
16
12
|
from bencher.variables.parametrised_sweep import ParametrizedSweep
|
17
13
|
|
18
14
|
|
@@ -0,0 +1,106 @@
|
|
1
|
+
import bencher as bch
|
2
|
+
|
3
|
+
from bencher.example.example_image import BenchPolygons
|
4
|
+
|
5
|
+
|
6
|
+
class BenchComposableContainerImage(BenchPolygons):
|
7
|
+
compose_method = bch.EnumSweep(bch.ComposeType)
|
8
|
+
labels = bch.BoolSweep()
|
9
|
+
num_frames = bch.IntSweep(default=5, bounds=[1, 100])
|
10
|
+
polygon_vid = bch.ResultVideo()
|
11
|
+
|
12
|
+
def __call__(self, **kwargs):
|
13
|
+
self.update_params_from_kwargs(**kwargs)
|
14
|
+
var_name = None
|
15
|
+
var_value = None
|
16
|
+
|
17
|
+
if self.labels:
|
18
|
+
var_name = "sides"
|
19
|
+
var_value = self.sides
|
20
|
+
vr = bch.ComposableContainerVideo()
|
21
|
+
for i in range(self.num_frames):
|
22
|
+
res = super().__call__(start_angle=i)
|
23
|
+
print(res)
|
24
|
+
vr.append(res["polygon"])
|
25
|
+
self.polygon_vid = vr.to_video(
|
26
|
+
bch.RenderCfg(
|
27
|
+
compose_method=self.compose_method,
|
28
|
+
var_name=var_name,
|
29
|
+
var_value=var_value,
|
30
|
+
max_frame_duration=1.0 / 20.0,
|
31
|
+
)
|
32
|
+
)
|
33
|
+
return self.get_results_values_as_dict()
|
34
|
+
|
35
|
+
|
36
|
+
class BenchComposableContainerVideo(bch.ParametrizedSweep):
|
37
|
+
unequal_length = bch.BoolSweep()
|
38
|
+
compose_method = bch.EnumSweep(bch.ComposeType)
|
39
|
+
labels = bch.BoolSweep()
|
40
|
+
polygon_vid = bch.ResultVideo()
|
41
|
+
|
42
|
+
def __call__(self, **kwargs):
|
43
|
+
self.update_params_from_kwargs(**kwargs)
|
44
|
+
vr = bch.ComposableContainerVideo()
|
45
|
+
for i in range(3, 5):
|
46
|
+
num_frames = i * 10 if self.unequal_length else 5
|
47
|
+
res = BenchComposableContainerImage().__call__(
|
48
|
+
compose_method=bch.ComposeType.sequence, sides=i, num_frames=num_frames
|
49
|
+
)
|
50
|
+
vr.append(res["polygon_vid"])
|
51
|
+
|
52
|
+
self.polygon_vid = vr.to_video(bch.RenderCfg(compose_method=kwargs.get("compose_method")))
|
53
|
+
return self.get_results_values_as_dict()
|
54
|
+
|
55
|
+
|
56
|
+
def example_composable_container_image(
|
57
|
+
run_cfg: bch.BenchRunCfg = None, report: bch.BenchReport = None
|
58
|
+
) -> bch.Bench:
|
59
|
+
bench = BenchComposableContainerImage().to_bench(run_cfg, report)
|
60
|
+
bench.result_vars = ["polygon_vid"]
|
61
|
+
# bench.add_plot_callback(bch.BenchResult.to_panes)
|
62
|
+
# bench.add_plot_callback(bch.BenchResult.to_video_grid, result_types=(bch.ResultVideo))
|
63
|
+
# bench.add_plot_callback(bch.BenchResult.to_video_summary, result_types=(bch.ResultVideo))
|
64
|
+
# bench.plot_sweep(input_vars=["compose_method", "labels"])
|
65
|
+
|
66
|
+
bench.plot_sweep(input_vars=["compose_method"])
|
67
|
+
|
68
|
+
# bench.compose_
|
69
|
+
# bench.plot_sweep(
|
70
|
+
# input_vars=[bch.p("num_frames", [2, 8, 20])],
|
71
|
+
# const_vars=dict(compose_method=bch.ComposeType.sequence),
|
72
|
+
# )
|
73
|
+
|
74
|
+
return bench
|
75
|
+
|
76
|
+
|
77
|
+
def example_composable_container_video(
|
78
|
+
run_cfg: bch.BenchRunCfg = None, report: bch.BenchReport = None
|
79
|
+
) -> bch.Bench:
|
80
|
+
bench = BenchComposableContainerVideo().to_bench(run_cfg, report)
|
81
|
+
|
82
|
+
bench.result_vars = ["polygon_vid"]
|
83
|
+
bench.add_plot_callback(bch.BenchResult.to_panes)
|
84
|
+
bench.add_plot_callback(bch.BenchResult.to_video_grid, result_types=(bch.ResultVideo))
|
85
|
+
bench.add_plot_callback(bch.BenchResult.to_video_summary, result_types=(bch.ResultVideo))
|
86
|
+
bench.plot_sweep(input_vars=["compose_method", "labels"], const_vars=dict(unequal_length=True))
|
87
|
+
|
88
|
+
res = bench.plot_sweep(
|
89
|
+
input_vars=[],
|
90
|
+
const_vars=dict(unequal_length=False, compose_method=bch.ComposeType.sequence),
|
91
|
+
plot_callbacks=False,
|
92
|
+
)
|
93
|
+
|
94
|
+
bench.report.append(res.to_video_grid())
|
95
|
+
|
96
|
+
return bench
|
97
|
+
|
98
|
+
|
99
|
+
if __name__ == "__main__":
|
100
|
+
ex_run_cfg = bch.BenchRunCfg()
|
101
|
+
ex_run_cfg.use_sample_cache = False
|
102
|
+
# ex_run_cfg.level = 2
|
103
|
+
ex_report = bch.BenchReport()
|
104
|
+
example_composable_container_image(ex_run_cfg, report=ex_report)
|
105
|
+
# example_composable_container_video(ex_run_cfg, report=ex_report)
|
106
|
+
ex_report.show()
|