redisbench-admin 0.11.19__py3-none-any.whl → 0.11.22__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- redisbench_admin/compare/args.py +19 -3
- redisbench_admin/compare/compare.py +42 -45
- redisbench_admin/deploy/deploy.py +10 -5
- redisbench_admin/export/export.py +7 -1
- redisbench_admin/profilers/perf.py +24 -24
- redisbench_admin/run/args.py +11 -0
- redisbench_admin/run/common.py +57 -32
- redisbench_admin/run/ftsb/ftsb.py +0 -1
- redisbench_admin/run/ssh.py +0 -15
- redisbench_admin/run_async/async_terraform.py +10 -2
- redisbench_admin/run_async/render_files.py +3 -3
- redisbench_admin/run_local/args.py +2 -0
- redisbench_admin/run_local/local_db.py +6 -2
- redisbench_admin/run_local/run_local.py +18 -12
- redisbench_admin/run_remote/remote_db.py +1 -0
- redisbench_admin/run_remote/remote_env.py +12 -0
- redisbench_admin/run_remote/remote_helpers.py +0 -1
- redisbench_admin/run_remote/run_remote.py +25 -18
- redisbench_admin/run_remote/standalone.py +5 -6
- redisbench_admin/run_remote/terraform.py +5 -1
- redisbench_admin/utils/benchmark_config.py +0 -8
- redisbench_admin/utils/remote.py +19 -56
- redisbench_admin/utils/utils.py +13 -2
- redisbench_admin/watchdog/watchdog.py +8 -9
- {redisbench_admin-0.11.19.dist-info → redisbench_admin-0.11.22.dist-info}/METADATA +2 -3
- {redisbench_admin-0.11.19.dist-info → redisbench_admin-0.11.22.dist-info}/RECORD +29 -29
- {redisbench_admin-0.11.19.dist-info → redisbench_admin-0.11.22.dist-info}/LICENSE +0 -0
- {redisbench_admin-0.11.19.dist-info → redisbench_admin-0.11.22.dist-info}/WHEEL +0 -0
- {redisbench_admin-0.11.19.dist-info → redisbench_admin-0.11.22.dist-info}/entry_points.txt +0 -0
redisbench_admin/compare/args.py
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
|
|
7
7
|
# environment variables
|
|
8
8
|
import datetime
|
|
9
|
-
|
|
9
|
+
import os
|
|
10
10
|
from redisbench_admin.run.common import get_start_time_vars, PERFORMANCE_GH_TOKEN
|
|
11
11
|
from redisbench_admin.utils.remote import (
|
|
12
12
|
PERFORMANCE_RTS_HOST,
|
|
@@ -29,6 +29,10 @@ _, NOW_UTC, _ = get_start_time_vars()
|
|
|
29
29
|
LAST_MONTH_UTC = NOW_UTC - (31 * 24 * 60 * 60 * 1000)
|
|
30
30
|
START_TIME_NOW_UTC, _, _ = get_start_time_vars()
|
|
31
31
|
START_TIME_LAST_MONTH_UTC = START_TIME_NOW_UTC - datetime.timedelta(days=30)
|
|
32
|
+
ARCH_X86 = "x86_64"
|
|
33
|
+
ARCH_ARM = "aarch64"
|
|
34
|
+
VALID_ARCHS = [ARCH_X86, ARCH_ARM]
|
|
35
|
+
ARCH = os.getenv("ARCH", ARCH_X86)
|
|
32
36
|
|
|
33
37
|
|
|
34
38
|
def create_compare_arguments(parser):
|
|
@@ -53,11 +57,23 @@ def create_compare_arguments(parser):
|
|
|
53
57
|
parser.add_argument("--deployment_type", type=str, default="oss-standalone")
|
|
54
58
|
parser.add_argument("--baseline_deployment_name", type=str, default="")
|
|
55
59
|
parser.add_argument("--comparison_deployment_name", type=str, default="")
|
|
56
|
-
parser.add_argument("--baseline_github_org", type=str, default=GITHUB_ORG)
|
|
57
|
-
parser.add_argument("--comparison_github_org", type=str, default=GITHUB_ORG)
|
|
58
60
|
parser.add_argument("--metric_name", type=str, default=None)
|
|
59
61
|
parser.add_argument("--running_platform", type=str, default=None)
|
|
60
62
|
parser.add_argument("--extra-filter", type=str, default=None)
|
|
63
|
+
parser.add_argument(
|
|
64
|
+
"--baseline_architecture",
|
|
65
|
+
type=str,
|
|
66
|
+
required=False,
|
|
67
|
+
default=ARCH,
|
|
68
|
+
help=f"Architecture to filter baseline time-series. One of {VALID_ARCHS}.",
|
|
69
|
+
)
|
|
70
|
+
parser.add_argument(
|
|
71
|
+
"--comparison_architecture",
|
|
72
|
+
type=str,
|
|
73
|
+
required=False,
|
|
74
|
+
default=ARCH,
|
|
75
|
+
help=f"Architecture to filter comparison time-series. One of {VALID_ARCHS}.",
|
|
76
|
+
)
|
|
61
77
|
parser.add_argument(
|
|
62
78
|
"--last_n",
|
|
63
79
|
type=int,
|
|
@@ -22,6 +22,7 @@ from redisbench_admin.run_remote.notifications import (
|
|
|
22
22
|
generate_new_pr_comment_notification,
|
|
23
23
|
)
|
|
24
24
|
from redisbench_admin.utils.remote import get_overall_dashboard_keynames
|
|
25
|
+
from redisbench_admin.compare.args import ARCH_X86
|
|
25
26
|
|
|
26
27
|
|
|
27
28
|
def get_project_compare_zsets(triggering_env, org, repo):
|
|
@@ -166,8 +167,6 @@ def compare_command_logic(args, project_name, project_version):
|
|
|
166
167
|
auto_approve = args.auto_approve
|
|
167
168
|
running_platform = args.running_platform
|
|
168
169
|
grafana_base_dashboard = args.grafana_base_dashboard
|
|
169
|
-
baseline_github_org = args.baseline_github_org
|
|
170
|
-
comparison_github_org = args.comparison_github_org
|
|
171
170
|
# using an access token
|
|
172
171
|
is_actionable_pr = False
|
|
173
172
|
contains_regression_comment = False
|
|
@@ -193,38 +192,32 @@ def compare_command_logic(args, project_name, project_version):
|
|
|
193
192
|
logging.info("Detected github token")
|
|
194
193
|
g = Github(github_token)
|
|
195
194
|
if pull_request is not None and pull_request != "":
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
regression_comment.html_url
|
|
214
|
-
)
|
|
195
|
+
pull_request_n = int(pull_request)
|
|
196
|
+
github_pr = (
|
|
197
|
+
g.get_user(tf_github_org)
|
|
198
|
+
.get_repo(tf_github_repo)
|
|
199
|
+
.get_issue(pull_request_n)
|
|
200
|
+
)
|
|
201
|
+
comments = github_pr.get_comments()
|
|
202
|
+
pr_link = github_pr.html_url
|
|
203
|
+
logging.info("Working on github PR already: {}".format(pr_link))
|
|
204
|
+
is_actionable_pr = True
|
|
205
|
+
contains_regression_comment, pos = check_regression_comment(comments)
|
|
206
|
+
if contains_regression_comment:
|
|
207
|
+
regression_comment = comments[pos]
|
|
208
|
+
old_regression_comment_body = regression_comment.body
|
|
209
|
+
logging.info(
|
|
210
|
+
"Already contains regression comment. Link: {}".format(
|
|
211
|
+
regression_comment.html_url
|
|
215
212
|
)
|
|
216
|
-
if verbose:
|
|
217
|
-
logging.info("Printing old regression comment:")
|
|
218
|
-
print("".join(["-" for x in range(1, 80)]))
|
|
219
|
-
print(regression_comment.body)
|
|
220
|
-
print("".join(["-" for x in range(1, 80)]))
|
|
221
|
-
else:
|
|
222
|
-
logging.info("Does not contain regression comment")
|
|
223
|
-
|
|
224
|
-
except Exception as e:
|
|
225
|
-
logging.error(
|
|
226
|
-
f"Error while working with github. exception {e.__str__()}"
|
|
227
213
|
)
|
|
214
|
+
if verbose:
|
|
215
|
+
logging.info("Printing old regression comment:")
|
|
216
|
+
print("".join(["-" for x in range(1, 80)]))
|
|
217
|
+
print(regression_comment.body)
|
|
218
|
+
print("".join(["-" for x in range(1, 80)]))
|
|
219
|
+
else:
|
|
220
|
+
logging.info("Does not contain regression comment")
|
|
228
221
|
|
|
229
222
|
grafana_dashboards_uids = {
|
|
230
223
|
"redisgraph": "SH9_rQYGz",
|
|
@@ -233,6 +226,8 @@ def compare_command_logic(args, project_name, project_version):
|
|
|
233
226
|
"redisjson": "UErSC0jGk",
|
|
234
227
|
"redistimeseries": "2WMw61UGz",
|
|
235
228
|
}
|
|
229
|
+
baseline_architecture = args.baseline_architecture
|
|
230
|
+
comparison_architecture = args.comparison_architecture
|
|
236
231
|
uid = None
|
|
237
232
|
if tf_github_repo.lower() in grafana_dashboards_uids:
|
|
238
233
|
uid = grafana_dashboards_uids[tf_github_repo.lower()]
|
|
@@ -282,8 +277,8 @@ def compare_command_logic(args, project_name, project_version):
|
|
|
282
277
|
to_ts_ms,
|
|
283
278
|
use_metric_context_path,
|
|
284
279
|
running_platform,
|
|
285
|
-
|
|
286
|
-
|
|
280
|
+
baseline_architecture,
|
|
281
|
+
comparison_architecture,
|
|
287
282
|
)
|
|
288
283
|
comment_body = ""
|
|
289
284
|
if total_comparison_points > 0:
|
|
@@ -508,8 +503,8 @@ def compute_regression_table(
|
|
|
508
503
|
to_ts_ms=None,
|
|
509
504
|
use_metric_context_path=None,
|
|
510
505
|
running_platform=None,
|
|
511
|
-
|
|
512
|
-
|
|
506
|
+
baseline_architecture=ARCH_X86,
|
|
507
|
+
comparison_architecture=ARCH_X86,
|
|
513
508
|
):
|
|
514
509
|
START_TIME_NOW_UTC, _, _ = get_start_time_vars()
|
|
515
510
|
START_TIME_LAST_MONTH_UTC = START_TIME_NOW_UTC - datetime.timedelta(days=31)
|
|
@@ -595,9 +590,9 @@ def compute_regression_table(
|
|
|
595
590
|
test_names,
|
|
596
591
|
tf_triggering_env,
|
|
597
592
|
verbose,
|
|
598
|
-
baseline_github_org,
|
|
599
|
-
comparison_github_org,
|
|
600
593
|
running_platform,
|
|
594
|
+
baseline_architecture,
|
|
595
|
+
comparison_architecture,
|
|
601
596
|
)
|
|
602
597
|
logging.info(
|
|
603
598
|
"Printing differential analysis between {} and {}".format(
|
|
@@ -667,7 +662,7 @@ def get_by_strings(
|
|
|
667
662
|
comparison_str = comparison_branch
|
|
668
663
|
|
|
669
664
|
if baseline_tag is not None:
|
|
670
|
-
if
|
|
665
|
+
if comparison_covered:
|
|
671
666
|
logging.error(
|
|
672
667
|
"--baseline-branch and --baseline-tag are mutually exclusive. Pick one..."
|
|
673
668
|
)
|
|
@@ -724,9 +719,9 @@ def from_rts_to_regression_table(
|
|
|
724
719
|
test_names,
|
|
725
720
|
tf_triggering_env,
|
|
726
721
|
verbose,
|
|
727
|
-
baseline_github_org="",
|
|
728
|
-
comparison_github_org="",
|
|
729
722
|
running_platform=None,
|
|
723
|
+
baseline_architecture=ARCH_X86,
|
|
724
|
+
comparison_architecture=ARCH_X86,
|
|
730
725
|
):
|
|
731
726
|
print_all = print_regressions_only is False and print_improvements_only is False
|
|
732
727
|
table = []
|
|
@@ -748,20 +743,22 @@ def from_rts_to_regression_table(
|
|
|
748
743
|
"{}={}".format(test_filter, test_name),
|
|
749
744
|
"deployment_name={}".format(baseline_deployment_name),
|
|
750
745
|
"triggering_env={}".format(tf_triggering_env),
|
|
751
|
-
"github_org={}".format(baseline_github_org),
|
|
752
746
|
]
|
|
753
747
|
if running_platform is not None:
|
|
754
748
|
filters_baseline.append("running_platform={}".format(running_platform))
|
|
749
|
+
if baseline_architecture != ARCH_X86:
|
|
750
|
+
filters_baseline.append(f"arch={baseline_architecture}")
|
|
755
751
|
filters_comparison = [
|
|
756
752
|
"{}={}".format(by_str_comparison, comparison_str),
|
|
757
753
|
"metric={}".format(metric_name),
|
|
758
754
|
"{}={}".format(test_filter, test_name),
|
|
759
755
|
"deployment_name={}".format(comparison_deployment_name),
|
|
760
756
|
"triggering_env={}".format(tf_triggering_env),
|
|
761
|
-
"github_org={}".format(comparison_github_org),
|
|
762
757
|
]
|
|
763
758
|
if running_platform is not None:
|
|
764
759
|
filters_comparison.append("running_platform={}".format(running_platform))
|
|
760
|
+
if comparison_architecture != ARCH_X86:
|
|
761
|
+
filters_comparison.append(f"arch={comparison_architecture}")
|
|
765
762
|
baseline_timeseries = rts.ts().queryindex(filters_baseline)
|
|
766
763
|
comparison_timeseries = rts.ts().queryindex(filters_comparison)
|
|
767
764
|
|
|
@@ -1061,9 +1058,9 @@ def get_v_pct_change_and_largest_var(
|
|
|
1061
1058
|
if last_n < 0 or (last_n > 0 and len(comparison_values) < last_n):
|
|
1062
1059
|
comparison_values.append(tuple[1])
|
|
1063
1060
|
comparison_df = pd.DataFrame(comparison_values)
|
|
1064
|
-
comparison_median = float(comparison_df.median()
|
|
1061
|
+
comparison_median = float(comparison_df.median())
|
|
1065
1062
|
comparison_v = comparison_median
|
|
1066
|
-
comparison_std = float(comparison_df.std()
|
|
1063
|
+
comparison_std = float(comparison_df.std())
|
|
1067
1064
|
if verbose:
|
|
1068
1065
|
logging.info(
|
|
1069
1066
|
"comparison_datapoints: {} value: {}; std-dev: {}; median: {}".format(
|
|
@@ -69,13 +69,19 @@ def deploy_command_logic(args, project_name, project_version):
|
|
|
69
69
|
)
|
|
70
70
|
tf_setup_name_sufix = "{}-{}".format(args.setup_name_sufix, tf_github_sha)
|
|
71
71
|
tf_setup_name = "{}{}".format(remote_setup, tf_setup_name_sufix)
|
|
72
|
-
terraform_backend_key = "benchmarks/infrastructure/{}.tfstate".format(
|
|
73
|
-
tf_setup_name
|
|
74
|
-
).replace("/", "-")
|
|
72
|
+
terraform_backend_key = "benchmarks/infrastructure/{}.tfstate".format(tf_setup_name)
|
|
75
73
|
tf_triggering_env = "redisbench-admin-deploy"
|
|
76
74
|
logging.info("Setting an infra timeout of {} secs".format(infra_timeout_secs))
|
|
77
75
|
if args.destroy is False:
|
|
78
|
-
(
|
|
76
|
+
(
|
|
77
|
+
tf_return_code,
|
|
78
|
+
_,
|
|
79
|
+
_,
|
|
80
|
+
_,
|
|
81
|
+
_,
|
|
82
|
+
_,
|
|
83
|
+
_,
|
|
84
|
+
) = setup_remote_environment(
|
|
79
85
|
tf,
|
|
80
86
|
tf_github_sha,
|
|
81
87
|
tf_github_actor,
|
|
@@ -103,7 +109,6 @@ def deploy_command_logic(args, project_name, project_version):
|
|
|
103
109
|
_, _, _ = tf.init(
|
|
104
110
|
capture_output=True,
|
|
105
111
|
backend_config={"key": terraform_backend_key},
|
|
106
|
-
reconfigure=True,
|
|
107
112
|
)
|
|
108
113
|
logging.info("Refreshing remote state")
|
|
109
114
|
_, _, _ = tf.refresh()
|
|
@@ -42,7 +42,13 @@ def export_command_logic(args, project_name, project_version):
|
|
|
42
42
|
deployment_name = args.deployment_name
|
|
43
43
|
deployment_type = args.deployment_type
|
|
44
44
|
results_format = args.results_format
|
|
45
|
-
(
|
|
45
|
+
(
|
|
46
|
+
_,
|
|
47
|
+
github_branch,
|
|
48
|
+
github_org,
|
|
49
|
+
github_repo,
|
|
50
|
+
_,
|
|
51
|
+
) = git_vars_crosscheck(
|
|
46
52
|
None, args.github_branch, args.github_org, args.github_repo, None
|
|
47
53
|
)
|
|
48
54
|
exporter_timemetric_path = None
|
|
@@ -400,9 +400,9 @@ class Perf:
|
|
|
400
400
|
"Main THREAD Flame Graph: " + use_case, details
|
|
401
401
|
)
|
|
402
402
|
if artifact_result is True:
|
|
403
|
-
outputs[
|
|
404
|
-
|
|
405
|
-
|
|
403
|
+
outputs["Main THREAD Flame Graph {}".format(identifier)] = (
|
|
404
|
+
flame_graph_output
|
|
405
|
+
)
|
|
406
406
|
result &= artifact_result
|
|
407
407
|
|
|
408
408
|
tid = self.pid
|
|
@@ -440,9 +440,9 @@ class Perf:
|
|
|
440
440
|
)
|
|
441
441
|
|
|
442
442
|
if artifact_result is True:
|
|
443
|
-
outputs[
|
|
444
|
-
|
|
445
|
-
|
|
443
|
+
outputs["perf report per dso,sym {}".format(identifier)] = (
|
|
444
|
+
perf_report_artifact
|
|
445
|
+
)
|
|
446
446
|
result &= artifact_result
|
|
447
447
|
|
|
448
448
|
# generate perf report per dso,sym
|
|
@@ -460,9 +460,9 @@ class Perf:
|
|
|
460
460
|
)
|
|
461
461
|
|
|
462
462
|
if artifact_result is True:
|
|
463
|
-
outputs[
|
|
464
|
-
|
|
465
|
-
|
|
463
|
+
outputs["perf report per dso,sym with callgraph {}".format(identifier)] = (
|
|
464
|
+
perf_report_artifact
|
|
465
|
+
)
|
|
466
466
|
result &= artifact_result
|
|
467
467
|
|
|
468
468
|
# generate perf report per dso,sym,srcline
|
|
@@ -487,9 +487,9 @@ class Perf:
|
|
|
487
487
|
)
|
|
488
488
|
|
|
489
489
|
if artifact_result is True:
|
|
490
|
-
outputs[
|
|
491
|
-
|
|
492
|
-
|
|
490
|
+
outputs["perf report per dso,sym,srcline {}".format(identifier)] = (
|
|
491
|
+
perf_report_artifact
|
|
492
|
+
)
|
|
493
493
|
result &= artifact_result
|
|
494
494
|
|
|
495
495
|
self.logger.info(
|
|
@@ -527,9 +527,9 @@ class Perf:
|
|
|
527
527
|
)
|
|
528
528
|
|
|
529
529
|
if artifact_result is True:
|
|
530
|
-
outputs[
|
|
531
|
-
|
|
532
|
-
|
|
530
|
+
outputs["perf report top self-cpu {}".format(identifier)] = (
|
|
531
|
+
perf_report_artifact
|
|
532
|
+
)
|
|
533
533
|
result &= artifact_result
|
|
534
534
|
|
|
535
535
|
# generate perf report --stdio report
|
|
@@ -546,9 +546,9 @@ class Perf:
|
|
|
546
546
|
)
|
|
547
547
|
|
|
548
548
|
if artifact_result is True:
|
|
549
|
-
outputs[
|
|
550
|
-
|
|
551
|
-
|
|
549
|
+
outputs["perf report top self-cpu (dso={})".format(binary)] = (
|
|
550
|
+
perf_report_artifact
|
|
551
|
+
)
|
|
552
552
|
result &= artifact_result
|
|
553
553
|
|
|
554
554
|
if self.callgraph_mode == "dwarf":
|
|
@@ -590,9 +590,9 @@ class Perf:
|
|
|
590
590
|
)
|
|
591
591
|
result &= artifact_result
|
|
592
592
|
if artifact_result is True:
|
|
593
|
-
outputs[
|
|
594
|
-
|
|
595
|
-
|
|
593
|
+
outputs["Top entries in text form by LOC"] = (
|
|
594
|
+
pprof_artifact_text_output
|
|
595
|
+
)
|
|
596
596
|
tabular_data_map["text-lines"] = tabular_data
|
|
597
597
|
self.logger.info("Generating pprof png output")
|
|
598
598
|
pprof_png_output = self.output + ".pprof.png"
|
|
@@ -604,9 +604,9 @@ class Perf:
|
|
|
604
604
|
self.output,
|
|
605
605
|
)
|
|
606
606
|
if artifact_result is True:
|
|
607
|
-
outputs[
|
|
608
|
-
|
|
609
|
-
|
|
607
|
+
outputs["Output graph image in PNG format"] = (
|
|
608
|
+
pprof_artifact_png_output
|
|
609
|
+
)
|
|
610
610
|
result &= artifact_result
|
|
611
611
|
|
|
612
612
|
# save stack collapsed
|
redisbench_admin/run/args.py
CHANGED
|
@@ -44,9 +44,20 @@ KEEP_ENV = bool(int(os.getenv("KEEP_ENV", "0")))
|
|
|
44
44
|
ALLOWED_TOOLS_DEFAULT = "memtier_benchmark,redis-benchmark,redisgraph-benchmark-go,ycsb,go-ycsb,tsbs_run_queries_redistimeseries,tsbs_load_redistimeseries,ftsb_redisearch,aibench_run_inference_redisai_vision,ann-benchmarks"
|
|
45
45
|
ALLOWED_BENCH_TOOLS = os.getenv("ALLOWED_BENCH_TOOLS", ALLOWED_TOOLS_DEFAULT)
|
|
46
46
|
SKIP_DB_SETUP = bool(int(os.getenv("SKIP_DB_SETUP", "0")))
|
|
47
|
+
ARCH_X86 = "x86_64"
|
|
48
|
+
ARCH_ARM = "aarch64"
|
|
49
|
+
VALID_ARCHS = [ARCH_X86, ARCH_ARM]
|
|
50
|
+
ARCH = os.getenv("ARCH", ARCH_X86)
|
|
47
51
|
|
|
48
52
|
|
|
49
53
|
def common_run_args(parser):
|
|
54
|
+
parser.add_argument(
|
|
55
|
+
"--architecture",
|
|
56
|
+
type=str,
|
|
57
|
+
required=False,
|
|
58
|
+
default=ARCH,
|
|
59
|
+
help=f"Architecture to run the benchmark on. One of {VALID_ARCHS}.",
|
|
60
|
+
)
|
|
50
61
|
parser.add_argument(
|
|
51
62
|
"--keep_env_and_topo",
|
|
52
63
|
required=False,
|
redisbench_admin/run/common.py
CHANGED
|
@@ -206,7 +206,10 @@ def prepare_benchmark_parameters_specif_tooling(
|
|
|
206
206
|
if isremote is True:
|
|
207
207
|
benchmark_tool = "/tmp/{}".format(benchmark_tool)
|
|
208
208
|
input_data_file = "/tmp/input.data"
|
|
209
|
-
(
|
|
209
|
+
(
|
|
210
|
+
command_arr,
|
|
211
|
+
command_str,
|
|
212
|
+
) = prepare_tsbs_benchmark_command(
|
|
210
213
|
benchmark_tool,
|
|
211
214
|
server_private_ip,
|
|
212
215
|
server_plaintext_port,
|
|
@@ -218,7 +221,10 @@ def prepare_benchmark_parameters_specif_tooling(
|
|
|
218
221
|
cluster_api_enabled,
|
|
219
222
|
)
|
|
220
223
|
if "memtier_benchmark" in benchmark_tool:
|
|
221
|
-
(
|
|
224
|
+
(
|
|
225
|
+
command_arr,
|
|
226
|
+
command_str,
|
|
227
|
+
) = prepare_memtier_benchmark_command(
|
|
222
228
|
benchmark_tool,
|
|
223
229
|
server_private_ip,
|
|
224
230
|
server_plaintext_port,
|
|
@@ -236,7 +242,10 @@ def prepare_benchmark_parameters_specif_tooling(
|
|
|
236
242
|
ann_path = stdout[0].strip() + "/run/ann/pkg/multirun.py"
|
|
237
243
|
logging.info("Remote ann-benchmark path: {}".format(ann_path))
|
|
238
244
|
|
|
239
|
-
(
|
|
245
|
+
(
|
|
246
|
+
command_arr,
|
|
247
|
+
command_str,
|
|
248
|
+
) = prepare_ann_benchmark_command(
|
|
240
249
|
server_private_ip,
|
|
241
250
|
server_plaintext_port,
|
|
242
251
|
cluster_api_enabled,
|
|
@@ -250,7 +259,10 @@ def prepare_benchmark_parameters_specif_tooling(
|
|
|
250
259
|
if isremote is True:
|
|
251
260
|
benchmark_tool = "/tmp/{}".format(benchmark_tool)
|
|
252
261
|
input_data_file = "/tmp/input.data"
|
|
253
|
-
(
|
|
262
|
+
(
|
|
263
|
+
command_arr,
|
|
264
|
+
command_str,
|
|
265
|
+
) = prepare_ftsb_benchmark_command(
|
|
254
266
|
benchmark_tool,
|
|
255
267
|
server_private_ip,
|
|
256
268
|
server_plaintext_port,
|
|
@@ -267,7 +279,10 @@ def prepare_benchmark_parameters_specif_tooling(
|
|
|
267
279
|
if isremote is True:
|
|
268
280
|
benchmark_tool = "/tmp/{}".format(benchmark_tool)
|
|
269
281
|
input_data_file = "/tmp/input.data"
|
|
270
|
-
(
|
|
282
|
+
(
|
|
283
|
+
command_arr,
|
|
284
|
+
command_str,
|
|
285
|
+
) = prepare_aibench_benchmark_command(
|
|
271
286
|
benchmark_tool,
|
|
272
287
|
server_private_ip,
|
|
273
288
|
server_plaintext_port,
|
|
@@ -619,11 +634,9 @@ def run_redis_pre_steps(benchmark_config, r, required_modules):
|
|
|
619
634
|
)
|
|
620
635
|
search_specific_init(r, module_names)
|
|
621
636
|
if required_modules is not None and len(required_modules) > 0:
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
f"Using version {version} in by version timeseries coming from {module_names[artifact_pos]}"
|
|
626
|
-
)
|
|
637
|
+
check_required_modules(module_names, required_modules)
|
|
638
|
+
|
|
639
|
+
version = artifact_versions[0]
|
|
627
640
|
else:
|
|
628
641
|
version = r.info("server")["redis_version"]
|
|
629
642
|
|
|
@@ -689,45 +702,54 @@ def dso_check(dso, local_module_file):
|
|
|
689
702
|
|
|
690
703
|
|
|
691
704
|
def dbconfig_keyspacelen_check(
|
|
692
|
-
benchmark_config, redis_conns, ignore_keyspace_errors=False
|
|
705
|
+
benchmark_config, redis_conns, ignore_keyspace_errors=False, timeout=60
|
|
693
706
|
):
|
|
694
|
-
|
|
707
|
+
start_time = time.time()
|
|
695
708
|
(
|
|
696
709
|
requires_keyspacelen_check,
|
|
697
710
|
keyspacelen,
|
|
698
711
|
) = check_dbconfig_keyspacelen_requirement(benchmark_config)
|
|
699
|
-
|
|
700
|
-
|
|
712
|
+
|
|
713
|
+
if not requires_keyspacelen_check:
|
|
714
|
+
return True
|
|
715
|
+
|
|
716
|
+
attempt = 0
|
|
717
|
+
while time.time() - start_time < timeout:
|
|
701
718
|
logging.info(
|
|
702
|
-
"Ensuring keyspace length requirement = {} is met."
|
|
719
|
+
f"Ensuring keyspace length requirement = {keyspacelen} is met. attempt #{attempt+1}"
|
|
703
720
|
)
|
|
704
721
|
total_keys = 0
|
|
705
722
|
for shard_conn in redis_conns:
|
|
706
723
|
keyspace_dict = shard_conn.info("keyspace")
|
|
707
724
|
for _, dbdict in keyspace_dict.items():
|
|
708
|
-
|
|
709
|
-
total_keys += shard_keys
|
|
725
|
+
total_keys += dbdict.get("keys", 0)
|
|
710
726
|
|
|
711
727
|
if total_keys == keyspacelen:
|
|
712
728
|
logging.info(
|
|
713
|
-
"The total
|
|
729
|
+
"The total number of keys in setup matches the expected spec: {} == {}".format(
|
|
714
730
|
keyspacelen, total_keys
|
|
715
731
|
)
|
|
716
732
|
)
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
)
|
|
733
|
+
return True
|
|
734
|
+
|
|
735
|
+
logging.warning(
|
|
736
|
+
"Keyspace length mismatch ({} != {}). Retrying in {} seconds...".format(
|
|
737
|
+
total_keys, keyspacelen, 2**attempt
|
|
723
738
|
)
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
739
|
+
)
|
|
740
|
+
time.sleep(2**attempt) # Exponential backoff
|
|
741
|
+
attempt += 1
|
|
742
|
+
|
|
743
|
+
logging.error(
|
|
744
|
+
f"The total number of keys in setup does not match the expected spec: {keyspacelen} != {total_keys}. Aborting after {attempt+1} tries..."
|
|
745
|
+
)
|
|
746
|
+
|
|
747
|
+
if not ignore_keyspace_errors:
|
|
748
|
+
raise Exception(
|
|
749
|
+
f"The total number of keys in setup does not match the expected spec: {keyspacelen} != {total_keys}. Aborting after {attempt+1} tries..."
|
|
750
|
+
)
|
|
751
|
+
|
|
752
|
+
return False
|
|
731
753
|
|
|
732
754
|
|
|
733
755
|
def common_properties_log(
|
|
@@ -765,7 +787,10 @@ def print_results_table_stdout(
|
|
|
765
787
|
metric_names=[],
|
|
766
788
|
):
|
|
767
789
|
# check which metrics to extract
|
|
768
|
-
(
|
|
790
|
+
(
|
|
791
|
+
_,
|
|
792
|
+
metrics,
|
|
793
|
+
) = merge_default_and_config_metrics(
|
|
769
794
|
benchmark_config,
|
|
770
795
|
default_metrics,
|
|
771
796
|
None,
|
|
@@ -29,7 +29,6 @@ def prepare_ftsb_benchmark_command(
|
|
|
29
29
|
:return: [string] containing the required command to run the benchmark given the configurations
|
|
30
30
|
"""
|
|
31
31
|
command_arr = [executable_path]
|
|
32
|
-
|
|
33
32
|
command_arr.extend(
|
|
34
33
|
["--host", "{}:{}".format(server_private_ip, server_plaintext_port)]
|
|
35
34
|
)
|
redisbench_admin/run/ssh.py
CHANGED
|
@@ -42,7 +42,6 @@ def ssh_tunnel_redisconn(
|
|
|
42
42
|
), # remote redis server
|
|
43
43
|
# Bind the socket to port 0. A random free port from 1024 to 65535 will be selected.
|
|
44
44
|
local_bind_address=("0.0.0.0", 0), # enable local forwarding port
|
|
45
|
-
set_keepalive=60,
|
|
46
45
|
)
|
|
47
46
|
ssh_tunel.start() # start tunnel
|
|
48
47
|
redis_conn = redis.Redis(
|
|
@@ -70,19 +69,6 @@ def check_connection(ssh_conn):
|
|
|
70
69
|
return False
|
|
71
70
|
|
|
72
71
|
|
|
73
|
-
def ensure_400_permissions(file_path):
|
|
74
|
-
import stat
|
|
75
|
-
|
|
76
|
-
file_stat = os.stat(file_path)
|
|
77
|
-
current_permissions = stat.S_IMODE(file_stat.st_mode)
|
|
78
|
-
|
|
79
|
-
if current_permissions != 0o400:
|
|
80
|
-
logging.info(f"Changing permissions of {file_path} to 400")
|
|
81
|
-
os.chmod(file_path, 0o400)
|
|
82
|
-
else:
|
|
83
|
-
logging.info(f"{file_path} already has 400 permissions.")
|
|
84
|
-
|
|
85
|
-
|
|
86
72
|
def ssh_pem_check(EC2_PRIVATE_PEM, private_key):
|
|
87
73
|
if os.path.exists(private_key) is False:
|
|
88
74
|
if EC2_PRIVATE_PEM is not None and EC2_PRIVATE_PEM != "":
|
|
@@ -103,4 +89,3 @@ def ssh_pem_check(EC2_PRIVATE_PEM, private_key):
|
|
|
103
89
|
logging.info(
|
|
104
90
|
"Confirmed that private key path artifact: '{}' exists!".format(private_key)
|
|
105
91
|
)
|
|
106
|
-
ensure_400_permissions(private_key)
|
|
@@ -114,7 +114,11 @@ class TerraformClass:
|
|
|
114
114
|
def async_runner_setup(
|
|
115
115
|
self,
|
|
116
116
|
):
|
|
117
|
-
(
|
|
117
|
+
(
|
|
118
|
+
remote_setup,
|
|
119
|
+
deployment_type,
|
|
120
|
+
remote_id,
|
|
121
|
+
) = fetch_remote_setup_from_config(
|
|
118
122
|
[{"type": "async", "setup": "runner"}],
|
|
119
123
|
"https://github.com/RedisLabsModules/testing-infrastructure.git",
|
|
120
124
|
"master",
|
|
@@ -229,7 +233,11 @@ def terraform_spin_or_reuse_env(
|
|
|
229
233
|
tf_override_name=None,
|
|
230
234
|
tf_folder_path=None,
|
|
231
235
|
):
|
|
232
|
-
(
|
|
236
|
+
(
|
|
237
|
+
remote_setup,
|
|
238
|
+
deployment_type,
|
|
239
|
+
remote_id,
|
|
240
|
+
) = fetch_remote_setup_from_config(
|
|
233
241
|
benchmark_config["remote"],
|
|
234
242
|
"https://github.com/RedisLabsModules/testing-infrastructure.git",
|
|
235
243
|
"master",
|
|
@@ -28,9 +28,9 @@ WantedBy=multi-user.target
|
|
|
28
28
|
argv.append("--private_key")
|
|
29
29
|
argv.append("/home/ubuntu/work_dir/tests/benchmarks/benchmarks.redislabs.pem")
|
|
30
30
|
else:
|
|
31
|
-
argv[
|
|
32
|
-
|
|
33
|
-
|
|
31
|
+
argv[argv.index(args.private_key)] = (
|
|
32
|
+
"/home/ubuntu/work_dir/tests/benchmarks/benchmarks.redislabs.pem"
|
|
33
|
+
)
|
|
34
34
|
if len(args.module_path) != 0:
|
|
35
35
|
argv[argv.index(args.module_path[0])] = (
|
|
36
36
|
"/home/ubuntu/work_dir/tests/benchmarks/"
|
|
@@ -12,6 +12,7 @@ FLUSHALL_AT_START = bool(int(os.getenv("FLUSHALL_AT_START", "1")))
|
|
|
12
12
|
IGNORE_KEYSPACE_ERRORS = bool(int(os.getenv("IGNORE_KEYSPACE_ERRORS", "0")))
|
|
13
13
|
SKIP_REDIS_SPIN = bool(int(os.getenv("SKIP_REDIS_SPIN", "0")))
|
|
14
14
|
REDIS_PORT = int(os.getenv("REDIS_PORT", "6379"))
|
|
15
|
+
REDIS_AUTH = os.getenv("REDIS_AUTH", None)
|
|
15
16
|
REDIS_HOST = os.getenv("REDIS_HOST", "127.0.0.1")
|
|
16
17
|
|
|
17
18
|
|
|
@@ -19,6 +20,7 @@ def create_run_local_arguments(parser):
|
|
|
19
20
|
parser = common_run_args(parser)
|
|
20
21
|
parser.add_argument("--port", type=int, default=REDIS_PORT)
|
|
21
22
|
parser.add_argument("--host", type=str, default=REDIS_HOST)
|
|
23
|
+
parser.add_argument("--password", type=str, default=REDIS_AUTH)
|
|
22
24
|
parser.add_argument("--redis-binary", type=str, default=REDIS_BINARY)
|
|
23
25
|
parser.add_argument(
|
|
24
26
|
"--flushall_on_every_test_start",
|