redisbench-admin 0.10.27__tar.gz → 0.10.29__tar.gz

{redisbench_admin-0.10.27 → redisbench_admin-0.10.29}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: redisbench-admin
-Version: 0.10.27
+Version: 0.10.29
 Summary: Redis benchmark run helper. A wrapper around Redis and Redis Modules benchmark tools ( ftsb_redisearch, memtier_benchmark, redis-benchmark, aibench, etc... ).
 Author: filipecosta90
 Author-email: filipecosta.90@gmail.com

{redisbench_admin-0.10.27 → redisbench_admin-0.10.29}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redisbench-admin"
-version = "0.10.27"
+version = "0.10.29"
 description = "Redis benchmark run helper. A wrapper around Redis and Redis Modules benchmark tools ( ftsb_redisearch, memtier_benchmark, redis-benchmark, aibench, etc... )."
 authors = ["filipecosta90 <filipecosta.90@gmail.com>","Redis Performance Group <performance@redis.com>"]
 readme = "README.md"

{redisbench_admin-0.10.27 → redisbench_admin-0.10.29}/redisbench_admin/run/common.py

@@ -489,15 +489,30 @@ def extract_test_feasible_setups(
                         if setup_name == setup["name"]:
                             feasible_setups_map[setup_name] = setup
     if len(feasible_setups_map.keys()) == 0 and backwards_compatible:
-        feasible_setups_map["oss-standalone"] = {
-            "name": "oss-standalone",
-            "type": "oss-standalone",
+        setup_name = "oss-standalone"
+        setup_type = "oss-standalone"
+        OVERRIDE_SETUP_TYPE = os.getenv("OVERRIDE_SETUP_TYPE", None)
+        if OVERRIDE_SETUP_TYPE is not None:
+            logging.info(
+                f"Overriding SETUP_TYPE with {OVERRIDE_SETUP_TYPE} (original value was {setup_type})"
+            )
+            setup_type = OVERRIDE_SETUP_TYPE
+        OVERRIDE_SETUP_NAME = os.getenv("OVERRIDE_SETUP_NAME", None)
+        if OVERRIDE_SETUP_NAME is not None:
+            logging.info(
+                f"Overriding SETUP_NAME with {OVERRIDE_SETUP_NAME} (original value was {setup_name})"
+            )
+            setup_name = OVERRIDE_SETUP_NAME
+
+        feasible_setups_map[setup_name] = {
+            "name": setup_name,
+            "type": setup_type,
             "redis_topology": {"primaries": 1, "replicas": 0},
             "resources": {"requests": {"cpu": "1000m"}, "limits": {"cpu": "2000m"}},
         }
         logging.info(
             "Using a backwards compatible 'oss-standalone' setup, with settings: {}".format(
-                feasible_setups_map["oss-standalone"]
+                feasible_setups_map[setup_name]
             )
         )
 
@@ -709,6 +724,7 @@ def print_results_table_stdout(
     default_metrics,
     results_dict,
     setup_name,
+    setup_type,
     test_name,
     cpu_usage=None,
     kv_overall={},
@@ -720,7 +736,9 @@ def print_results_table_stdout(
         default_metrics,
         None,
     )
-    table_name = "Results for {} test-case on {} topology".format(test_name, setup_name)
+    table_name = "Results for {} test-case on {} topology (type={})".format(
+        test_name, setup_name, setup_type
+    )
     results_matrix_headers = [
         "Metric JSON Path",
         "Metric Value",

{redisbench_admin-0.10.27 → redisbench_admin-0.10.29}/redisbench_admin/run/redistimeseries.py

@@ -269,8 +269,8 @@ def timeseries_test_sucess_flow(
         )
     if push_results_redistimeseries:
         logging.info(
-            "Pushing results to RedisTimeSeries. Have {} distinct data-points to insert.".format(
-                len(timeseries_dict.keys())
+            "Pushing results to RedisTimeSeries. Have {} distinct data-points to insert. (deployment_name={}, deployment_type={})".format(
+                len(timeseries_dict.keys()), deployment_name, deployment_type
             )
         )
         push_data_to_redistimeseries(rts, timeseries_dict)

{redisbench_admin-0.10.27 → redisbench_admin-0.10.29}/redisbench_admin/run_local/run_local.py

@@ -13,6 +13,11 @@ import traceback
 import redis
 from redisbench_admin.run.git import git_vars_crosscheck
 
+from redisbench_admin.utils.remote import (
+    get_project_ts_tags,
+    push_data_to_redistimeseries,
+)
+
 import redisbench_admin.run.metrics
 from redisbench_admin.profilers.perf import PERF_CALLGRAPH_MODE
 from redisbench_admin.profilers.profilers_schema import (
@@ -428,6 +433,7 @@ def run_local_command_logic(args, project_name, project_version):
                                         default_metrics,
                                         results_dict,
                                         setup_name,
+                                        setup_type,
                                         test_name,
                                         total_shards_cpu_usage,
                                         overall_end_time_metrics,
@@ -436,6 +442,21 @@ def run_local_command_logic(args, project_name, project_version):
                                             "memory_used_memory_dataset",
                                         ],
                                     )
+                                    export_redis_metrics(
+                                        artifact_version,
+                                        end_time_ms,
+                                        overall_end_time_metrics,
+                                        rts,
+                                        setup_name,
+                                        setup_type,
+                                        test_name,
+                                        tf_github_branch,
+                                        tf_github_org,
+                                        tf_github_repo,
+                                        tf_triggering_env,
+                                        {"metric-type": "redis-metrics"},
+                                        0,
+                                    )
 
                                     # check KPIs
                                     return_code = results_dict_kpi_check(
@@ -464,6 +485,7 @@ def run_local_command_logic(args, project_name, project_version):
                                     github_org_name,
                                     github_repo_name,
                                     tf_triggering_env,
+                                    metadata_tags,
                                 )
 
                                 if setup_details["env"] is None:
@@ -526,3 +548,151 @@ def teardown_local_setup(redis_conns, redis_processes, setup_name):
     for conn in redis_conns:
         conn.shutdown(nosave=True)
     logging.info("Tear-down completed")
+
+
+def export_redis_metrics(
+    artifact_version,
+    end_time_ms,
+    overall_end_time_metrics,
+    rts,
+    setup_name,
+    setup_type,
+    test_name,
+    tf_github_branch,
+    tf_github_org,
+    tf_github_repo,
+    tf_triggering_env,
+    metadata_dict=None,
+    expire_ms=0,
+):
+    datapoint_errors = 0
+    datapoint_inserts = 0
+    sprefix = (
+        "ci.benchmarks.redislabs/"
+        + "{triggering_env}/{github_org}/{github_repo}".format(
+            triggering_env=tf_triggering_env,
+            github_org=tf_github_org,
+            github_repo=tf_github_repo,
+        )
+    )
+
+    logging.info(
+        "Adding a total of {} server side metrics collected at the end of benchmark (deployment_name={}, deployment_type={})".format(
+            len(list(overall_end_time_metrics.items())), setup_name, setup_type
+        )
+    )
+    timeseries_dict = {}
+    by_variants = {}
+    if tf_github_branch is not None and tf_github_branch != "":
+        by_variants["by.branch/{}".format(tf_github_branch)] = {
+            "branch": tf_github_branch
+        }
+    if artifact_version is not None and artifact_version != "":
+        by_variants["by.version/{}".format(artifact_version)] = {
+            "version": artifact_version
+        }
+    for (
+        by_variant,
+        variant_labels_dict,
+    ) in by_variants.items():
+        for (
+            metric_name,
+            metric_value,
+        ) in overall_end_time_metrics.items():
+            tsname_metric = "{}/{}/{}/benchmark_end/{}/{}".format(
+                sprefix,
+                test_name,
+                by_variant,
+                setup_name,
+                metric_name,
+            )
+
+            logging.debug(
+                "Adding a redis server side metric collected at the end of benchmark."
+                + " metric_name={} metric_value={} time-series name: {}".format(
+                    metric_name,
+                    metric_value,
+                    tsname_metric,
+                )
+            )
+            variant_labels_dict["metric"] = metric_name
+            commandstats_latencystats_process_name(
+                metric_name, "commandstats_cmdstat_", setup_name, variant_labels_dict
+            )
+            commandstats_latencystats_process_name(
+                metric_name,
+                "latencystats_latency_percentiles_usec_",
+                setup_name,
+                variant_labels_dict,
+            )
+
+            variant_labels_dict["test_name"] = test_name
+            if metadata_dict is not None:
+                variant_labels_dict.update(metadata_dict)
+
+            timeseries_dict[tsname_metric] = {
+                "labels": get_project_ts_tags(
+                    tf_github_org,
+                    tf_github_repo,
+                    setup_name,
+                    setup_type,
+                    tf_triggering_env,
+                    variant_labels_dict,
+                    None,
+                    None,
+                ),
+                "data": {end_time_ms: metric_value},
+            }
+    i_errors, i_inserts = push_data_to_redistimeseries(rts, timeseries_dict, expire_ms)
+    datapoint_errors = datapoint_errors + i_errors
+    datapoint_inserts = datapoint_inserts + i_inserts
+    return datapoint_errors, datapoint_inserts
+
+
+def commandstats_latencystats_process_name(
+    metric_name, prefix, setup_name, variant_labels_dict
+):
+    if prefix in metric_name:
+        command_and_metric_and_shard = metric_name[len(prefix) :]
+        command = (
+            command_and_metric_and_shard[0]
+            + command_and_metric_and_shard[1:].split("_", 1)[0]
+        )
+        metric_and_shard = command_and_metric_and_shard[1:].split("_", 1)[1]
+        metric = metric_and_shard
+        shard = "1"
+        if "_shard_" in metric_and_shard:
+            metric = metric_and_shard.split("_shard_")[0]
+            shard = metric_and_shard.split("_shard_")[1]
+        variant_labels_dict["metric"] = metric
+        variant_labels_dict["command"] = command
+        variant_labels_dict["command_and_metric"] = "{} - {}".format(command, metric)
+        variant_labels_dict["command_and_metric_and_setup"] = "{} - {} - {}".format(
+            command, metric, setup_name
+        )
+        variant_labels_dict["command_and_setup"] = "{} - {}".format(command, setup_name)
+        variant_labels_dict["shard"] = shard
+        variant_labels_dict["metric_and_shard"] = metric_and_shard
+
+        version = None
+        branch = None
+        if "version" in variant_labels_dict:
+            version = variant_labels_dict["version"]
+        if "branch" in variant_labels_dict:
+            branch = variant_labels_dict["branch"]
+
+        if version is not None:
+            variant_labels_dict[
+                "command_and_metric_and_version"
+            ] = "{} - {} - {}".format(command, metric, version)
+            variant_labels_dict[
+                "command_and_metric_and_setup_and_version"
+            ] = "{} - {} - {} - {}".format(command, metric, setup_name, version)
+
+        if branch is not None:
+            variant_labels_dict[
+                "command_and_metric_and_branch"
+            ] = "{} - {} - {}".format(command, metric, branch)
+            variant_labels_dict[
+                "command_and_metric_and_setup_and_branch"
+            ] = "{} - {} - {} - {}".format(command, metric, setup_name, branch)

{redisbench_admin-0.10.27 → redisbench_admin-0.10.29}/redisbench_admin/run_remote/run_remote.py

@@ -1008,6 +1008,7 @@ def run_remote_command_logic(args, project_name, project_version):
                                             default_metrics,
                                             results_dict,
                                             setup_name,
+                                            setup_type,
                                             test_name,
                                             total_shards_cpu_usage,
                                         )

{redisbench_admin-0.10.27 → redisbench_admin-0.10.29}/setup.py

@@ -82,7 +82,7 @@ entry_points = \
 
 setup_kwargs = {
     'name': 'redisbench-admin',
-    'version': '0.10.27',
+    'version': '0.10.29',
     'description': 'Redis benchmark run helper. A wrapper around Redis and Redis Modules benchmark tools ( ftsb_redisearch, memtier_benchmark, redis-benchmark, aibench, etc... ).',
     'long_description': '[![codecov](https://codecov.io/gh/redis-performance/redisbench-admin/branch/master/graph/badge.svg)](https://codecov.io/gh/redis-performance/redisbench-admin)\n![Actions](https://github.com/redis-performance/redisbench-admin/workflows/Run%20Tests/badge.svg?branch=master)\n![Actions](https://badge.fury.io/py/redisbench-admin.svg)\n\n# [redisbench-admin](https://github.com/redis-performance/redisbench-admin)\n\nRedis benchmark run helper can help you with the following tasks:\n\n- Setup abd teardown of benchmarking infrastructure specified\n  on [redis-performance/testing-infrastructure](https://github.com/redis-performance/testing-infrastructure)\n- Setup and teardown of an Redis and Redis Modules DBs for benchmarking\n- Management of benchmark data and specifications across different setups\n- Running benchmarks and recording results\n- Exporting performance results in several formats (CSV, RedisTimeSeries, JSON)\n- Finding on-cpu, off-cpu, io, and threading performance problems by attaching profiling tools/probers ( perf (a.k.a. perf_events), bpf tooling, vtune )\n- **[SOON]** Finding performance problems by attaching telemetry probes\n\nCurrent supported benchmark tools:\n\n- [redis-benchmark](https://github.com/redis/redis)\n- [memtier_benchmark](https://github.com/RedisLabs/memtier_benchmark)\n- [redis-benchmark-go](https://github.com/redis-performance/redis-benchmark-go)\n- [YCSB](https://github.com/RediSearch/YCSB)\n- [tsbs](https://github.com/RedisTimeSeries/tsbs)\n- [redisgraph-benchmark-go](https://github.com/RedisGraph/redisgraph-benchmark-go)\n- [ftsb_redisearch](https://github.com/RediSearch/ftsb)\n- [ann-benchmarks](https://github.com/RedisAI/ann-benchmarks)\n\n## Installation\n\nInstallation is done using pip, the package installer for Python, in the following manner:\n\n```bash\npython3 -m pip install redisbench-admin\n```\n\n## Profiler daemon\n\nYou can use the profiler daemon by itself in the following manner. \nOn the target machine do as follow:\n\n```bash\npip3 install --upgrade pip\npip3 install redisbench-admin --ignore-installed PyYAML\n\n# install perf\napt install linux-tools-common linux-tools-generic linux-tools-`uname -r` -y\n\n# ensure perf is working\nperf --version\n\n# install awscli\nsnap install aws-cli --classic\n\n\n# configure aws\naws configure\n\n# start the perf-daemon\nperf-daemon start\nWARNING:root:Unable to detected github_actor. caught the following error: No section: \'user\'\nWritting log to /tmp/perf-daemon.log\nStarting perf-daemon. PID file /tmp/perfdaemon.pid. Daemon workdir: /root/RedisGraph\n\n# check daemon is working appropriatelly\ncurl localhost:5000/ping\n\n# start a profile\ncurl -X POST localhost:5000/profiler/perf/start/<pid to profile>\n\n# stop a profile\ncurl -X POST -d \'{"aws_access_key_id":$AWS_ACCESS_KEY_ID,"aws_secret_access_key":$AWS_SECRET_ACCESS_KEY}\' localhost:5000/profiler/perf/stop/<pid to profile>\n```\n\n\n## Development\n\n1. Install [pypoetry](https://python-poetry.org/) to manage your dependencies and trigger tooling.\n```sh\npip install poetry\n```\n\n2. Installing dependencies from lock file\n\n```\npoetry install\n```\n\n### Running formaters\n\n```sh\npoetry run black .\n```\n\n\n### Running linters\n\n```sh\npoetry run flake8\n```\n\n\n### Running tests\n\nA test suite is provided, and can be run with:\n\n```sh\n$ tox\n```\n\nTo run a specific test:\n```sh\n$ tox -- tests/test_redistimeseries.py\n```\n\n## License\n\nredisbench-admin is distributed under the BSD3 license - see [LICENSE](LICENSE)\n',
     'author': 'filipecosta90',