redis-benchmarks-specification 0.1.299__py3-none-any.whl → 0.1.301__py3-none-any.whl

redis_benchmarks_specification/__common__/runner.py

@@ -3,13 +3,13 @@ import logging
 import os
 import pathlib
 import re
-
+import datetime as dt
 import redis
-from redisbench_admin.run.metrics import collect_redis_metrics
-from redisbench_admin.run_remote.run_remote import export_redis_metrics
 
 from redis_benchmarks_specification.__common__.timeseries import (
     timeseries_test_sucess_flow,
+    push_data_to_redistimeseries,
+    get_project_ts_tags,
 )
 
 
@@ -119,6 +119,210 @@ def extract_testsuites(args):
     return testsuite_spec_files
 
 
+
+def commandstats_latencystats_process_name(
+    metric_name, prefix, setup_name, variant_labels_dict
+):
+    if prefix in metric_name:
+        command_and_metric_and_shard = metric_name[len(prefix) :]
+        command = (
+            command_and_metric_and_shard[0]
+            + command_and_metric_and_shard[1:].split("_", 1)[0]
+        )
+        metric_and_shard = command_and_metric_and_shard[1:].split("_", 1)[1]
+        metric = metric_and_shard
+        shard = "1"
+        if "_shard_" in metric_and_shard:
+            metric = metric_and_shard.split("_shard_")[0]
+            shard = metric_and_shard.split("_shard_")[1]
+        variant_labels_dict["metric"] = metric
+        variant_labels_dict["command"] = command
+        variant_labels_dict["command_and_metric"] = "{} - {}".format(command, metric)
+        variant_labels_dict["command_and_metric_and_setup"] = "{} - {} - {}".format(
+            command, metric, setup_name
+        )
+        variant_labels_dict["command_and_setup"] = "{} - {}".format(command, setup_name)
+        variant_labels_dict["shard"] = shard
+        variant_labels_dict["metric_and_shard"] = metric_and_shard
+
+        version = None
+        branch = None
+        if "version" in variant_labels_dict:
+            version = variant_labels_dict["version"]
+        if "branch" in variant_labels_dict:
+            branch = variant_labels_dict["branch"]
+
+        if version is not None:
+            variant_labels_dict["command_and_metric_and_version"] = (
+                "{} - {} - {}".format(command, metric, version)
+            )
+            variant_labels_dict["command_and_metric_and_setup_and_version"] = (
+                "{} - {} - {} - {}".format(command, metric, setup_name, version)
+            )
+
+        if branch is not None:
+            variant_labels_dict["command_and_metric_and_branch"] = (
+                "{} - {} - {}".format(command, metric, branch)
+            )
+            variant_labels_dict["command_and_metric_and_setup_and_branch"] = (
+                "{} - {} - {} - {}".format(command, metric, setup_name, branch)
+            )
+
+
+def collect_redis_metrics(
+    redis_conns,
+    sections=["memory", "cpu", "commandstats", "latencystats"],
+    section_filter=None,
+):
+    start_time = dt.datetime.utcnow()
+    start_time_ms = int((start_time - dt.datetime(1970, 1, 1)).total_seconds() * 1000)
+    res = []
+    overall = {}
+    multi_shard = False
+    if len(redis_conns) > 1:
+        multi_shard = True
+    for conn_n, conn in enumerate(redis_conns):
+        conn_res = {}
+        for section in sections:
+            info = conn.info(section)
+            conn_res[section] = info
+            if section not in overall:
+                overall[section] = {}
+            for k, v in info.items():
+                collect = True
+                if section_filter is not None:
+                    if section in section_filter:
+                        if k not in section_filter[section]:
+                            collect = False
+                if collect and type(v) is float or type(v) is int:
+                    if k not in overall[section]:
+                        overall[section][k] = 0
+                    overall[section][k] += v
+                if collect and type(v) is dict:
+                    for inner_k, inner_v in v.items():
+                        if type(inner_v) is float or type(inner_v) is int:
+                            final_str_k = "{}_{}".format(k, inner_k)
+                            if multi_shard:
+                                final_str_k += "_shard_{}".format(conn_n + 1)
+                            if final_str_k not in overall[section]:
+                                overall[section][final_str_k] = inner_v
+
+        res.append(conn_res)
+
+    kv_overall = {}
+    for sec, kv_detail in overall.items():
+        for k, metric_value in kv_detail.items():
+            metric_name = "{}_{}".format(sec, k)
+            kv_overall[metric_name] = metric_value
+
+    return start_time_ms, res, kv_overall
+
+def export_redis_metrics(
+    artifact_version,
+    end_time_ms,
+    overall_end_time_metrics,
+    rts,
+    setup_name,
+    setup_type,
+    test_name,
+    tf_github_branch,
+    tf_github_org,
+    tf_github_repo,
+    tf_triggering_env,
+    metadata_dict=None,
+    expire_ms=0,
+    git_hash=None,
+    running_platform=None,
+):
+    datapoint_errors = 0
+    datapoint_inserts = 0
+    sprefix = (
+        "ci.benchmarks.redis/"
+        + "{triggering_env}/{github_org}/{github_repo}".format(
+            triggering_env=tf_triggering_env,
+            github_org=tf_github_org,
+            github_repo=tf_github_repo,
+        )
+    )
+    logging.info(
+        "Adding a total of {} server side metrics collected at the end of benchmark".format(
+            len(list(overall_end_time_metrics.items()))
+        )
+    )
+    timeseries_dict = {}
+    by_variants = {}
+    if tf_github_branch is not None and tf_github_branch != "":
+        by_variants["by.branch/{}".format(tf_github_branch)] = {
+            "branch": tf_github_branch
+        }
+    if git_hash is not None and git_hash != "":
+        by_variants["by.hash/{}".format(git_hash)] = {
+            "hash": git_hash
+        }
+    if artifact_version is not None and artifact_version != "":
+        by_variants["by.version/{}".format(artifact_version)] = {
+            "version": artifact_version
+        }
+    for (
+        by_variant,
+        variant_labels_dict,
+    ) in by_variants.items():
+        for (
+            metric_name,
+            metric_value,
+        ) in overall_end_time_metrics.items():
+            tsname_metric = "{}/{}/{}/benchmark_end/{}/{}".format(
+                sprefix,
+                test_name,
+                by_variant,
+                setup_name,
+                metric_name,
+            )
+
+            logging.debug(
+                "Adding a redis server side metric collected at the end of benchmark."
+                + " metric_name={} metric_value={} time-series name: {}".format(
+                    metric_name,
+                    metric_value,
+                    tsname_metric,
+                )
+            )
+            variant_labels_dict["metric"] = metric_name
+            commandstats_latencystats_process_name(
+                metric_name, "commandstats_cmdstat_", setup_name, variant_labels_dict
+            )
+            commandstats_latencystats_process_name(
+                metric_name,
+                "latencystats_latency_percentiles_usec_",
+                setup_name,
+                variant_labels_dict,
+            )
+
+            variant_labels_dict["test_name"] = test_name
+            if metadata_dict is not None:
+                variant_labels_dict.update(metadata_dict)
+
+            timeseries_dict[tsname_metric] = {
+                "labels": get_project_ts_tags(
+                    tf_github_org,
+                    tf_github_repo,
+                    setup_name,
+                    setup_type,
+                    tf_triggering_env,
+                    variant_labels_dict,
+                    None,
+                    running_platform,
+                ),
+                "data": {end_time_ms: metric_value},
+            }
+    i_errors, i_inserts = push_data_to_redistimeseries(rts, timeseries_dict, expire_ms)
+    datapoint_errors = datapoint_errors + i_errors
+    datapoint_inserts = datapoint_inserts + i_inserts
+    return datapoint_errors, datapoint_inserts
+
+
+
+
 def reset_commandstats(redis_conns):
     for pos, redis_conn in enumerate(redis_conns):
         logging.info("Resetting commmandstats for shard {}".format(pos))

redis_benchmarks_specification/__runner__/runner.py

@@ -28,7 +28,10 @@ from redisbench_admin.run.common import (
     prepare_benchmark_parameters,
     dbconfig_keyspacelen_check,
 )
-from redisbench_admin.run_remote.run_remote import export_redis_metrics
+
+from redis_benchmarks_specification.__common__.runner import (
+    export_redis_metrics,
+)
 
 from redisbench_admin.run.metrics import extract_results_table
 from redisbench_admin.run.run import calculate_client_tool_duration_and_check
@@ -56,6 +59,7 @@ from redis_benchmarks_specification.__common__.runner import (
     exporter_datasink_common,
     reset_commandstats,
     execute_init_commands,
+    export_redis_metrics,
 )
 from redis_benchmarks_specification.__common__.spec import (
     extract_client_container_image,
@@ -1935,8 +1939,10 @@ def process_self_contained_coordinator_stream(
                                 tf_github_org,
                                 tf_github_repo,
                                 tf_triggering_env,
-                                {"metric-type": "memory-stats"},
+                                metadata,
                                 expire_redis_metrics_ms,
+                                git_hash,
+                                running_platform,
                             )
                             
                             exporter_datasink_common(
@@ -1962,8 +1968,8 @@ def process_self_contained_coordinator_stream(
                                 topology_spec_name,
                                 default_metrics,
                                 git_hash,
-                                collect_commandstats=False,
-                                collect_memory_metrics=True,
+                                False,
+                                True,
                             )
 
                         # Send MEMORY PURGE after memory comparison (if FLUSHALL at test end is not enabled)

{redis_benchmarks_specification-0.1.299.dist-info → redis_benchmarks_specification-0.1.301.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: redis-benchmarks-specification
-Version: 0.1.299
+Version: 0.1.301
 Summary: The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute.
 Author: filipecosta90
 Author-email: filipecosta.90@gmail.com

{redis_benchmarks_specification-0.1.299.dist-info → redis_benchmarks_specification-0.1.301.dist-info}/RECORD

@@ -15,7 +15,7 @@ redis_benchmarks_specification/__common__/builder_schema.py,sha256=kfDpRIk7NkJrb
 redis_benchmarks_specification/__common__/env.py,sha256=kvJ8Ll-fvI_Tc0vynrzUEr22TqnJizzvJ4Lu9RjNr_M,3119
 redis_benchmarks_specification/__common__/github.py,sha256=9TZtnISsSgXTSAN_VQejo5YRPDPhlU0gjxgKGPw_sP8,10699
 redis_benchmarks_specification/__common__/package.py,sha256=4uVt1BAZ999LV2rZkq--Tk6otAVIf9YR3g3KGeUpiW4,834
-redis_benchmarks_specification/__common__/runner.py,sha256=3eZ3GUKfWMUbvNhoryRnbennqPVRqzQ9189-B9N2YdE,8314
+redis_benchmarks_specification/__common__/runner.py,sha256=s5IJnxO527SoImhQmOU2UrqR4_h3MnHM2i7dh7wNV1Y,15649
 redis_benchmarks_specification/__common__/spec.py,sha256=D_SN48wg6NMthW_-OS1H5bydSDiuZpfd4WPPj7Vfwmc,5760
 redis_benchmarks_specification/__common__/timeseries.py,sha256=w8XQrGPEFuuemDXXz4iny7lYsNbYH0ycQyq3jHIr80g,52916
 redis_benchmarks_specification/__compare__/__init__.py,sha256=DtBXRp0Q01XgCFmY-1OIePMyyYihVNAjZ1Y8zwqSDN0,101
@@ -25,7 +25,7 @@ redis_benchmarks_specification/__init__.py,sha256=YQIEx2sLPPA0JR9OuCuMNMNtm-f_gq
 redis_benchmarks_specification/__runner__/__init__.py,sha256=l-G1z-t6twUgi8QLueqoTQLvJmv3hJoEYskGm6H7L6M,83
 redis_benchmarks_specification/__runner__/args.py,sha256=YeN7-eOGoqCHKh5FrKz9z5Ee-Rh-3DF9tm2Hb-3m7sQ,10648
 redis_benchmarks_specification/__runner__/remote_profiling.py,sha256=JS46mFxnRB2HSU_HO90WE4w7P7usHzX1dwZg1bJpTMc,18588
-redis_benchmarks_specification/__runner__/runner.py,sha256=ULuDKCiXk5BCk_I3lyDkggBNq_xmfk47n5poQCeT3yQ,137882
+redis_benchmarks_specification/__runner__/runner.py,sha256=pHgPkZktwlsIrvYaGais0SAvkH5olDeTDO4R5kdxudU,137953
 redis_benchmarks_specification/__self_contained_coordinator__/__init__.py,sha256=l-G1z-t6twUgi8QLueqoTQLvJmv3hJoEYskGm6H7L6M,83
 redis_benchmarks_specification/__self_contained_coordinator__/args.py,sha256=uxBjdQ78klvsVi6lOfGYQVaWIxc8OI-DwYKY16SgvCY,5952
 redis_benchmarks_specification/__self_contained_coordinator__/artifacts.py,sha256=OVHqJzDgeSSRfUSiKp1ZTAVv14PvSbk-5yJsAAoUfpw,936
@@ -273,8 +273,8 @@ redis_benchmarks_specification/test-suites/memtier_benchmark-nokeys-pubsub-publi
 redis_benchmarks_specification/test-suites/memtier_benchmark-nokeys-server-time-pipeline-10.yml,sha256=7G_J8kUFay7jXhZvsZK5jvVHSLZvhMV0uuDMkZBbeSQ,675
 redis_benchmarks_specification/test-suites/template.txt,sha256=d_edIE7Sxa5X7I2yG-Io0bPdbDIHR0oWFoCA3XUt_EU,435
 redis_benchmarks_specification/vector-search-test-suites/vector_db_benchmark_test.yml,sha256=PD7ow-k4Ll2BkhEC3aIqiaCZt8Hc4aJIp96Lw3J3mcI,791
-redis_benchmarks_specification-0.1.299.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-redis_benchmarks_specification-0.1.299.dist-info/METADATA,sha256=xs8UTp9pDMtka8HrMqorK1nMtwEeQxptBkB2EviqG-A,22726
-redis_benchmarks_specification-0.1.299.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-redis_benchmarks_specification-0.1.299.dist-info/entry_points.txt,sha256=x5WBXCZsnDRTZxV7SBGmC65L2k-ygdDOxV8vuKN00Nk,715
-redis_benchmarks_specification-0.1.299.dist-info/RECORD,,
+redis_benchmarks_specification-0.1.301.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+redis_benchmarks_specification-0.1.301.dist-info/METADATA,sha256=C3ytp_y6zvhWnBn06yKTK1XZgSySycj3i9IJU5Ry9q8,22726
+redis_benchmarks_specification-0.1.301.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+redis_benchmarks_specification-0.1.301.dist-info/entry_points.txt,sha256=x5WBXCZsnDRTZxV7SBGmC65L2k-ygdDOxV8vuKN00Nk,715
+redis_benchmarks_specification-0.1.301.dist-info/RECORD,,