redis-benchmarks-specification 0.1.89__tar.gz → 0.1.91__tar.gz

{redis_benchmarks_specification-0.1.89 → redis_benchmarks_specification-0.1.91}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: redis-benchmarks-specification
-Version: 0.1.89
+Version: 0.1.91
 Summary: The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute.
 Author: filipecosta90
 Author-email: filipecosta.90@gmail.com

{redis_benchmarks_specification-0.1.89 → redis_benchmarks_specification-0.1.91}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redis-benchmarks-specification"
-version = "0.1.89"
+version = "0.1.91"
 description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
 authors = ["filipecosta90 <filipecosta.90@gmail.com>","Redis Performance Group <performance@redis.com>"]
 readme = "Readme.md"

{redis_benchmarks_specification-0.1.89 → redis_benchmarks_specification-0.1.91}/redis_benchmarks_specification/__common__/github.py

@@ -222,6 +222,7 @@ def generate_benchmark_started_pr_comment(
     benchmark_stream_id,
     total_pending,
     total_benchmarks,
+    total_failed,
 ):
     comment_body = (
         "### CE Performance Automation : step 2 of 2 (benchmark) RUNNING...\n\n"
@@ -231,12 +232,15 @@ def generate_benchmark_started_pr_comment(
     )
 
     completed = total_benchmarks - total_pending
+    successful = completed - total_failed
     comment_body += (
         f"Status: {markdown_progress_bar(completed,total_benchmarks,80)} completed.\n\n"
     )
     comment_body += f"In total will run {total_benchmarks} benchmarks.\n"
     comment_body += f"    - {total_pending} pending.\n"
-    comment_body += f"    - {completed} completed.\n"
+    comment_body += f"    - {completed} completed:\n"
+    comment_body += f"      - {successful} successful.\n"
+    comment_body += f"      - {total_failed} failed.\n"
 
     if not isinstance(benchmark_stream_id, str):
         benchmark_stream_id = benchmark_stream_id.decode()

{redis_benchmarks_specification-0.1.89 → redis_benchmarks_specification-0.1.91}/redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py

@@ -531,7 +531,9 @@ def process_self_contained_coordinator_stream(
 
                 stream_test_list_pending = f"ci.benchmarks.redis/ci/redis/redis:benchmarks:{stream_id}:{running_platform}:tests_pending"
                 stream_test_list_running = f"ci.benchmarks.redis/ci/redis/redis:benchmarks:{stream_id}:{running_platform}:tests_running"
+                stream_test_list_failed = f"ci.benchmarks.redis/ci/redis/redis:benchmarks:{stream_id}:{running_platform}:tests_failed"
                 stream_test_list_completed = f"ci.benchmarks.redis/ci/redis/redis:benchmarks:{stream_id}:{running_platform}:tests_completed"
+
                 for test_file in testsuite_spec_files:
                     if defaults_filename in test_file:
                         continue
@@ -573,8 +575,9 @@ def process_self_contained_coordinator_stream(
                         )
                     filtered_test_files.append(test_file)
                 pending_tests = len(filtered_test_files)
+                failed_tests = 0
                 comment_body = generate_benchmark_started_pr_comment(
-                    stream_id, pending_tests, len(filtered_test_files)
+                    stream_id, pending_tests, len(filtered_test_files), failed_tests
                 )
                 # update on github if needed
                 if is_actionable_pr:
@@ -1011,31 +1014,46 @@ def process_self_contained_coordinator_stream(
                                     )
 
                                 dataset_load_duration_seconds = 0
+                                try:
+                                    exporter_datasink_common(
+                                        benchmark_config,
+                                        benchmark_duration_seconds,
+                                        build_variant_name,
+                                        datapoint_time_ms,
+                                        dataset_load_duration_seconds,
+                                        datasink_conn,
+                                        datasink_push_results_redistimeseries,
+                                        git_branch,
+                                        git_version,
+                                        metadata,
+                                        redis_conns,
+                                        results_dict,
+                                        running_platform,
+                                        setup_name,
+                                        setup_type,
+                                        test_name,
+                                        tf_github_org,
+                                        tf_github_repo,
+                                        tf_triggering_env,
+                                        topology_spec_name,
+                                        default_metrics,
+                                    )
+                                    r.shutdown(save=False)
+
+                                except redis.exceptions.ConnectionError as e:
+                                    logging.critical(
+                                        "Some unexpected exception was caught during metric fetching. Skipping it..."
+                                    )
+                                    logging.critical(
+                                        f"Exception type: {type(e).__name__}"
+                                    )
+                                    logging.critical(f"Exception message: {str(e)}")
+                                    logging.critical("Traceback details:")
+                                    logging.critical(traceback.format_exc())
+                                    print("-" * 60)
+                                    traceback.print_exc(file=sys.stdout)
+                                    print("-" * 60)
 
-                                exporter_datasink_common(
-                                    benchmark_config,
-                                    benchmark_duration_seconds,
-                                    build_variant_name,
-                                    datapoint_time_ms,
-                                    dataset_load_duration_seconds,
-                                    datasink_conn,
-                                    datasink_push_results_redistimeseries,
-                                    git_branch,
-                                    git_version,
-                                    metadata,
-                                    redis_conns,
-                                    results_dict,
-                                    running_platform,
-                                    setup_name,
-                                    setup_type,
-                                    test_name,
-                                    tf_github_org,
-                                    tf_github_repo,
-                                    tf_triggering_env,
-                                    topology_spec_name,
-                                    default_metrics,
-                                )
-                                r.shutdown(save=False)
                                 test_result = True
                                 total_test_suite_runs = total_test_suite_runs + 1
 
@@ -1054,10 +1072,20 @@ def process_self_contained_coordinator_stream(
                                     logging.critical("Printing redis container log....")
 
                                     print("-" * 60)
-
-                                    print(
-                                        redis_container.logs(stdout=True, stderr=True)
-                                    )
+                                    try:
+                                        print(
+                                            redis_container.logs(
+                                                stdout=True, stderr=True
+                                            )
+                                        )
+                                    except docker.errors.NotFound:
+                                        logging.info(
+                                            "When trying to stop DB container with id {} and image {} it was already stopped".format(
+                                                redis_container.id,
+                                                redis_container.image,
+                                            )
+                                        )
+                                    pass
 
                                     print("-" * 60)
 
@@ -1100,12 +1128,21 @@ def process_self_contained_coordinator_stream(
                     conn.lrem(stream_test_list_running, 1, test_name)
                     conn.lpush(stream_test_list_completed, test_name)
                     conn.expire(stream_test_list_completed, REDIS_BINS_EXPIRE_SECS)
+                    if test_result is False:
+                        conn.lpush(stream_test_list_failed, test_name)
+                        failed_tests = failed_tests + 1
+                        logging.warning(
+                            f"updating key {stream_test_list_failed} with the failed test: {test_name}. Total failed tests {failed_tests}."
+                        )
                     pending_tests = pending_tests - 1
 
                     # update on github if needed
                     if is_actionable_pr:
                         comment_body = generate_benchmark_started_pr_comment(
-                            stream_id, pending_tests, len(filtered_test_files)
+                            stream_id,
+                            pending_tests,
+                            len(filtered_test_files),
+                            failed_tests,
                         )
                         update_comment_if_needed(
                             auto_approve_github,

{redis_benchmarks_specification-0.1.89 → redis_benchmarks_specification-0.1.91}/redis_benchmarks_specification/test-suites/memtier_benchmark-1Mkeys-string-get-2MB.yml

@@ -9,7 +9,7 @@ dbconfig:
   preload_tool:
     run_image: redislabs/memtier_benchmark:edge
     tool: memtier_benchmark
-    arguments: '"--data-size" "2000000" "--ratio" "1:0" "--key-pattern" "P:P" "-c" "50" "-t" "2" "--hide-histogram" "--key-minimum" "1"'
+    arguments: '"--data-size" "2000000" "--ratio" "1:0" "--key-pattern" "P:P" "-c" "50" "-t" "2" "--hide-histogram" "--key-minimum" "1" "--key-minimum" "1000"'
   resources:
     requests:
       memory: 2g
@@ -22,7 +22,7 @@ build-variants:
 clientconfig:
   run_image: redislabs/memtier_benchmark:edge
   tool: memtier_benchmark
-  arguments: --data-size 2000000  --ratio 0:1 --key-pattern R:R -c 25 -t 4 --hide-histogram --test-time 180
+  arguments: --data-size 2000000 --key-minimum 1 --key-maximum 1000 --ratio 0:1 --key-pattern R:R -c 25 -t 4 --hide-histogram --test-time 180
   resources:
     requests:
       cpus: '4'