redis-benchmarks-specification 0.1.207__tar.gz → 0.1.209__tar.gz

{redis_benchmarks_specification-0.1.207 → redis_benchmarks_specification-0.1.209}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: redis-benchmarks-specification
-Version: 0.1.207
+Version: 0.1.209
 Summary: The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute.
 Author: filipecosta90
 Author-email: filipecosta.90@gmail.com
@@ -9,6 +9,7 @@ Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Requires-Dist: Flask (>=2.0.3,<3.0.0)
 Requires-Dist: Flask-HTTPAuth (>=4.4.0,<5.0.0)
 Requires-Dist: GitPython (>=3.1.20,<4.0.0)

{redis_benchmarks_specification-0.1.207 → redis_benchmarks_specification-0.1.209}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redis-benchmarks-specification"
-version = "0.1.207"
+version = "0.1.209"
 description = "The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute."
 authors = ["filipecosta90 <filipecosta.90@gmail.com>","Redis Performance Group <performance@redis.com>"]
 readme = "Readme.md"

{redis_benchmarks_specification-0.1.207 → redis_benchmarks_specification-0.1.209}/redis_benchmarks_specification/__cli__/cli.py

@@ -73,6 +73,10 @@ def get_commits_by_branch(args, repo):
     commits = []
     for commit in repo.iter_commits():
         commit_datetime = commit.committed_datetime
+        git_timestamp_ms = int(
+            datetime.datetime.utcfromtimestamp(commit_datetime.timestamp()).timestamp()
+            * 1000
+        )
         if (
             args.from_date
             <= datetime.datetime.utcfromtimestamp(commit_datetime.timestamp())
@@ -87,6 +91,7 @@ def get_commits_by_branch(args, repo):
                         "git_branch": repo.active_branch.name,
                         "commit_summary": commit.summary,
                         "commit_datetime": str(commit_datetime),
+                        "git_timestamp_ms": git_timestamp_ms,
                     }
                 )
     return commits, total_commits
@@ -107,6 +112,14 @@ def get_commits_by_tags(args, repo):
 
     tags = sorted(repo.tags, key=lambda t: t.commit.committed_datetime)
     for tag in tags:
+
+        git_timestamp_ms = int(
+            datetime.datetime.utcfromtimestamp(
+                tag.commit.committed_datetime.timestamp()
+            ).timestamp()
+            * 1000
+        )
+
         if (
             args.from_date
             <= datetime.datetime.utcfromtimestamp(
@@ -137,6 +150,7 @@ def get_commits_by_tags(args, repo):
                             "git_version": git_version,
                             "commit_summary": tag.commit.summary,
                             "commit_datetime": commit_datetime,
+                            "git_timestamp_ms": git_timestamp_ms,
                         }
                     )
             except packaging.version.InvalidVersion:
@@ -265,6 +279,7 @@ def trigger_tests_cli_command_logic(args, project_name, project_version):
     for cdict in commits:
         commit_hash = cdict["git_hash"]
         commit_summary = cdict["commit_summary"]
+        commit_datetime = cdict["commit_datetime"]
         match_obj = re.search(hash_regexp_string, commit_hash)
         if match_obj is None:
             logging.info(
@@ -274,9 +289,7 @@ def trigger_tests_cli_command_logic(args, project_name, project_version):
             )
         else:
             print(
-                "Commit with hash: {} added. summary: {}".format(
-                    commit_hash, commit_summary
-                )
+                f"Commit with hash: {commit_hash} from {commit_datetime} added. summary: {commit_summary}"
             )
             filtered_hash_commits.append(cdict)
 

{redis_benchmarks_specification-0.1.207 → redis_benchmarks_specification-0.1.209}/redis_benchmarks_specification/__common__/builder_schema.py

@@ -107,7 +107,8 @@ def get_commit_dict_from_sha(
             commit.commit.author.date.timestamp() * 1000.0
         )
     else:
-        use_git_timestamp = False
+        if "git_timestamp_ms" not in commit_dict:
+            use_git_timestamp = False
     commit_dict["use_git_timestamp"] = str(use_git_timestamp)
     commit_dict["git_hash"] = git_hash
     if gh_branch is not None:

{redis_benchmarks_specification-0.1.207 → redis_benchmarks_specification-0.1.209}/redis_benchmarks_specification/__compare__/compare.py

@@ -16,6 +16,9 @@ import os
 from tqdm import tqdm
 import argparse
 
+from io import StringIO
+import sys
+
 from redis_benchmarks_specification.__common__.github import (
     update_comment_if_needed,
     create_new_pr_comment,
@@ -583,7 +586,11 @@ def compute_regression_table(
         )
     (
         detected_regressions,
-        table,
+        table_full,
+        table_stable,
+        table_unstable,
+        table_improvements,
+        table_regressions,
         total_improvements,
         total_regressions,
         total_stable,
@@ -619,13 +626,60 @@ def compute_regression_table(
             baseline_str, comparison_str
         )
     )
-    writer = MarkdownTableWriter(
-        table_name="Comparison between {} and {}.\n\nTime Period from {}. (environment used: {})\n".format(
-            baseline_str,
-            comparison_str,
-            from_human_str,
-            baseline_deployment_name,
-        ),
+
+    table_output = "# Comparison between {} and {}.\n\nTime Period from {}. (environment used: {})\n\n".format(
+        baseline_str,
+        comparison_str,
+        from_human_str,
+        baseline_deployment_name,
+    )
+
+    if total_regressions > 0:
+        old_stdout = sys.stdout
+        sys.stdout = mystdout = StringIO()
+        table_output += "#### Regressions Table\n\n"
+        writer_regressions = MarkdownTableWriter(
+            table_name="",
+            headers=[
+                "Test Case",
+                "Baseline {} (median obs. +- std.dev)".format(baseline_str),
+                "Comparison {} (median obs. +- std.dev)".format(comparison_str),
+                "% change ({})".format(metric_mode),
+                "Note",
+            ],
+            value_matrix=table_regressions,
+        )
+        writer_regressions.dump(mystdout, False)
+        table_output += mystdout.getvalue()
+        table_output += "\n\n"
+        mystdout.close()
+        sys.stdout = old_stdout
+
+    if total_improvements > 0:
+        old_stdout = sys.stdout
+        sys.stdout = mystdout = StringIO()
+        table_output += "#### Improvements Table\n\n"
+        writer_regressions = MarkdownTableWriter(
+            table_name="",
+            headers=[
+                "Test Case",
+                "Baseline {} (median obs. +- std.dev)".format(baseline_str),
+                "Comparison {} (median obs. +- std.dev)".format(comparison_str),
+                "% change ({})".format(metric_mode),
+                "Note",
+            ],
+            value_matrix=table_improvements,
+        )
+        writer_regressions.dump(mystdout, False)
+        table_output += mystdout.getvalue()
+        table_output += "\n\n"
+        mystdout.close()
+        sys.stdout = old_stdout
+
+    old_stdout = sys.stdout
+    sys.stdout = mystdout = StringIO()
+    writer_full = MarkdownTableWriter(
+        table_name="",
         headers=[
             "Test Case",
             "Baseline {} (median obs. +- std.dev)".format(baseline_str),
@@ -633,21 +687,15 @@ def compute_regression_table(
             "% change ({})".format(metric_mode),
             "Note",
         ],
-        value_matrix=table,
+        value_matrix=table_full,
     )
-    table_output = ""
-
-    from io import StringIO
-    import sys
-
-    old_stdout = sys.stdout
-    sys.stdout = mystdout = StringIO()
+    table_output += "<details>\n  <summary>Full Results table:</summary>\n\n"
 
-    writer.dump(mystdout, False)
+    writer_full.dump(mystdout, False)
 
     sys.stdout = old_stdout
-
-    table_output = mystdout.getvalue()
+    table_output += mystdout.getvalue()
+    table_output += "\n</details>\n"
 
     return (
         detected_regressions,
@@ -742,7 +790,11 @@ def from_rts_to_regression_table(
     running_platform=None,
 ):
     print_all = print_regressions_only is False and print_improvements_only is False
-    table = []
+    table_full = []
+    table_unstable = []
+    table_stable = []
+    table_regressions = []
+    table_improvements = []
     detected_regressions = []
     total_improvements = 0
     total_stable = 0
@@ -752,6 +804,10 @@ def from_rts_to_regression_table(
     noise_waterline = 3
     progress = tqdm(unit="benchmark time-series", total=len(test_names))
     for test_name in test_names:
+        compare_version = "v0.1.208"
+        github_link = "https://github.com/redis/redis-benchmarks-specification/blob"
+        test_path = f"redis_benchmarks_specification/test-suites/{test_name}.yml"
+        test_link = f"[{test_name}]({github_link}/{compare_version}/{test_path})"
         multi_value_baseline = check_multi_value_filter(baseline_str)
         multi_value_comparison = check_multi_value_filter(comparison_str)
 
@@ -940,6 +996,25 @@ def from_rts_to_regression_table(
                 total_unstable += 1
 
             should_add_line = False
+            line = get_line(
+                baseline_v_str,
+                comparison_v_str,
+                note,
+                percentage_change,
+                test_link,
+            )
+            if detected_regression:
+                table_regressions.append(line)
+
+            if detected_improvement:
+                table_improvements.append(line)
+
+            if unstable:
+                table_unstable.append(line)
+            else:
+                if not detected_regression and not detected_improvement:
+                    table_stable.append(line)
+
             if print_regressions_only and detected_regression:
                 should_add_line = True
             if print_improvements_only and detected_improvement:
@@ -951,17 +1026,14 @@ def from_rts_to_regression_table(
 
             if should_add_line:
                 total_comparison_points = total_comparison_points + 1
-                add_line(
-                    baseline_v_str,
-                    comparison_v_str,
-                    note,
-                    percentage_change,
-                    table,
-                    test_name,
-                )
+                table_full.append(line)
     return (
         detected_regressions,
-        table,
+        table_full,
+        table_stable,
+        table_unstable,
+        table_improvements,
+        table_regressions,
         total_improvements,
         total_regressions,
         total_stable,
@@ -1037,6 +1109,23 @@ def get_test_names_from_db(rts, tags_regex_string, test_names, used_key):
     return test_names
 
 
+def get_line(
+    baseline_v_str,
+    comparison_v_str,
+    note,
+    percentage_change,
+    test_name,
+):
+    percentage_change_str = "{:.1f}% ".format(percentage_change)
+    return [
+        test_name,
+        baseline_v_str,
+        comparison_v_str,
+        percentage_change_str,
+        note.strip(),
+    ]
+
+
 def add_line(
     baseline_v_str,
     comparison_v_str,
@@ -1045,15 +1134,14 @@ def add_line(
     table,
     test_name,
 ):
-    percentage_change_str = "{:.1f}% ".format(percentage_change)
     table.append(
-        [
-            test_name,
+        get_line(
             baseline_v_str,
             comparison_v_str,
-            percentage_change_str,
-            note.strip(),
-        ]
+            note,
+            percentage_change,
+            test_name,
+        )
     )
 
 

{redis_benchmarks_specification-0.1.207 → redis_benchmarks_specification-0.1.209}/redis_benchmarks_specification/__spec__/cli.py

@@ -92,13 +92,12 @@ def cli_command_logic(args, project_name, project_version):
     total_commits = 0
     if args.use_branch:
         for commit in repo.iter_commits():
-            if (
-                args.from_date
-                <= datetime.datetime.utcfromtimestamp(
-                    commit.committed_datetime.timestamp()
-                )
-                <= args.to_date
-            ):
+
+            commit_datetime_utc = datetime.datetime.utcfromtimestamp(
+                commit.committed_datetime.timestamp()
+            )
+            git_timestamp_ms = int(commit_datetime_utc.timestamp() * 1000)
+            if args.from_date <= commit_datetime_utc <= args.to_date:
                 if (
                     args.last_n > 0 and total_commits < args.last_n
                 ) or args.last_n == -1:
@@ -109,6 +108,7 @@ def cli_command_logic(args, project_name, project_version):
                             "git_hash": commit.hexsha,
                             "git_branch": repo.active_branch.name,
                             "commit_summary": commit.summary,
+                            "git_timestamp_ms": git_timestamp_ms,
                         }
                     )
     if args.use_tags: