redis-benchmarks-specification 0.1.340__py3-none-any.whl → 0.2.0__py3-none-any.whl

redis_benchmarks_specification/__builder__/builder.py

@@ -28,6 +28,7 @@ from redis_benchmarks_specification.__common__.env import (
     SPECS_PATH_SETUPS,
     STREAM_GH_EVENTS_COMMIT_BUILDERS_CG,
     STREAM_KEYNAME_NEW_BUILD_EVENTS,
+    get_arch_specific_stream_name,
     REDIS_HEALTH_CHECK_INTERVAL,
     REDIS_SOCKET_TIMEOUT,
     REDIS_BINS_EXPIRE_SECS,
@@ -371,6 +372,11 @@ def builder_process_stream(
                     )
                 )
             return previous_id, new_builds_count, build_stream_fields_arr
+        else:
+            logging.info(
+                "No arch info found on the stream. Using default arch {}.".format(arch)
+            )
+            build_request_arch = arch
 
         home = str(Path.home())
         if b"git_hash" in testDetails:
@@ -532,7 +538,7 @@ def builder_process_stream(
                     server_name = testDetails[b"server_name"].decode()
 
                 # Check if artifacts already exist before building
-                prefix = f"github_org={github_org}/github_repo={github_repo}/git_branch={str(git_branch)}/git_version={str(git_version)}/git_hash={str(git_hash)}"
+                prefix = f"build_spec={build_spec}/github_org={github_org}/github_repo={github_repo}/git_branch={str(git_branch)}/git_version={str(git_version)}/git_hash={str(git_hash)}"
 
                 # Create a comprehensive build signature that includes all build-affecting parameters
                 import hashlib
@@ -598,6 +604,32 @@ def builder_process_stream(
                         github_repo,
                         artifact_keys,  # Pass existing artifact keys
                     )
+                    # Add to benchmark stream even when reusing artifacts
+                    if result is True:
+                        arch_specific_stream = get_arch_specific_stream_name(build_arch)
+                        logging.info(
+                            f"Adding reused build work to architecture-specific stream: {arch_specific_stream}"
+                        )
+                        benchmark_stream_id = conn.xadd(
+                            arch_specific_stream, build_stream_fields
+                        )
+                        logging.info(
+                            "successfully reused build variant {} for redis git_sha {}. Stream id: {}".format(
+                                id, git_hash, benchmark_stream_id
+                            )
+                        )
+                        streamId_decoded = streamId.decode()
+                        benchmark_stream_id_decoded = benchmark_stream_id.decode()
+                        builder_list_completed = (
+                            f"builder:{streamId_decoded}:builds_completed"
+                        )
+                        conn.lpush(builder_list_completed, benchmark_stream_id_decoded)
+                        conn.expire(builder_list_completed, REDIS_BINS_EXPIRE_SECS)
+                        logging.info(
+                            f"Adding information of build->benchmark stream info in list {builder_list_completed}. Adding benchmark stream id: {benchmark_stream_id_decoded}"
+                        )
+                        build_stream_fields_arr.append(build_stream_fields)
+                        new_builds_count = new_builds_count + 1
                     continue  # Skip to next build spec
 
                 logging.info(
@@ -685,8 +717,12 @@ def builder_process_stream(
                     None,  # existing_artifact_keys - None for new builds
                 )
                 if result is True:
+                    arch_specific_stream = get_arch_specific_stream_name(build_arch)
+                    logging.info(
+                        f"Adding new build work to architecture-specific stream: {arch_specific_stream}"
+                    )
                     benchmark_stream_id = conn.xadd(
-                        STREAM_KEYNAME_NEW_BUILD_EVENTS, build_stream_fields
+                        arch_specific_stream, build_stream_fields
                     )
                     logging.info(
                         "sucessfully built build variant {} for redis git_sha {}. Stream id: {}".format(

redis_benchmarks_specification/__cli__/args.py

@@ -138,7 +138,7 @@ def spec_cli_args(parser):
     parser.add_argument("--gh_repo", type=str, default="redis")
     parser.add_argument("--server_name", type=str, default=None)
     parser.add_argument("--run_image", type=str, default="redis")
-    parser.add_argument("--build_arch", type=str, default=None)
+    parser.add_argument("--arch", type=str, default="amd64")
     parser.add_argument("--id", type=str, default="dockerhub")
     parser.add_argument("--mnt_point", type=str, default="")
     parser.add_argument("--trigger-unstable-commits", type=bool, default=True)

redis_benchmarks_specification/__cli__/cli.py

@@ -44,6 +44,7 @@ from redis_benchmarks_specification.__common__.env import (
     STREAM_KEYNAME_GH_EVENTS_COMMIT,
     STREAM_GH_EVENTS_COMMIT_BUILDERS_CG,
     STREAM_KEYNAME_NEW_BUILD_EVENTS,
+    get_arch_specific_stream_name,
 )
 from redis_benchmarks_specification.__common__.package import (
     get_version_string,
@@ -84,7 +85,7 @@ def trigger_tests_dockerhub_cli_command_logic(args, project_name, project_versio
         args.id,
         conn,
         args.run_image,
-        args.build_arch,
+        args.arch,
         testDetails,
         "n/a",
         [],
@@ -124,9 +125,12 @@ def trigger_tests_dockerhub_cli_command_logic(args, project_name, project_versio
         store_airgap_image_redis(conn, docker_client, args.run_image)
 
     if result is True:
-        benchmark_stream_id = conn.xadd(
-            STREAM_KEYNAME_NEW_BUILD_EVENTS, build_stream_fields
+        # Use architecture-specific stream
+        arch_specific_stream = get_arch_specific_stream_name(args.arch)
+        logging.info(
+            f"CLI adding work to architecture-specific stream: {arch_specific_stream}"
         )
+        benchmark_stream_id = conn.xadd(arch_specific_stream, build_stream_fields)
         logging.info(
             "sucessfully requested a new run {}. Stream id: {}".format(
                 build_stream_fields, benchmark_stream_id
@@ -438,9 +442,9 @@ def trigger_tests_cli_command_logic(args, project_name, project_version):
             commit_dict["tests_groups_regexp"] = tests_groups_regexp
             commit_dict["github_org"] = args.gh_org
             commit_dict["github_repo"] = args.gh_repo
-            if args.build_arch is not None:
-                commit_dict["build_arch"] = args.build_arch
-                commit_dict["arch"] = args.build_arch
+            if args.arch is not None:
+                commit_dict["build_arch"] = args.arch
+                commit_dict["arch"] = args.arch
             if args.server_name is not None and args.server_name != "":
                 commit_dict["server_name"] = args.server_name
             if args.build_artifacts != "":

redis_benchmarks_specification/__common__/env.py

@@ -32,6 +32,20 @@ STREAM_KEYNAME_NEW_BUILD_EVENTS = os.getenv(
     "STREAM_KEYNAME_NEW_BUILD_EVENTS", "oss:api:gh/redis/redis/builds"
 )
 
+
+# Function to get architecture-specific build events stream name
+def get_arch_specific_stream_name(arch):
+    """Get architecture-specific stream name for build events"""
+    base_stream = STREAM_KEYNAME_NEW_BUILD_EVENTS
+    if arch in ["amd64", "x86_64"]:
+        return f"{base_stream}:amd64"
+    elif arch in ["arm64", "aarch64"]:
+        return f"{base_stream}:arm64"
+    else:
+        # Fallback to base stream for unknown architectures
+        return base_stream
+
+
 STREAM_GH_NEW_BUILD_RUNNERS_CG = os.getenv(
     "STREAM_GH_NEW_BUILD_RUNNERS_CG", "runners-cg:redis/redis/commits"
 )

redis_benchmarks_specification/__self_contained_coordinator__/runners.py

@@ -38,6 +38,7 @@ from redisbench_admin.utils.results import post_process_benchmark_results
 
 from redis_benchmarks_specification.__common__.env import (
     STREAM_KEYNAME_NEW_BUILD_EVENTS,
+    get_arch_specific_stream_name,
     STREAM_GH_NEW_BUILD_RUNNERS_CG,
     S3_BUCKET_NAME,
 )
@@ -71,12 +72,16 @@ from redis_benchmarks_specification.__self_contained_coordinator__.prepopulation
 )
 
 
-def build_runners_consumer_group_create(conn, running_platform, id="$"):
+def build_runners_consumer_group_create(conn, running_platform, arch="amd64", id="$"):
     consumer_group_name = get_runners_consumer_group_name(running_platform)
+    arch_specific_stream = get_arch_specific_stream_name(arch)
     logging.info("Will use consumer group named {}.".format(consumer_group_name))
+    logging.info(
+        "Will read from architecture-specific stream: {}.".format(arch_specific_stream)
+    )
     try:
         conn.xgroup_create(
-            STREAM_KEYNAME_NEW_BUILD_EVENTS,
+            arch_specific_stream,
             consumer_group_name,
             mkstream=True,
             id=id,
@@ -99,17 +104,23 @@ def get_runners_consumer_group_name(running_platform):
     return consumer_group_name
 
 
-def clear_pending_messages_for_consumer(conn, running_platform, consumer_pos):
+def clear_pending_messages_for_consumer(
+    conn, running_platform, consumer_pos, arch="amd64"
+):
     """Clear all pending messages for a specific consumer on startup"""
     consumer_group_name = get_runners_consumer_group_name(running_platform)
     consumer_name = "{}-self-contained-proc#{}".format(
         consumer_group_name, consumer_pos
     )
+    arch_specific_stream = get_arch_specific_stream_name(arch)
+    logging.info(
+        f"Clearing pending messages from architecture-specific stream: {arch_specific_stream}"
+    )
 
     try:
         # Get pending messages for this specific consumer
         pending_info = conn.xpending_range(
-            STREAM_KEYNAME_NEW_BUILD_EVENTS,
+            arch_specific_stream,
             consumer_group_name,
             min="-",
             max="+",
@@ -125,7 +136,7 @@ def clear_pending_messages_for_consumer(conn, running_platform, consumer_pos):
 
             # Acknowledge all pending messages to clear them
             ack_count = conn.xack(
-                STREAM_KEYNAME_NEW_BUILD_EVENTS, consumer_group_name, *message_ids
+                arch_specific_stream, consumer_group_name, *message_ids
             )
 
             logging.info(
@@ -143,15 +154,19 @@ def clear_pending_messages_for_consumer(conn, running_platform, consumer_pos):
         logging.error(f"Unexpected error clearing pending messages: {e}")
 
 
-def reset_consumer_group_to_latest(conn, running_platform):
+def reset_consumer_group_to_latest(conn, running_platform, arch="amd64"):
     """Reset the consumer group position to only read new messages (skip old ones)"""
     consumer_group_name = get_runners_consumer_group_name(running_platform)
+    arch_specific_stream = get_arch_specific_stream_name(arch)
+    logging.info(
+        f"Resetting consumer group position for architecture-specific stream: {arch_specific_stream}"
+    )
 
     try:
         # Set the consumer group position to '$' (latest) to skip all existing messages
-        conn.xgroup_setid(STREAM_KEYNAME_NEW_BUILD_EVENTS, consumer_group_name, id="$")
+        conn.xgroup_setid(arch_specific_stream, consumer_group_name, id="$")
         logging.info(
-            f"Reset consumer group {consumer_group_name} position to latest - will only process new messages"
+            f"Reset consumer group {consumer_group_name} position to latest on stream {arch_specific_stream} - will only process new messages"
         )
 
     except redis.exceptions.ResponseError as e:

redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py

@@ -102,6 +102,7 @@ from redisbench_admin.utils.results import post_process_benchmark_results
 
 from redis_benchmarks_specification.__common__.env import (
     STREAM_KEYNAME_NEW_BUILD_EVENTS,
+    get_arch_specific_stream_name,
     S3_BUCKET_NAME,
 )
 from redis_benchmarks_specification.__common__.spec import (
@@ -650,7 +651,7 @@ def main():
 
     logging.info("checking build spec requirements")
     running_platform = args.platform_name
-    build_runners_consumer_group_create(gh_event_conn, running_platform)
+    build_runners_consumer_group_create(gh_event_conn, running_platform, args.arch)
 
     # Clear pending messages and reset consumer group position by default (unless explicitly skipped)
     if not args.skip_clear_pending_on_startup:
@@ -659,9 +660,9 @@ def main():
             "Clearing pending messages and resetting consumer group position on startup (default behavior)"
         )
         clear_pending_messages_for_consumer(
-            gh_event_conn, running_platform, consumer_pos
+            gh_event_conn, running_platform, consumer_pos, args.arch
         )
-        reset_consumer_group_to_latest(gh_event_conn, running_platform)
+        reset_consumer_group_to_latest(gh_event_conn, running_platform, args.arch)
     else:
         logging.info(
             "Skipping pending message cleanup and consumer group reset as requested"
@@ -819,10 +820,15 @@ def self_contained_coordinator_blocking_read(
             get_runners_consumer_group_name(platform_name), consumer_name
         )
     )
+    # Use architecture-specific stream
+    arch_specific_stream = get_arch_specific_stream_name(arch)
+    logging.info(
+        f"Reading work from architecture-specific stream: {arch_specific_stream}"
+    )
     newTestInfo = github_event_conn.xreadgroup(
         get_runners_consumer_group_name(platform_name),
         consumer_name,
-        {STREAM_KEYNAME_NEW_BUILD_EVENTS: stream_id},
+        {arch_specific_stream: stream_id},
         count=1,
         block=0,
     )
@@ -874,8 +880,9 @@ def self_contained_coordinator_blocking_read(
         num_process_test_suites = num_process_test_suites + total_test_suite_runs
 
         # Always acknowledge the message, even if it was filtered out
+        arch_specific_stream = get_arch_specific_stream_name(arch)
         ack_reply = github_event_conn.xack(
-            STREAM_KEYNAME_NEW_BUILD_EVENTS,
+            arch_specific_stream,
             get_runners_consumer_group_name(platform_name),
             stream_id,
         )

{redis_benchmarks_specification-0.1.340.dist-info → redis_benchmarks_specification-0.2.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: redis-benchmarks-specification
-Version: 0.1.340
+Version: 0.2.0
 Summary: The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute.
 Author: filipecosta90
 Author-email: filipecosta.90@gmail.com

{redis_benchmarks_specification-0.1.340.dist-info → redis_benchmarks_specification-0.2.0.dist-info}/RECORD

@@ -4,15 +4,15 @@ redis_benchmarks_specification/__api__/api.py,sha256=k_CMICtMm1z8jY3hByaL0hIr_5v
 redis_benchmarks_specification/__api__/app.py,sha256=JzQm84DjIVdfLbDO423BJbrds6gFzMbA0syRkHE_aUU,7063
 redis_benchmarks_specification/__builder__/Readme.md,sha256=O6MV_J3OSgzW-ir2TbukP8Vhkm_LOzQJJndG1Cykqic,111
 redis_benchmarks_specification/__builder__/__init__.py,sha256=l-G1z-t6twUgi8QLueqoTQLvJmv3hJoEYskGm6H7L6M,83
-redis_benchmarks_specification/__builder__/builder.py,sha256=5GhZ76itxcXfaHr7YtNrJp2zcYC_XHskpqZjIbMh3UI,38862
+redis_benchmarks_specification/__builder__/builder.py,sha256=vG6Cp0SAAgvZt9zyjTQB_-mSUGbPBbM3Ue9hJpb7oYM,40984
 redis_benchmarks_specification/__builder__/schema.py,sha256=1wcmyVJBcWrBvK58pghN9NCoWLCO3BzPsmdKWYfkVog,584
 redis_benchmarks_specification/__cli__/__init__.py,sha256=l-G1z-t6twUgi8QLueqoTQLvJmv3hJoEYskGm6H7L6M,83
-redis_benchmarks_specification/__cli__/args.py,sha256=X7VlHJvX3n85ZPUQFoovmaFDnY4t7irUrDLf07QAfaA,7430
-redis_benchmarks_specification/__cli__/cli.py,sha256=GMd_Swn6HA8JAFd7hokLsLO3_F2qnd_2eavVQ66M1lk,22204
+redis_benchmarks_specification/__cli__/args.py,sha256=y_FHtkjhRKsU532sutlXPeXgtFZu7KGOp7lTSi1C5uc,7427
+redis_benchmarks_specification/__cli__/cli.py,sha256=0S3Lwci-oxpPJMekPIFeG5XJiSF53rpa5sRnS-aAZnA,22416
 redis_benchmarks_specification/__cli__/stats.py,sha256=r9JIfwGCSR3maozYbDZfZrkthNFQSs0xIymS86yZ6Iw,55574
 redis_benchmarks_specification/__common__/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 redis_benchmarks_specification/__common__/builder_schema.py,sha256=kfDpRIk7NkJrb5qj9jzsBhLVNO7K_W2Clumj4pxrkG8,5938
-redis_benchmarks_specification/__common__/env.py,sha256=kvJ8Ll-fvI_Tc0vynrzUEr22TqnJizzvJ4Lu9RjNr_M,3119
+redis_benchmarks_specification/__common__/env.py,sha256=5E5LS8o7_wISw-gX11bcMJx1A9DHo0j3cbV383lMFeo,3591
 redis_benchmarks_specification/__common__/github.py,sha256=9TZtnISsSgXTSAN_VQejo5YRPDPhlU0gjxgKGPw_sP8,10699
 redis_benchmarks_specification/__common__/package.py,sha256=4uVt1BAZ999LV2rZkq--Tk6otAVIf9YR3g3KGeUpiW4,834
 redis_benchmarks_specification/__common__/runner.py,sha256=TKMUFJ3nLSfmSU7P_ok9oM5-pI4L4tFxsWLUWaUHhbI,16733
@@ -36,8 +36,8 @@ redis_benchmarks_specification/__self_contained_coordinator__/cpuset.py,sha256=s
 redis_benchmarks_specification/__self_contained_coordinator__/docker.py,sha256=09SyAfqlzs1KG9ZAajClNWtiNk4Jqzd--4-m3n1rLjU,3156
 redis_benchmarks_specification/__self_contained_coordinator__/post_processing.py,sha256=sVLKNnWdAqYY9DjVdqRC5tDaIrVSaI3Ca7w8-DQ-LRM,776
 redis_benchmarks_specification/__self_contained_coordinator__/prepopulation.py,sha256=1UeFr2T1ZQBcHCSd4W1ZtaWgXyFPfjLyDi_DgDc1eTA,2957
-redis_benchmarks_specification/__self_contained_coordinator__/runners.py,sha256=1bpGiybmeQrdHh-z-fAyMvzOggZk4_MNHMTWrAfGwQU,32826
-redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py,sha256=vtBJzzH6eq2v7M7EK1Ve5EZyUfpgSFKTHQfIUY8agFE,113065
+redis_benchmarks_specification/__self_contained_coordinator__/runners.py,sha256=1SvzQI8tCh5dBvd87PMpwEgycyhEcbr7NviqTxXl65I,33444
+redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py,sha256=dHmRfiEk2UgI-fg1m7Yavy8RoBtAe8B0XueOV7r_IJg,113386
 redis_benchmarks_specification/__setups__/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 redis_benchmarks_specification/__setups__/topologies.py,sha256=xQ1IJkcTji_ZjLiJd3vOxZpvbNtBLZw9cPkw5hGJKHU,481
 redis_benchmarks_specification/__spec__/__init__.py,sha256=l-G1z-t6twUgi8QLueqoTQLvJmv3hJoEYskGm6H7L6M,83
@@ -282,8 +282,8 @@ redis_benchmarks_specification/test-suites/memtier_benchmark-playbook-session-st
 redis_benchmarks_specification/test-suites/memtier_benchmark-playbook-session-storage-1k-sessions.yml,sha256=2egtIxPxCze2jlbAfgsk4v9JSQHNMoPLbDWFEW8olDg,7006
 redis_benchmarks_specification/test-suites/template.txt,sha256=ezqGiRPOvuSDO0iG7GEf-AGXNfHbgXI89_G0RUEzL88,481
 redis_benchmarks_specification/vector-search-test-suites/vector_db_benchmark_test.yml,sha256=PD7ow-k4Ll2BkhEC3aIqiaCZt8Hc4aJIp96Lw3J3mcI,791
-redis_benchmarks_specification-0.1.340.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-redis_benchmarks_specification-0.1.340.dist-info/METADATA,sha256=DDrmuBuSVQ7qSeKz2z-mgLP8oNV0DNDuAfO3a_oxZ0I,22768
-redis_benchmarks_specification-0.1.340.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-redis_benchmarks_specification-0.1.340.dist-info/entry_points.txt,sha256=x5WBXCZsnDRTZxV7SBGmC65L2k-ygdDOxV8vuKN00Nk,715
-redis_benchmarks_specification-0.1.340.dist-info/RECORD,,
+redis_benchmarks_specification-0.2.0.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+redis_benchmarks_specification-0.2.0.dist-info/METADATA,sha256=Ozd9_XjKBhEa3mF9FzmLA87rYAMXEP6sXtmgMQ3BRRw,22766
+redis_benchmarks_specification-0.2.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+redis_benchmarks_specification-0.2.0.dist-info/entry_points.txt,sha256=x5WBXCZsnDRTZxV7SBGmC65L2k-ygdDOxV8vuKN00Nk,715
+redis_benchmarks_specification-0.2.0.dist-info/RECORD,,