redis-benchmarks-specification 0.1.338__py3-none-any.whl → 0.1.339__py3-none-any.whl

redis_benchmarks_specification/__builder__/builder.py

@@ -429,6 +429,79 @@ def builder_process_stream(
                 if b"server_name" in testDetails:
                     server_name = testDetails[b"server_name"].decode()
 
+                # Check if artifacts already exist before building
+                prefix = f"github_org={github_org}/github_repo={github_repo}/git_branch={str(git_branch)}/git_version={str(git_version)}/git_hash={str(git_hash)}"
+
+                # Create a comprehensive build signature that includes all build-affecting parameters
+                import hashlib
+
+                build_signature_parts = [
+                    str(id),  # build config ID
+                    str(build_command),  # build command
+                    str(build_vars_str),  # environment variables
+                    str(compiler),  # compiler
+                    str(cpp_compiler),  # C++ compiler
+                    str(build_image),  # build image
+                    str(build_os),  # OS
+                    str(build_arch),  # architecture
+                    ",".join(sorted(build_artifacts)),  # artifacts list
+                ]
+                build_signature = hashlib.sha256(
+                    ":".join(build_signature_parts).encode()
+                ).hexdigest()[:16]
+
+                # Check if all artifacts already exist
+                all_artifacts_exist = True
+                artifact_keys = {}
+                for artifact in build_artifacts:
+                    bin_key = f"zipped:artifacts:{prefix}:{id}:{build_signature}:{artifact}.zip"
+                    artifact_keys[artifact] = bin_key
+                    if not conn.exists(bin_key):
+                        all_artifacts_exist = False
+                        break
+
+                if all_artifacts_exist:
+                    logging.info(
+                        f"Artifacts for {git_hash}:{id} with build signature {build_signature} already exist, reusing them"
+                    )
+                    # Skip build and reuse existing artifacts
+                    build_stream_fields, result = generate_benchmark_stream_request(
+                        id,
+                        conn,
+                        run_image,
+                        build_arch,
+                        testDetails,
+                        build_os,
+                        build_artifacts,
+                        build_command,
+                        build_config_metadata,
+                        build_image,
+                        build_vars_str,
+                        compiler,
+                        cpp_compiler,
+                        git_branch,
+                        git_hash,
+                        git_timestamp_ms,
+                        git_version,
+                        pull_request,
+                        None,  # redis_temporary_dir not needed for reuse
+                        tests_groups_regexp,
+                        tests_priority_lower_limit,
+                        tests_priority_upper_limit,
+                        tests_regexp,
+                        ".*",  # command_regexp - default to all commands
+                        use_git_timestamp,
+                        server_name,
+                        github_org,
+                        github_repo,
+                        artifact_keys,  # Pass existing artifact keys
+                    )
+                    continue  # Skip to next build spec
+
+                logging.info(
+                    f"Building artifacts for {git_hash}:{id} with build signature {build_signature}"
+                )
+
                 build_start_datetime = datetime.datetime.utcnow()
                 logging.info(
                     "Using the following build command {}.".format(build_command)
@@ -507,6 +580,7 @@ def builder_process_stream(
                     server_name,
                     github_org,
                     github_repo,
+                    None,  # existing_artifact_keys - None for new builds
                 )
                 if result is True:
                     benchmark_stream_id = conn.xadd(
@@ -648,6 +722,7 @@ def generate_benchmark_stream_request(
     server_name="redis",
     github_org="redis",
     github_repo="redis",
+    existing_artifact_keys=None,
 ):
     build_stream_fields = {
         "id": id,
@@ -691,21 +766,50 @@ def generate_benchmark_stream_request(
     if git_timestamp_ms is not None:
         build_stream_fields["git_timestamp_ms"] = git_timestamp_ms
 
-    prefix = f"github_org={github_org}/github_repo={github_repo}/git_branch={str(git_branch)}/git_version={str(git_version)}/git_hash={str(git_hash)}"
-    for artifact in build_artifacts:
-        bin_key = f"zipped:artifacts:{prefix}:{id}:{artifact}.zip"
-        if artifact == "redisearch.so":
-            bin_artifact = open(
-                f"{redis_temporary_dir}modules/redisearch/src/bin/linux-x64-release/search-community/{artifact}",
-                "rb",
-            ).read()
-        else:
-            bin_artifact = open(f"{redis_temporary_dir}src/{artifact}", "rb").read()
-        bin_artifact_len = len(bytes(bin_artifact))
-        assert bin_artifact_len > 0
-        conn.set(bin_key, bytes(bin_artifact), ex=REDIS_BINS_EXPIRE_SECS)
-        build_stream_fields[artifact] = bin_key
-        build_stream_fields["{}_len_bytes".format(artifact)] = bin_artifact_len
+    if existing_artifact_keys is not None:
+        # Use existing artifact keys (for reuse case)
+        for artifact in build_artifacts:
+            bin_key = existing_artifact_keys[artifact]
+            build_stream_fields[artifact] = bin_key
+            # Get the length from the existing artifact
+            bin_artifact_len = conn.strlen(bin_key)
+            build_stream_fields["{}_len_bytes".format(artifact)] = bin_artifact_len
+    else:
+        # Build new artifacts and store them
+        prefix = f"github_org={github_org}/github_repo={github_repo}/git_branch={str(git_branch)}/git_version={str(git_version)}/git_hash={str(git_hash)}"
+
+        # Create build signature for new artifacts
+        import hashlib
+
+        build_signature_parts = [
+            str(id),  # build config ID
+            str(build_command),  # build command
+            str(build_vars_str),  # environment variables
+            str(compiler),  # compiler
+            str(cpp_compiler),  # C++ compiler
+            str(build_image),  # build image
+            str(build_os),  # OS
+            str(build_arch),  # architecture
+            ",".join(sorted(build_artifacts)),  # artifacts list
+        ]
+        build_signature = hashlib.sha256(
+            ":".join(build_signature_parts).encode()
+        ).hexdigest()[:16]
+
+        for artifact in build_artifacts:
+            bin_key = f"zipped:artifacts:{prefix}:{id}:{build_signature}:{artifact}.zip"
+            if artifact == "redisearch.so":
+                bin_artifact = open(
+                    f"{redis_temporary_dir}modules/redisearch/src/bin/linux-x64-release/search-community/{artifact}",
+                    "rb",
+                ).read()
+            else:
+                bin_artifact = open(f"{redis_temporary_dir}src/{artifact}", "rb").read()
+            bin_artifact_len = len(bytes(bin_artifact))
+            assert bin_artifact_len > 0
+            conn.set(bin_key, bytes(bin_artifact), ex=REDIS_BINS_EXPIRE_SECS)
+            build_stream_fields[artifact] = bin_key
+            build_stream_fields["{}_len_bytes".format(artifact)] = bin_artifact_len
     result = True
     if b"platform" in testDetails:
         build_stream_fields["platform"] = testDetails[b"platform"]

redis_benchmarks_specification/__cli__/cli.py

@@ -104,6 +104,12 @@ def trigger_tests_dockerhub_cli_command_logic(args, project_name, project_versio
         0,
         10000,
         args.tests_regexp,
+        ".*",  # command_regexp
+        False,  # use_git_timestamp
+        "redis",  # server_name
+        "redis",  # github_org
+        "redis",  # github_repo
+        None,  # existing_artifact_keys
     )
     build_stream_fields["github_repo"] = args.gh_repo
     build_stream_fields["github_org"] = args.gh_org

redis_benchmarks_specification/__self_contained_coordinator__/runners.py

@@ -149,11 +149,7 @@ def reset_consumer_group_to_latest(conn, running_platform):
 
     try:
         # Set the consumer group position to '$' (latest) to skip all existing messages
-        conn.xgroup_setid(
-            STREAM_KEYNAME_NEW_BUILD_EVENTS,
-            consumer_group_name,
-            id="$"
-        )
+        conn.xgroup_setid(STREAM_KEYNAME_NEW_BUILD_EVENTS, consumer_group_name, id="$")
         logging.info(
             f"Reset consumer group {consumer_group_name} position to latest - will only process new messages"
         )

redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py

@@ -655,13 +655,17 @@ def main():
     # Clear pending messages and reset consumer group position by default (unless explicitly skipped)
     if not args.skip_clear_pending_on_startup:
         consumer_pos = args.consumer_pos
-        logging.info("Clearing pending messages and resetting consumer group position on startup (default behavior)")
+        logging.info(
+            "Clearing pending messages and resetting consumer group position on startup (default behavior)"
+        )
         clear_pending_messages_for_consumer(
             gh_event_conn, running_platform, consumer_pos
         )
         reset_consumer_group_to_latest(gh_event_conn, running_platform)
     else:
-        logging.info("Skipping pending message cleanup and consumer group reset as requested")
+        logging.info(
+            "Skipping pending message cleanup and consumer group reset as requested"
+        )
 
     stream_id = None
     docker_client = docker.from_env()

{redis_benchmarks_specification-0.1.338.dist-info → redis_benchmarks_specification-0.1.339.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: redis-benchmarks-specification
-Version: 0.1.338
+Version: 0.1.339
 Summary: The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute.
 Author: filipecosta90
 Author-email: filipecosta.90@gmail.com

{redis_benchmarks_specification-0.1.338.dist-info → redis_benchmarks_specification-0.1.339.dist-info}/RECORD

@@ -4,11 +4,11 @@ redis_benchmarks_specification/__api__/api.py,sha256=k_CMICtMm1z8jY3hByaL0hIr_5v
 redis_benchmarks_specification/__api__/app.py,sha256=JzQm84DjIVdfLbDO423BJbrds6gFzMbA0syRkHE_aUU,7063
 redis_benchmarks_specification/__builder__/Readme.md,sha256=O6MV_J3OSgzW-ir2TbukP8Vhkm_LOzQJJndG1Cykqic,111
 redis_benchmarks_specification/__builder__/__init__.py,sha256=l-G1z-t6twUgi8QLueqoTQLvJmv3hJoEYskGm6H7L6M,83
-redis_benchmarks_specification/__builder__/builder.py,sha256=lAoEQ8ab9AWstYcpF2hoixZ_HFmMKf9Icwzc0WV0t_I,29867
+redis_benchmarks_specification/__builder__/builder.py,sha256=NiigX_UPeCulCBfrgZvntLIFRfgHe-j43CqWKSlvguk,34555
 redis_benchmarks_specification/__builder__/schema.py,sha256=1wcmyVJBcWrBvK58pghN9NCoWLCO3BzPsmdKWYfkVog,584
 redis_benchmarks_specification/__cli__/__init__.py,sha256=l-G1z-t6twUgi8QLueqoTQLvJmv3hJoEYskGm6H7L6M,83
 redis_benchmarks_specification/__cli__/args.py,sha256=X7VlHJvX3n85ZPUQFoovmaFDnY4t7irUrDLf07QAfaA,7430
-redis_benchmarks_specification/__cli__/cli.py,sha256=iTjINQ-RV_q2ovq1neSoRCAggpGdeP5mX3_1aFxSScY,22001
+redis_benchmarks_specification/__cli__/cli.py,sha256=GMd_Swn6HA8JAFd7hokLsLO3_F2qnd_2eavVQ66M1lk,22204
 redis_benchmarks_specification/__cli__/stats.py,sha256=r9JIfwGCSR3maozYbDZfZrkthNFQSs0xIymS86yZ6Iw,55574
 redis_benchmarks_specification/__common__/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 redis_benchmarks_specification/__common__/builder_schema.py,sha256=kfDpRIk7NkJrb5qj9jzsBhLVNO7K_W2Clumj4pxrkG8,5938
@@ -36,8 +36,8 @@ redis_benchmarks_specification/__self_contained_coordinator__/cpuset.py,sha256=s
 redis_benchmarks_specification/__self_contained_coordinator__/docker.py,sha256=09SyAfqlzs1KG9ZAajClNWtiNk4Jqzd--4-m3n1rLjU,3156
 redis_benchmarks_specification/__self_contained_coordinator__/post_processing.py,sha256=sVLKNnWdAqYY9DjVdqRC5tDaIrVSaI3Ca7w8-DQ-LRM,776
 redis_benchmarks_specification/__self_contained_coordinator__/prepopulation.py,sha256=1UeFr2T1ZQBcHCSd4W1ZtaWgXyFPfjLyDi_DgDc1eTA,2957
-redis_benchmarks_specification/__self_contained_coordinator__/runners.py,sha256=CXCyYb5G8IBHrm_6Czi2ArPQO7Pd1NJPGUYn836WadE,32872
-redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py,sha256=Jpcby63kpVAOx0HiYzehJd6-gS5b-LNnUz0TRAdQ48M,112752
+redis_benchmarks_specification/__self_contained_coordinator__/runners.py,sha256=1bpGiybmeQrdHh-z-fAyMvzOggZk4_MNHMTWrAfGwQU,32826
+redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py,sha256=BhOSyinsF8AkaPtR5jjB8vj3VTC0UwyBtCY7iGkWw_I,112796
 redis_benchmarks_specification/__setups__/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 redis_benchmarks_specification/__setups__/topologies.py,sha256=xQ1IJkcTji_ZjLiJd3vOxZpvbNtBLZw9cPkw5hGJKHU,481
 redis_benchmarks_specification/__spec__/__init__.py,sha256=l-G1z-t6twUgi8QLueqoTQLvJmv3hJoEYskGm6H7L6M,83
@@ -282,8 +282,8 @@ redis_benchmarks_specification/test-suites/memtier_benchmark-playbook-session-st
 redis_benchmarks_specification/test-suites/memtier_benchmark-playbook-session-storage-1k-sessions.yml,sha256=2egtIxPxCze2jlbAfgsk4v9JSQHNMoPLbDWFEW8olDg,7006
 redis_benchmarks_specification/test-suites/template.txt,sha256=ezqGiRPOvuSDO0iG7GEf-AGXNfHbgXI89_G0RUEzL88,481
 redis_benchmarks_specification/vector-search-test-suites/vector_db_benchmark_test.yml,sha256=PD7ow-k4Ll2BkhEC3aIqiaCZt8Hc4aJIp96Lw3J3mcI,791
-redis_benchmarks_specification-0.1.338.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-redis_benchmarks_specification-0.1.338.dist-info/METADATA,sha256=Ak3-Pw7UjP7ftlTuKAivQBA_MuZhMBIRQNFsqI6XLiM,22768
-redis_benchmarks_specification-0.1.338.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-redis_benchmarks_specification-0.1.338.dist-info/entry_points.txt,sha256=x5WBXCZsnDRTZxV7SBGmC65L2k-ygdDOxV8vuKN00Nk,715
-redis_benchmarks_specification-0.1.338.dist-info/RECORD,,
+redis_benchmarks_specification-0.1.339.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+redis_benchmarks_specification-0.1.339.dist-info/METADATA,sha256=xFGR8MNuEXNuxwQJKb2SRDKcXKkWeN64LPxQpD4RG5o,22768
+redis_benchmarks_specification-0.1.339.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+redis_benchmarks_specification-0.1.339.dist-info/entry_points.txt,sha256=x5WBXCZsnDRTZxV7SBGmC65L2k-ygdDOxV8vuKN00Nk,715
+redis_benchmarks_specification-0.1.339.dist-info/RECORD,,