redis-benchmarks-specification 0.1.332__py3-none-any.whl → 0.1.334__py3-none-any.whl

redis_benchmarks_specification/__runner__/runner.py

@@ -182,7 +182,7 @@ def validate_benchmark_metrics(
             "all_stats.totals.ops/sec",
         ]
 
-        latency_patterns = ["latency", "p50", "p95", "p99", "p999", "usec", "msec"]
+        latency_patterns = ["p50", "p95", "p99", "p999", "percentile"]
 
         validation_errors = []
 
@@ -195,6 +195,22 @@ def validate_benchmark_metrics(
             elif isinstance(data, (int, float)):
                 metric_path_lower = path.lower()
 
+                # Skip Waits metrics as they can legitimately be 0
+                if "waits" in metric_path_lower:
+                    return
+
+                # Skip general latency metrics that can legitimately be 0
+                # Only validate specific percentile latencies (p50, p95, etc.)
+                if any(
+                    pattern in metric_path_lower
+                    for pattern in [
+                        "average latency",
+                        "totals.latency",
+                        "all_stats.totals.latency",
+                    ]
+                ):
+                    return
+
                 # Check throughput metrics
                 for pattern in throughput_patterns:
                     if pattern in metric_path_lower:

redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py

@@ -122,6 +122,7 @@ _reset_queue_requested = False
 _exclusive_hardware = False
 _http_auth_username = None
 _http_auth_password = None
+_flush_timestamp = None
 
 
 class CoordinatorHTTPHandler(BaseHTTPRequestHandler):
@@ -190,6 +191,22 @@ class CoordinatorHTTPHandler(BaseHTTPRequestHandler):
                 "service": "redis-benchmarks-self-contained-coordinator",
             }
             self.wfile.write(json.dumps(response).encode())
+
+        elif parsed_path.path == "/containers":
+            # Check for stuck containers
+            stuck_containers = self._check_stuck_containers()
+
+            self.send_response(200)
+            self.send_header("Content-type", "application/json")
+            self.end_headers()
+            response = {
+                "status": "success",
+                "stuck_containers": stuck_containers,
+                "total_stuck": len(stuck_containers),
+                "timestamp": datetime.datetime.utcnow().isoformat(),
+            }
+            self.wfile.write(json.dumps(response).encode())
+
         else:
             self.send_response(404)
             self.send_header("Content-type", "application/json")
@@ -203,7 +220,7 @@ class CoordinatorHTTPHandler(BaseHTTPRequestHandler):
             self._send_auth_required()
             return
 
-        global _reset_queue_requested
+        global _reset_queue_requested, _flush_timestamp
 
         parsed_path = urlparse(self.path)
 
@@ -240,12 +257,175 @@ class CoordinatorHTTPHandler(BaseHTTPRequestHandler):
                 self.send_header("Content-type", "application/json")
                 self.end_headers()
                 self.wfile.write(json.dumps({"error": str(e)}).encode())
+
+        elif parsed_path.path == "/flush":
+            try:
+                # Read request body (optional)
+                content_length = int(self.headers.get("Content-Length", 0))
+                if content_length > 0:
+                    post_data = self.rfile.read(content_length)
+                    try:
+                        request_data = json.loads(post_data.decode())
+                    except json.JSONDecodeError:
+                        request_data = {}
+                else:
+                    request_data = {}
+
+                # Record flush timestamp
+                flush_time = datetime.datetime.utcnow()
+                _flush_timestamp = flush_time
+
+                logging.info(
+                    "Flush requested via HTTP endpoint - stopping all containers and processes"
+                )
+
+                # Perform flush cleanup
+                self._perform_flush_cleanup()
+
+                self.send_response(200)
+                self.send_header("Content-type", "application/json")
+                self.end_headers()
+                response = {
+                    "status": "success",
+                    "message": "Flush completed - all containers stopped and processes killed",
+                    "flush_timestamp": flush_time.isoformat(),
+                    "timestamp": datetime.datetime.utcnow().isoformat(),
+                }
+                self.wfile.write(json.dumps(response).encode())
+
+            except Exception as e:
+                logging.error(f"Error during flush operation: {e}")
+                self.send_response(500)
+                self.send_header("Content-type", "application/json")
+                self.end_headers()
+                response = {
+                    "status": "error",
+                    "message": f"Flush failed: {str(e)}",
+                    "timestamp": datetime.datetime.utcnow().isoformat(),
+                }
+                self.wfile.write(json.dumps(response).encode())
+
         else:
             self.send_response(404)
             self.send_header("Content-type", "application/json")
             self.end_headers()
             self.wfile.write(json.dumps({"error": "Not found"}).encode())
 
+    def _perform_flush_cleanup(self):
+        """Perform flush cleanup: stop all containers and kill memtier processes"""
+        import subprocess
+
+        # Kill all memtier processes
+        try:
+            logging.info("Killing all memtier_benchmark processes")
+            result = subprocess.run(
+                ["pkill", "-f", "memtier_benchmark"], capture_output=True, text=True
+            )
+            if result.returncode == 0:
+                logging.info("Successfully killed memtier_benchmark processes")
+            else:
+                logging.info("No memtier_benchmark processes found to kill")
+
+            result = subprocess.run(
+                ["pkill", "-f", "memtier"], capture_output=True, text=True
+            )
+            if result.returncode == 0:
+                logging.info("Successfully killed memtier processes")
+            else:
+                logging.info("No memtier processes found to kill")
+        except Exception as e:
+            logging.warning(f"Error killing memtier processes: {e}")
+
+        # Stop all Docker containers with force if needed
+        try:
+            logging.info("Stopping all Docker containers")
+            client = docker.from_env()
+            containers = client.containers.list()
+
+            if not containers:
+                logging.info("No running containers found")
+                return
+
+            logging.info(f"Found {len(containers)} running containers")
+
+            for container in containers:
+                try:
+                    # Get container info
+                    created_time = container.attrs["Created"]
+                    uptime = (
+                        datetime.datetime.utcnow()
+                        - datetime.datetime.fromisoformat(
+                            created_time.replace("Z", "+00:00")
+                        )
+                    )
+
+                    logging.info(
+                        f"Stopping container: {container.name} ({container.id[:12]}) - uptime: {uptime}"
+                    )
+
+                    # Try graceful stop first
+                    container.stop(timeout=10)
+                    logging.info(f"Successfully stopped container: {container.name}")
+
+                except Exception as e:
+                    logging.warning(f"Error stopping container {container.name}: {e}")
+                    try:
+                        # Force kill if graceful stop failed
+                        logging.info(f"Force killing container: {container.name}")
+                        container.kill()
+                        logging.info(
+                            f"Successfully force killed container: {container.name}"
+                        )
+                    except Exception as e2:
+                        logging.error(
+                            f"Failed to force kill container {container.name}: {e2}"
+                        )
+
+        except Exception as e:
+            logging.warning(f"Error accessing Docker client: {e}")
+
+        logging.info("Flush cleanup completed")
+
+    def _check_stuck_containers(self, max_hours=2):
+        """Check for containers running longer than max_hours and return info"""
+        try:
+            client = docker.from_env()
+            containers = client.containers.list()
+            stuck_containers = []
+
+            for container in containers:
+                try:
+                    created_time = container.attrs["Created"]
+                    uptime = (
+                        datetime.datetime.utcnow()
+                        - datetime.datetime.fromisoformat(
+                            created_time.replace("Z", "+00:00")
+                        )
+                    )
+                    uptime_hours = uptime.total_seconds() / 3600
+
+                    if uptime_hours > max_hours:
+                        stuck_containers.append(
+                            {
+                                "name": container.name,
+                                "id": container.id[:12],
+                                "image": (
+                                    container.image.tags[0]
+                                    if container.image.tags
+                                    else "unknown"
+                                ),
+                                "uptime_hours": round(uptime_hours, 2),
+                                "status": container.status,
+                            }
+                        )
+                except Exception as e:
+                    logging.warning(f"Error checking container {container.name}: {e}")
+
+            return stuck_containers
+        except Exception as e:
+            logging.warning(f"Error accessing Docker client: {e}")
+            return []
+
 
 def start_http_server(port=8080):
     """Start the HTTP server in a separate thread"""
@@ -256,9 +436,13 @@ def start_http_server(port=8080):
             logging.info(f"Starting HTTP server on port {port}")
             logging.info(f"Available endpoints:")
             logging.info(f"  GET  /ping - Health check")
+            logging.info(f"  GET  /containers - Check for stuck containers")
             logging.info(
                 f"  POST /reset-queue - Reset pending streams and skip running tests"
             )
+            logging.info(
+                f"  POST /flush - Stop all containers and processes, ignore work before flush time"
+            )
             server.serve_forever()
         except Exception as e:
             logging.error(f"HTTP server error: {e}")
@@ -784,6 +968,23 @@ def process_self_contained_coordinator_stream(
                 git_timestamp_ms,
                 run_arch,
             ) = extract_build_info_from_streamdata(testDetails)
+
+            # Check if this work should be ignored due to flush
+            global _flush_timestamp
+            if (
+                _flush_timestamp is not None
+                and use_git_timestamp
+                and git_timestamp_ms is not None
+            ):
+                # Convert flush timestamp to milliseconds for comparison
+                flush_timestamp_ms = int(_flush_timestamp.timestamp() * 1000)
+                if git_timestamp_ms < flush_timestamp_ms:
+                    logging.info(
+                        f"Ignoring work with git_timestamp_ms {git_timestamp_ms} "
+                        f"(before flush timestamp {flush_timestamp_ms}). Stream id: {stream_id}"
+                    )
+                    return stream_id, False, 0
+
             tf_github_org = default_github_org
             if b"github_org" in testDetails:
                 tf_github_org = testDetails[b"github_org"].decode()
@@ -954,7 +1155,9 @@ def process_self_contained_coordinator_stream(
                     command_regexp,
                 )
 
-                logging.info(f"Adding {len(filtered_test_files)} tests to pending test list")
+                logging.info(
+                    f"Adding {len(filtered_test_files)} tests to pending test list"
+                )
 
                 # Use pipeline for efficient bulk operations
                 pipeline = github_event_conn.pipeline()
@@ -977,7 +1180,9 @@ def process_self_contained_coordinator_stream(
                 pipeline.expire(stream_test_list_pending, REDIS_BINS_EXPIRE_SECS)
                 pipeline.execute()
 
-                logging.info(f"Successfully added {len(test_names_added)} tests to pending test list in key {stream_test_list_pending}")
+                logging.info(
+                    f"Successfully added {len(test_names_added)} tests to pending test list in key {stream_test_list_pending}"
+                )
 
                 pending_tests = len(filtered_test_files)
                 failed_tests = 0
@@ -1335,25 +1540,111 @@ def process_self_contained_coordinator_stream(
                                 )
                                 # run the benchmark
                                 benchmark_start_time = datetime.datetime.now()
+
+                                # Calculate container timeout
+                                container_timeout = 300  # 5 minutes default
+                                buffer_timeout = 60  # Default buffer
+
+                                # Try to extract test time from command and add buffer
+                                import re
+
+                                test_time_match = re.search(
+                                    r"--?test-time[=\s]+(\d+)", benchmark_command_str
+                                )
+                                if test_time_match:
+                                    test_time = int(test_time_match.group(1))
+                                    container_timeout = test_time + buffer_timeout
+                                    logging.info(
+                                        f"Set container timeout to {container_timeout}s (test-time: {test_time}s + {buffer_timeout}s buffer)"
+                                    )
+                                else:
+                                    logging.info(
+                                        f"Using default container timeout: {container_timeout}s"
+                                    )
+
                                 try:
-                                    client_container_stdout = (
-                                        docker_client.containers.run(
-                                            image=client_container_image,
-                                            volumes={
-                                                temporary_dir_client: {
-                                                    "bind": client_mnt_point,
-                                                    "mode": "rw",
-                                                },
+                                    # Start container with detach=True to enable timeout handling
+                                    container = docker_client.containers.run(
+                                        image=client_container_image,
+                                        volumes={
+                                            temporary_dir_client: {
+                                                "bind": client_mnt_point,
+                                                "mode": "rw",
                                             },
-                                            auto_remove=True,
-                                            privileged=True,
-                                            working_dir=benchmark_tool_workdir,
-                                            command=benchmark_command_str,
-                                            network_mode="host",
-                                            detach=False,
-                                            cpuset_cpus=client_cpuset_cpus,
-                                        )
+                                        },
+                                        auto_remove=False,  # Don't auto-remove so we can get logs if timeout
+                                        privileged=True,
+                                        working_dir=benchmark_tool_workdir,
+                                        command=benchmark_command_str,
+                                        network_mode="host",
+                                        detach=True,  # Detach to enable timeout
+                                        cpuset_cpus=client_cpuset_cpus,
                                     )
+
+                                    logging.info(
+                                        f"Started container {container.name} ({container.id[:12]}) with {container_timeout}s timeout"
+                                    )
+
+                                    # Wait for container with timeout
+                                    try:
+                                        result = container.wait(
+                                            timeout=container_timeout
+                                        )
+                                        client_container_stdout = container.logs(
+                                            stdout=True, stderr=False
+                                        ).decode("utf-8")
+                                        container_stderr = container.logs(
+                                            stdout=False, stderr=True
+                                        ).decode("utf-8")
+
+                                        # Check exit code
+                                        if result["StatusCode"] != 0:
+                                            logging.error(
+                                                f"Container exited with code {result['StatusCode']}"
+                                            )
+                                            logging.error(
+                                                f"Container stderr: {container_stderr}"
+                                            )
+                                            raise docker.errors.ContainerError(
+                                                container,
+                                                result["StatusCode"],
+                                                benchmark_command_str,
+                                                client_container_stdout,
+                                                container_stderr,
+                                            )
+
+                                        logging.info(
+                                            f"Container {container.name} completed successfully"
+                                        )
+
+                                    except Exception as timeout_error:
+                                        if "timeout" in str(timeout_error).lower():
+                                            logging.error(
+                                                f"Container {container.name} timed out after {container_timeout}s"
+                                            )
+                                            # Get logs before killing
+                                            try:
+                                                timeout_logs = container.logs(
+                                                    stdout=True, stderr=True
+                                                ).decode("utf-8")
+                                                logging.error(
+                                                    f"Container logs before timeout: {timeout_logs}"
+                                                )
+                                            except:
+                                                pass
+                                            # Kill the container
+                                            container.kill()
+                                            raise Exception(
+                                                f"Container timed out after {container_timeout} seconds"
+                                            )
+                                        else:
+                                            raise timeout_error
+                                    finally:
+                                        # Clean up container
+                                        try:
+                                            container.remove(force=True)
+                                        except:
+                                            pass
                                 except docker.errors.ContainerError as e:
                                     logging.info(
                                         "stdout: {}".format(
@@ -2117,22 +2408,92 @@ def data_prepopulation_step(
         # run the benchmark
         preload_start_time = datetime.datetime.now()
 
-        client_container_stdout = docker_client.containers.run(
-            image=preload_image,
-            volumes={
-                temporary_dir: {
-                    "bind": client_mnt_point,
-                    "mode": "rw",
+        # Set preload timeout (preload can take longer than benchmarks)
+        preload_timeout = 1800  # 30 minutes default for data loading
+        logging.info(f"Starting preload container with {preload_timeout}s timeout")
+
+        try:
+            # Start container with detach=True to enable timeout handling
+            container = docker_client.containers.run(
+                image=preload_image,
+                volumes={
+                    temporary_dir: {
+                        "bind": client_mnt_point,
+                        "mode": "rw",
+                    },
                 },
-            },
-            auto_remove=True,
-            privileged=True,
-            working_dir=benchmark_tool_workdir,
-            command=preload_command_str,
-            network_mode="host",
-            detach=False,
-            cpuset_cpus=client_cpuset_cpus,
-        )
+                auto_remove=False,  # Don't auto-remove so we can get logs if timeout
+                privileged=True,
+                working_dir=benchmark_tool_workdir,
+                command=preload_command_str,
+                network_mode="host",
+                detach=True,  # Detach to enable timeout
+                cpuset_cpus=client_cpuset_cpus,
+            )
+
+            logging.info(
+                f"Started preload container {container.name} ({container.id[:12]}) with {preload_timeout}s timeout"
+            )
+
+            # Wait for container with timeout
+            try:
+                result = container.wait(timeout=preload_timeout)
+                client_container_stdout = container.logs(
+                    stdout=True, stderr=False
+                ).decode("utf-8")
+                container_stderr = container.logs(stdout=False, stderr=True).decode(
+                    "utf-8"
+                )
+
+                # Check exit code
+                if result["StatusCode"] != 0:
+                    logging.error(
+                        f"Preload container exited with code {result['StatusCode']}"
+                    )
+                    logging.error(f"Preload container stderr: {container_stderr}")
+                    raise docker.errors.ContainerError(
+                        container,
+                        result["StatusCode"],
+                        preload_command_str,
+                        client_container_stdout,
+                        container_stderr,
+                    )
+
+                logging.info(
+                    f"Preload container {container.name} completed successfully"
+                )
+
+            except Exception as timeout_error:
+                if "timeout" in str(timeout_error).lower():
+                    logging.error(
+                        f"Preload container {container.name} timed out after {preload_timeout}s"
+                    )
+                    # Get logs before killing
+                    try:
+                        timeout_logs = container.logs(stdout=True, stderr=True).decode(
+                            "utf-8"
+                        )
+                        logging.error(
+                            f"Preload container logs before timeout: {timeout_logs}"
+                        )
+                    except:
+                        pass
+                    # Kill the container
+                    container.kill()
+                    raise Exception(
+                        f"Preload container timed out after {preload_timeout} seconds"
+                    )
+                else:
+                    raise timeout_error
+            finally:
+                # Clean up container
+                try:
+                    container.remove(force=True)
+                except:
+                    pass
+        except Exception as e:
+            logging.error(f"Preload container failed: {e}")
+            raise e
 
         preload_end_time = datetime.datetime.now()
         preload_duration_seconds = calculate_client_tool_duration_and_check(

{redis_benchmarks_specification-0.1.332.dist-info → redis_benchmarks_specification-0.1.334.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: redis-benchmarks-specification
-Version: 0.1.332
+Version: 0.1.334
 Summary: The Redis benchmarks specification describes the cross-language/tools requirements and expectations to foster performance and observability standards around redis related technologies. Members from both industry and academia, including organizations and individuals are encouraged to contribute.
 Author: filipecosta90
 Author-email: filipecosta.90@gmail.com

{redis_benchmarks_specification-0.1.332.dist-info → redis_benchmarks_specification-0.1.334.dist-info}/RECORD

@@ -26,7 +26,7 @@ redis_benchmarks_specification/__init__.py,sha256=YQIEx2sLPPA0JR9OuCuMNMNtm-f_gq
 redis_benchmarks_specification/__runner__/__init__.py,sha256=l-G1z-t6twUgi8QLueqoTQLvJmv3hJoEYskGm6H7L6M,83
 redis_benchmarks_specification/__runner__/args.py,sha256=K3VGmBC0-9lSv9H6VDp0N-6FGMWvc_4H0pG_TOXN5u8,11312
 redis_benchmarks_specification/__runner__/remote_profiling.py,sha256=R7obNQju8mmY9oKkcndjI4aAuxi84OCLhDSqqaYu1SU,18610
-redis_benchmarks_specification/__runner__/runner.py,sha256=-BDFxOLgkFe4LvVX1FnqmuszuyRMR8AJZW0SvPX0utw,155496
+redis_benchmarks_specification/__runner__/runner.py,sha256=V6PnV5Tt2qupLcfJg0yhj_AqI5pIzFj-0EXc_JtYo84,156096
 redis_benchmarks_specification/__self_contained_coordinator__/__init__.py,sha256=l-G1z-t6twUgi8QLueqoTQLvJmv3hJoEYskGm6H7L6M,83
 redis_benchmarks_specification/__self_contained_coordinator__/args.py,sha256=1LePhRkDsoMPFclM_DoXBIoMBN8zcVoQMnm9wTK5Uqw,6961
 redis_benchmarks_specification/__self_contained_coordinator__/artifacts.py,sha256=OVHqJzDgeSSRfUSiKp1ZTAVv14PvSbk-5yJsAAoUfpw,936
@@ -37,7 +37,7 @@ redis_benchmarks_specification/__self_contained_coordinator__/docker.py,sha256=0
 redis_benchmarks_specification/__self_contained_coordinator__/post_processing.py,sha256=sVLKNnWdAqYY9DjVdqRC5tDaIrVSaI3Ca7w8-DQ-LRM,776
 redis_benchmarks_specification/__self_contained_coordinator__/prepopulation.py,sha256=1UeFr2T1ZQBcHCSd4W1ZtaWgXyFPfjLyDi_DgDc1eTA,2957
 redis_benchmarks_specification/__self_contained_coordinator__/runners.py,sha256=F11zO_ILnpmiVwTeCQnP5nDHQk3kNnajPftwKsbhlXE,30209
-redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py,sha256=3QOuaaNJ1HQl4Ld4ZtgweTu2P0Hu-l6cgp3GhGpU8rw,95332
+redis_benchmarks_specification/__self_contained_coordinator__/self_contained_coordinator.py,sha256=2QdjjUyGB-GCyyJtSe984GY634i3qE019qWCnDBuI0U,111825
 redis_benchmarks_specification/__setups__/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 redis_benchmarks_specification/__setups__/topologies.py,sha256=xQ1IJkcTji_ZjLiJd3vOxZpvbNtBLZw9cPkw5hGJKHU,481
 redis_benchmarks_specification/__spec__/__init__.py,sha256=l-G1z-t6twUgi8QLueqoTQLvJmv3hJoEYskGm6H7L6M,83
@@ -282,8 +282,8 @@ redis_benchmarks_specification/test-suites/memtier_benchmark-playbook-session-st
 redis_benchmarks_specification/test-suites/memtier_benchmark-playbook-session-storage-1k-sessions.yml,sha256=2egtIxPxCze2jlbAfgsk4v9JSQHNMoPLbDWFEW8olDg,7006
 redis_benchmarks_specification/test-suites/template.txt,sha256=ezqGiRPOvuSDO0iG7GEf-AGXNfHbgXI89_G0RUEzL88,481
 redis_benchmarks_specification/vector-search-test-suites/vector_db_benchmark_test.yml,sha256=PD7ow-k4Ll2BkhEC3aIqiaCZt8Hc4aJIp96Lw3J3mcI,791
-redis_benchmarks_specification-0.1.332.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-redis_benchmarks_specification-0.1.332.dist-info/METADATA,sha256=Y4eicJqinSR2jRYOPGMTUcn2sMXqW6XHhLJRDdDz4zc,22768
-redis_benchmarks_specification-0.1.332.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-redis_benchmarks_specification-0.1.332.dist-info/entry_points.txt,sha256=x5WBXCZsnDRTZxV7SBGmC65L2k-ygdDOxV8vuKN00Nk,715
-redis_benchmarks_specification-0.1.332.dist-info/RECORD,,
+redis_benchmarks_specification-0.1.334.dist-info/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+redis_benchmarks_specification-0.1.334.dist-info/METADATA,sha256=dE2ReU_PQYen1kcQ2YRh5AfPmkb1Hrx-j15Qj2vt508,22768
+redis_benchmarks_specification-0.1.334.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+redis_benchmarks_specification-0.1.334.dist-info/entry_points.txt,sha256=x5WBXCZsnDRTZxV7SBGmC65L2k-ygdDOxV8vuKN00Nk,715
+redis_benchmarks_specification-0.1.334.dist-info/RECORD,,