redis-benchmarks-specification 0.1.280__py3-none-any.whl → 0.1.282__py3-none-any.whl

redis_benchmarks_specification/__runner__/remote_profiling.py

@@ -3,7 +3,7 @@ Remote profiling utilities for Redis benchmark runner.
 
 This module provides functionality to trigger remote profiling of Redis processes
 via HTTP GET endpoints during benchmark execution. Profiles are collected in
-folded format for performance analysis.
+pprof binary format for performance analysis.
 """
 
 import datetime
@@ -19,10 +19,10 @@ import requests
 def extract_redis_pid(redis_conn) -> Optional[int]:
     """
     Extract Redis process ID from Redis INFO command.
-    
+
     Args:
         redis_conn: Redis connection object
-        
+
     Returns:
         Redis process ID as integer, or None if not found
     """
@@ -43,10 +43,10 @@ def extract_redis_pid(redis_conn) -> Optional[int]:
 def extract_redis_metadata(redis_conn) -> Dict[str, Any]:
     """
     Extract Redis metadata for profile comments.
-    
+
     Args:
         redis_conn: Redis connection object
-        
+
     Returns:
         Dictionary containing Redis metadata
     """
@@ -60,18 +60,20 @@ def extract_redis_metadata(redis_conn) -> Dict[str, Any]:
             "process_id": redis_info.get("process_id", "unknown"),
             "tcp_port": redis_info.get("tcp_port", "unknown"),
         }
-        
+
         # Use build_id if git_sha1 is empty or 0
         if metadata["redis_git_sha1"] in ("", 0, "0"):
             metadata["redis_git_sha1"] = metadata["redis_build_id"]
-            
-        logging.info(f"Extracted Redis metadata: version={metadata['redis_version']}, sha={metadata['redis_git_sha1']}, pid={metadata['process_id']}")
+
+        logging.info(
+            f"Extracted Redis metadata: version={metadata['redis_version']}, sha={metadata['redis_git_sha1']}, pid={metadata['process_id']}"
+        )
         return metadata
     except Exception as e:
         logging.error(f"Failed to extract Redis metadata: {e}")
         return {
             "redis_version": "unknown",
-            "redis_git_sha1": "unknown", 
+            "redis_git_sha1": "unknown",
             "redis_git_dirty": "unknown",
             "redis_build_id": "unknown",
             "process_id": "unknown",
@@ -82,16 +84,18 @@ def extract_redis_metadata(redis_conn) -> Dict[str, Any]:
 def calculate_profile_duration(benchmark_duration_seconds: int) -> int:
     """
     Calculate profiling duration based on benchmark duration.
-    
+
     Args:
         benchmark_duration_seconds: Expected benchmark duration in seconds
-        
+
     Returns:
         Profiling duration in seconds (minimum: benchmark duration, maximum: 30)
     """
     # Minimum duration is the benchmark duration, maximum is 30 seconds
     duration = min(max(benchmark_duration_seconds, 10), 30)
-    logging.info(f"Calculated profile duration: {duration}s (benchmark: {benchmark_duration_seconds}s)")
+    logging.info(
+        f"Calculated profile duration: {duration}s (benchmark: {benchmark_duration_seconds}s)"
+    )
     return duration
 
 
@@ -102,10 +106,10 @@ def trigger_remote_profile(
     duration: int,
     timeout: int = 60,
     username: Optional[str] = None,
-    password: Optional[str] = None
-) -> Optional[str]:
+    password: Optional[str] = None,
+) -> Optional[bytes]:
     """
-    Trigger remote profiling via HTTP GET request.
+    Trigger remote profiling via HTTP GET request using pprof endpoint.
 
     Args:
         host: Remote host address
@@ -117,13 +121,10 @@ def trigger_remote_profile(
         password: Optional password for HTTP basic authentication
 
     Returns:
-        Profile content in folded format, or None if failed
+        Profile content in pprof binary format, or None if failed
     """
-    url = f"http://{host}:{port}/debug/folded/profile"
-    params = {
-        "pid": pid,
-        "seconds": duration
-    }
+    url = f"http://{host}:{port}/debug/pprof/profile"
+    params = {"pid": pid, "seconds": duration}
 
     # Prepare authentication if provided
     auth = None
@@ -132,12 +133,14 @@ def trigger_remote_profile(
         logging.info(f"Using HTTP basic authentication with username: {username}")
 
     try:
-        logging.info(f"Triggering remote profile: {url} with PID={pid}, duration={duration}s")
+        logging.info(
+            f"Triggering remote profile: {url} with PID={pid}, duration={duration}s"
+        )
         response = requests.get(url, params=params, timeout=timeout, auth=auth)
         response.raise_for_status()
 
-        profile_content = response.text
-        logging.info(f"Successfully collected profile: {len(profile_content)} characters")
+        profile_content = response.content
+        logging.info(f"Successfully collected profile: {len(profile_content)} bytes")
         return profile_content
 
     except requests.exceptions.Timeout:
@@ -157,55 +160,65 @@ def trigger_remote_profile(
 
 
 def save_profile_with_metadata(
-    profile_content: str,
+    profile_content: bytes,
     benchmark_name: str,
     output_dir: str,
     redis_metadata: Dict[str, Any],
-    duration: int
+    duration: int,
 ) -> Optional[str]:
     """
-    Save profile content to file with metadata comments.
-    
+    Save profile content to file in pprof binary format.
+
     Args:
-        profile_content: Profile data in folded format
+        profile_content: Profile data in pprof binary format
         benchmark_name: Name of the benchmark
         output_dir: Output directory path
         redis_metadata: Redis metadata dictionary
         duration: Profiling duration in seconds
-        
+
     Returns:
         Path to saved file, or None if failed
     """
     try:
         # Create output directory if it doesn't exist
         Path(output_dir).mkdir(parents=True, exist_ok=True)
-        
-        # Generate filename
-        filename = f"{benchmark_name}.folded"
+
+        # Generate filename with .pb.gz extension
+        filename = f"{benchmark_name}.pb.gz"
         filepath = os.path.join(output_dir, filename)
-        
+
         # Generate timestamp
         timestamp = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-        
-        # Create metadata comment
-        metadata_comment = (
-            f"# profile from redis sha = {redis_metadata['redis_git_sha1']} "
+
+        # Write binary profile content directly
+        with open(filepath, "wb") as f:
+            f.write(profile_content)
+
+        # Create a separate metadata file
+        metadata_filename = f"{benchmark_name}.metadata.txt"
+        metadata_filepath = os.path.join(output_dir, metadata_filename)
+
+        metadata_content = (
+            f"Profile from redis sha = {redis_metadata['redis_git_sha1']} "
             f"and pid {redis_metadata['process_id']} for duration of {duration}s. "
-            f"collection in date {timestamp}\n"
-            f"# benchmark_name={benchmark_name}\n"
-            f"# redis_git_sha1={redis_metadata['redis_git_sha1']}\n"
-            f"# redis_version={redis_metadata['redis_version']}\n"
-            f"# redis_git_dirty={redis_metadata['redis_git_dirty']}\n"
+            f"Collection date: {timestamp}\n"
+            f"benchmark_name={benchmark_name}\n"
+            f"redis_git_sha1={redis_metadata['redis_git_sha1']}\n"
+            f"redis_version={redis_metadata['redis_version']}\n"
+            f"redis_git_dirty={redis_metadata['redis_git_dirty']}\n"
+            f"redis_build_id={redis_metadata['redis_build_id']}\n"
+            f"process_id={redis_metadata['process_id']}\n"
+            f"tcp_port={redis_metadata['tcp_port']}\n"
+            f"duration_seconds={duration}\n"
         )
-        
-        # Write file with metadata and profile content
-        with open(filepath, 'w') as f:
-            f.write(metadata_comment)
-            f.write(profile_content)
-            
+
+        with open(metadata_filepath, "w") as f:
+            f.write(metadata_content)
+
         logging.info(f"Saved profile to: {filepath}")
+        logging.info(f"Saved metadata to: {metadata_filepath}")
         return filepath
-        
+
     except Exception as e:
         logging.error(f"Failed to save profile file: {e}")
         return None
@@ -216,7 +229,14 @@ class RemoteProfiler:
     Remote profiler class to handle threaded profiling execution.
     """
 
-    def __init__(self, host: str, port: int, output_dir: str, username: Optional[str] = None, password: Optional[str] = None):
+    def __init__(
+        self,
+        host: str,
+        port: int,
+        output_dir: str,
+        username: Optional[str] = None,
+        password: Optional[str] = None,
+    ):
         self.host = host
         self.port = port
         self.output_dir = output_dir
@@ -225,21 +245,18 @@ class RemoteProfiler:
         self.profile_thread = None
         self.profile_result = None
         self.profile_error = None
-        
+
     def start_profiling(
-        self,
-        redis_conn,
-        benchmark_name: str,
-        benchmark_duration_seconds: int
+        self, redis_conn, benchmark_name: str, benchmark_duration_seconds: int
     ) -> bool:
         """
         Start profiling in a separate thread.
-        
+
         Args:
             redis_conn: Redis connection object
             benchmark_name: Name of the benchmark
             benchmark_duration_seconds: Expected benchmark duration
-            
+
         Returns:
             True if profiling thread started successfully, False otherwise
         """
@@ -247,44 +264,60 @@ class RemoteProfiler:
             # Extract Redis metadata and PID
             redis_metadata = extract_redis_metadata(redis_conn)
             pid = redis_metadata.get("process_id")
-            
+
             if pid == "unknown" or pid is None:
                 logging.error("Cannot start remote profiling: Redis PID not available")
                 return False
-                
+
             # Calculate profiling duration
             duration = calculate_profile_duration(benchmark_duration_seconds)
-            
+
             # Start profiling thread
             self.profile_thread = threading.Thread(
                 target=self._profile_worker,
                 args=(pid, duration, benchmark_name, redis_metadata),
-                daemon=True
+                daemon=True,
             )
             self.profile_thread.start()
-            
-            logging.info(f"Started remote profiling thread for benchmark: {benchmark_name}")
+
+            logging.info(
+                f"Started remote profiling thread for benchmark: {benchmark_name}"
+            )
             return True
-            
+
         except Exception as e:
             logging.error(f"Failed to start remote profiling: {e}")
             return False
-    
-    def _profile_worker(self, pid: int, duration: int, benchmark_name: str, redis_metadata: Dict[str, Any]):
+
+    def _profile_worker(
+        self,
+        pid: int,
+        duration: int,
+        benchmark_name: str,
+        redis_metadata: Dict[str, Any],
+    ):
         """
         Worker function for profiling thread.
         """
         try:
             # Trigger remote profiling
             profile_content = trigger_remote_profile(
-                self.host, self.port, pid, duration,
-                username=self.username, password=self.password
+                self.host,
+                self.port,
+                pid,
+                duration,
+                username=self.username,
+                password=self.password,
             )
 
-            if profile_content:
+            if profile_content is not None:
                 # Save profile with metadata
                 filepath = save_profile_with_metadata(
-                    profile_content, benchmark_name, self.output_dir, redis_metadata, duration
+                    profile_content,
+                    benchmark_name,
+                    self.output_dir,
+                    redis_metadata,
+                    duration,
                 )
                 self.profile_result = filepath
             else:
@@ -293,38 +326,42 @@ class RemoteProfiler:
         except Exception as e:
             self.profile_error = f"Profile worker error: {e}"
             logging.error(self.profile_error)
-    
+
     def wait_for_completion(self, timeout: int = 60) -> bool:
         """
         Wait for profiling thread to complete.
-        
+
         Args:
             timeout: Maximum time to wait in seconds
-            
+
         Returns:
             True if completed successfully, False if timed out or failed
         """
         if self.profile_thread is None:
             return False
-            
+
         try:
             self.profile_thread.join(timeout=timeout)
-            
+
             if self.profile_thread.is_alive():
-                logging.warning(f"Remote profiling thread did not complete within {timeout}s")
+                logging.warning(
+                    f"Remote profiling thread did not complete within {timeout}s"
+                )
                 return False
-                
+
             if self.profile_error:
                 logging.error(f"Remote profiling failed: {self.profile_error}")
                 return False
-                
+
             if self.profile_result:
-                logging.info(f"Remote profiling completed successfully: {self.profile_result}")
+                logging.info(
+                    f"Remote profiling completed successfully: {self.profile_result}"
+                )
                 return True
             else:
                 logging.warning("Remote profiling completed but no result available")
                 return False
-                
+
         except Exception as e:
             logging.error(f"Error waiting for remote profiling completion: {e}")
             return False