matrice-compute 0.1.19__py3-none-any.whl → 0.1.21__py3-none-any.whl

matrice_compute/resources_tracker.py

@@ -56,7 +56,7 @@ class ResourcesTracker:
             return cpu_utilization, memory_utilization
         return 0, 0
 
-    @log_errors(default_return=(0, 0), raise_exception=False)
+    @log_errors(default_return=(0, 0), raise_exception=False, log_error=False)
     def get_container_cpu_and_memory_with_container_id(self, container_id: str) -> Tuple[float, float]:
         """
         Get CPU and memory usage for a specific container by its ID.
@@ -67,32 +67,46 @@ class ResourcesTracker:
         Returns:
             Tuple[float, float]: CPU utilization percentage and memory usage in MB.
         """
-        stats_result = subprocess.run(
-            [
-                "docker",
-                "stats",
-                "--no-stream",
-                "--format",
-                "{{.ID}}: {{.CPUPerc}} CPU, {{.MemUsage}} RAM",
-                container_id,
-            ],
-            capture_output=True,
-            text=True,
-            check=True,
-        )
-        stats = stats_result.stdout.strip().split(": ")[1].split(", ")
-        cpu_usage = float(stats[0].replace("% CPU", "").strip())
-        memory_usage = stats[1].split(" / ")[0]
-        mem_value, mem_unit = memory_usage[:-3], memory_usage[-3:]
-        if mem_unit == "KiB":
-            memory_usage_mb = float(mem_value) / 1024
-        elif mem_unit == "MiB":
-            memory_usage_mb = float(mem_value)
-        elif mem_unit == "GiB":
-            memory_usage_mb = float(mem_value) * 1024
-        else:
-            memory_usage_mb = float(mem_value)
-        return cpu_usage, memory_usage_mb
+        try:
+            stats_result = subprocess.run(
+                [
+                    "docker",
+                    "stats",
+                    "--no-stream",
+                    "--format",
+                    "{{.ID}}: {{.CPUPerc}} CPU, {{.MemUsage}} RAM",
+                    container_id,
+                ],
+                capture_output=True,
+                text=True,
+                check=False,
+                timeout=10,
+            )
+            if stats_result.returncode != 0:
+                logging.debug("docker stats command failed for container %s", container_id)
+                return 0, 0
+            stats = stats_result.stdout.strip().split(": ")[1].split(", ")
+            cpu_usage = float(stats[0].replace("% CPU", "").strip())
+            memory_usage = stats[1].split(" / ")[0]
+            mem_value, mem_unit = memory_usage[:-3], memory_usage[-3:]
+            if mem_unit == "KiB":
+                memory_usage_mb = float(mem_value) / 1024
+            elif mem_unit == "MiB":
+                memory_usage_mb = float(mem_value)
+            elif mem_unit == "GiB":
+                memory_usage_mb = float(mem_value) * 1024
+            else:
+                memory_usage_mb = float(mem_value)
+            return cpu_usage, memory_usage_mb
+        except subprocess.TimeoutExpired:
+            logging.debug("docker stats command timed out for container %s", container_id)
+            return 0, 0
+        except (ValueError, IndexError) as e:
+            logging.debug("Error parsing docker stats for container %s: %s", container_id, e)
+            return 0, 0
+        except Exception as e:
+            logging.debug("Unexpected error getting container stats for %s: %s", container_id, e)
+            return 0, 0
 
     @log_errors(default_return=(0, 0), raise_exception=False, log_error=False)
     def get_container_gpu_info(self, container_id: str) -> Tuple[float, int]:
@@ -110,7 +124,7 @@ class ResourcesTracker:
         gpu_mem_used = self.get_container_gpu_memory_usage(container_pid)
         return gpu_util, gpu_mem_used
 
-    @log_errors(default_return="", raise_exception=False)
+    @log_errors(default_return="", raise_exception=False, log_error=False)
     def get_pid_id_by_container_id(self, container_id: str) -> str:
         """
         Get PID for a container ID.
@@ -121,20 +135,31 @@ class ResourcesTracker:
         Returns:
             str: PID of the container.
         """
-        pid_result = subprocess.run(
-            [
-                "docker",
-                "inspect",
-                "--format",
-                "{{.State.Pid}}",
-                container_id,
-            ],
-            capture_output=True,
-            text=True,
-            check=True,
-        )
-        container_pid = pid_result.stdout.strip()
-        return container_pid
+        try:
+            pid_result = subprocess.run(
+                [
+                    "docker",
+                    "inspect",
+                    "--format",
+                    "{{.State.Pid}}",
+                    container_id,
+                ],
+                capture_output=True,
+                text=True,
+                check=False,
+                timeout=10,
+            )
+            if pid_result.returncode != 0:
+                logging.debug("docker inspect command failed for container %s", container_id)
+                return ""
+            container_pid = pid_result.stdout.strip()
+            return container_pid
+        except subprocess.TimeoutExpired:
+            logging.debug("docker inspect command timed out for container %s", container_id)
+            return ""
+        except Exception as e:
+            logging.debug("Error getting PID for container %s: %s", container_id, e)
+            return ""
 
     @log_errors(default_return=0, raise_exception=False, log_error=False)
     def get_container_gpu_usage(self, container_pid: str) -> float:
@@ -155,9 +180,12 @@ class ResourcesTracker:
                 ["nvidia-smi", "pmon", "-c", "1"],
                 capture_output=True,
                 text=True,
-                check=True,
+                check=False,
                 timeout=5,
             )
+            if result.returncode != 0:
+                logging.debug("nvidia-smi pmon command failed in get_container_gpu_usage")
+                return 0
             pmon_output = result.stdout.strip().split("\n")
             for line in pmon_output[2:]:
                 parts = line.split()
@@ -167,7 +195,16 @@ class ResourcesTracker:
                     if pid == str(container_pid):
                         gpu_util += float(gpu_usage) if gpu_usage != "-" else 0
         except subprocess.TimeoutExpired:
-            logging.warning("nvidia-smi pmon command timed out after 5 seconds in get_container_gpu_usage")
+            logging.debug("nvidia-smi pmon command timed out after 5 seconds in get_container_gpu_usage")
+            return 0
+        except (ValueError, IndexError) as e:
+            logging.debug("Error parsing GPU usage info: %s", e)
+            return 0
+        except FileNotFoundError:
+            logging.debug("nvidia-smi not found on this system")
+            return 0
+        except Exception as e:
+            logging.debug("Unexpected error in get_container_gpu_usage: %s", e)
             return 0
         return gpu_util
 
@@ -196,9 +233,12 @@ class ResourcesTracker:
                 stdout=subprocess.PIPE,
                 stderr=subprocess.PIPE,
                 text=True,
-                check=True,
+                check=False,
                 timeout=5,
             )
+            if result.returncode != 0:
+                logging.debug("nvidia-smi command failed in get_container_gpu_memory_usage")
+                return 0
             for line in result.stdout.splitlines():
                 parts = line.strip().split(", ")
                 if len(parts) == 2:
@@ -206,7 +246,16 @@ class ResourcesTracker:
                     if process_pid == str(container_pid):
                         total_memory += int(used_memory)
         except subprocess.TimeoutExpired:
-            logging.warning("nvidia-smi command timed out after 5 seconds in get_container_gpu_memory_usage")
+            logging.debug("nvidia-smi command timed out after 5 seconds in get_container_gpu_memory_usage")
+            return 0
+        except (ValueError, IndexError) as e:
+            logging.debug("Error parsing GPU memory usage info: %s", e)
+            return 0
+        except FileNotFoundError:
+            logging.debug("nvidia-smi not found on this system")
+            return 0
+        except Exception as e:
+            logging.debug("Unexpected error in get_container_gpu_memory_usage: %s", e)
             return 0
         return total_memory
 
@@ -238,17 +287,40 @@ class ResourcesTracker:
             return gpu_memory_free, gpu_utilization
 
         try:
-            subprocess.check_output("nvidia-smi", timeout=5)
+            result = subprocess.run(
+                ["nvidia-smi"],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                timeout=5,
+                check=False,
+            )
+            if result.returncode != 0:
+                logging.debug("nvidia-smi command failed in _get_gpu_resources")
+                return 0, 0.0
         except subprocess.TimeoutExpired:
-            logging.warning("nvidia-smi command timed out after 5 seconds in _get_gpu_resources")
+            logging.debug("nvidia-smi command timed out after 5 seconds in _get_gpu_resources")
+            return 0, 0.0
+        except FileNotFoundError:
+            logging.debug("nvidia-smi not found on this system")
+            return 0, 0.0
+        except Exception as e:
+            logging.debug("Error running nvidia-smi in _get_gpu_resources: %s", e)
             return 0, 0.0
         
         info_list = get_gpu_info()
-        for info in info_list:
-            info_split = info.split(", ")
-            gpu_memory_free += int(info_split[5])
-            gpu_utilization += float(info_split[2])
-        gpu_utilization /= len(info_list) if info_list else 1
+        if not info_list:
+            return 0, 0.0
+        
+        try:
+            for info in info_list:
+                info_split = info.split(", ")
+                if len(info_split) >= 6:
+                    gpu_memory_free += int(info_split[5])
+                    gpu_utilization += float(info_split[2])
+            gpu_utilization /= len(info_list) if info_list else 1
+        except (ValueError, IndexError) as e:
+            logging.debug("Error parsing GPU resources: %s", e)
+            return 0, 0.0
 
         return gpu_memory_free, gpu_utilization