matrice-compute 0.1.41__py3-none-any.whl → 0.1.42__py3-none-any.whl

matrice_compute/action_instance.py

@@ -433,10 +433,9 @@ class ActionInstance:
         cmd_parts = [
             f"docker run -d {use_gpu} {use_restart_policy} ",
             network_config,
+            f"--name {self.action_record_id}_{self.action_type} ",
             *[f"-e {key}={shlex.quote(str(value))}" for key, value in env_vars.items()],
             *volumes,
-            # Container configuration and startup commands
-            f"--cidfile ./{self.action_record_id}.cid ",
             f"--shm-size=30G --pull=always {shlex.quote(self.docker_container)}",
             f'/bin/bash -c "cd {docker_workdir} && '
             f"{env_exports} && "
@@ -846,51 +845,6 @@ class ActionInstance:
                 env={**os.environ},
                 start_new_session=True,
             )
-        
-        self.container_id = None
-
-        cid_file_path = f"./{self.action_record_id}.cid"
-        max_retries = 5
-        retry_delay = 1  # seconds
-        for attempt in range(max_retries):
-            try:
-                with open(cid_file_path, "r") as cid_file:
-                    container_id = cid_file.read().strip()
-                    self.container_id = container_id
-                    logging.info(
-                        "Started process for action %s with container ID: %s",
-                        self.action_record_id,
-                        self.container_id,
-                    )
-                    break
-            except FileNotFoundError:
-                logging.warning(
-                    "CID file not found for action %s, attempt %d/%d",
-                    self.action_record_id,
-                    attempt + 1,
-                    max_retries,
-                )
-                time.sleep(retry_delay)
-            except Exception as e:
-                logging.error(
-                    "Error reading CID file for action %s: %s",
-                    self.action_record_id,
-                    str(e),
-                )
-                time.sleep(retry_delay)
-        else:
-            logging.error(
-                "Failed to read CID file for action %s after %d attempts",
-                self.action_record_id,
-                max_retries,
-            )
-            raise Exception("Failed to start process: CID file not found")
-        
-        # report container id to scaling service
-        self.scaling.update_action_container_id(
-            action_record_id=self.action_record_id,
-            container_id=self.container_id,
-        )
     
 
     @log_errors(raise_exception=False)
@@ -1184,9 +1138,8 @@ def database_setup_execute(self: ActionInstance):
     cmd = (
         f"docker run --pull=always --net=host "
         f"-v {dbPath}:{dbPath} "
-        f"--name database_setup_{self.action_record_id} "
+        f"--name {self.action_record_id}_{self.action_type} "
         f"-v /var/run/docker.sock:/var/run/docker.sock "
-        f"--cidfile ./{self.action_record_id}.cid "
         f"-e ACTION_RECORD_ID={self.action_record_id} "
         f"-e MATRICE_ACCESS_KEY_ID={self.matrice_access_key_id} "
         f"-e MATRICE_SECRET_ACCESS_KEY={self.matrice_secret_access_key} "
@@ -1228,10 +1181,8 @@ def facial_recognition_setup_execute(self: ActionInstance):
     # Facial recognition worker container with --net=host (Port: 8081)
     worker_cmd = (
         f"docker run -d --pull=always --net=host "
-        f"--name worker "
-         f"--cidfile ./{self.action_record_id}.cid "
+        f"--name {self.action_record_id}_{self.action_type} "
         f"-v matrice_myvol:/matrice_data "
-        f"--cidfile ./{self.action_record_id}.cid "
         f'-e ENV="{os.environ.get("ENV", "prod")}" '
         f'-e MATRICE_SECRET_ACCESS_KEY="{self.matrice_secret_access_key}" '
         f'-e MATRICE_ACCESS_KEY_ID="{self.matrice_access_key_id}" '
@@ -1271,8 +1222,7 @@ def lpr_setup_execute(self: ActionInstance):
     # LPR worker container with --net=host (Port: 8082)
     worker_cmd = (
         f"docker run -d --net=host --pull=always "
-        f"--name lpr-worker "
-        f"--cidfile ./{self.action_record_id}.cid "
+        f"--name {self.action_record_id}_{self.action_type} "
         f"-v matrice_myvol:/matrice_data "
         f'-e ENV="{os.environ.get("ENV", "prod")}" '
         f'-e MATRICE_SECRET_ACCESS_KEY="{self.matrice_secret_access_key}" '
@@ -1323,8 +1273,7 @@ def inference_ws_server_execute(self: ActionInstance):
     # Inference WebSocket server with --net=host (Port: 8102)
     worker_cmd = (
         f"docker run -d --pull=always --net=host "
-        f"--name inference "
-        f"--cidfile ./{self.action_record_id}.cid "
+        f"--name {self.action_record_id}_{self.action_type} "
         f'-e ENV="{os.environ.get("ENV", "prod")}" '
         f'-e MATRICE_SECRET_ACCESS_KEY="{self.matrice_secret_access_key}" '
         f'-e MATRICE_ACCESS_KEY_ID="{self.matrice_access_key_id}" '
@@ -1372,8 +1321,7 @@ def fe_fs_streaming_execute(self: ActionInstance):
     # Frontend streaming with --net=host (Port: 3000)
     worker_cmd = (
         f"docker run -d --pull=always --net=host "
-        f"--name fe_streaming "
-        f"--cidfile ./{self.action_record_id}.cid "
+        f"--name {self.action_record_id}_{self.action_type} "
         f"-v matrice_myvol:/matrice_data "
         f'-e ENV="{os.environ.get("ENV", "prod")}" '
         f'-e MATRICE_SECRET_ACCESS_KEY="{self.matrice_secret_access_key}" '
@@ -1418,8 +1366,7 @@ def fe_analytics_service_execute(self: ActionInstance):
     # Frontend analytics service with --net=host (Port: 3001)
     worker_cmd = (
         f"docker run -d --pull=always --net=host "
-        f"--name fe-analytics "
-        f"--cidfile ./{self.action_record_id}.cid "
+        f"--name {self.action_record_id}_{self.action_type} "
         f'-e NEXT_PUBLIC_DEPLOYMENT_ENV="{os.environ.get("ENV", "prod")}" '
         f'-e MATRICE_SECRET_ACCESS_KEY="{self.matrice_secret_access_key}" '
         f'-e MATRICE_ACCESS_KEY_ID="{self.matrice_access_key_id}" '
@@ -1520,7 +1467,7 @@ def redis_setup_execute(self: ActionInstance):
         self.start(cmd, "redis_setup")
 
         # Redis container restart
-        redis_restart_cmd = "docker restart redis_container"
+        redis_restart_cmd = f"docker restart {self.action_record_id}_{self.action_type}_redis_container"
         self.start(redis_restart_cmd, "redis")
 
         return
@@ -1528,7 +1475,7 @@ def redis_setup_execute(self: ActionInstance):
     # Redis container with --net=host (Port: 6379)
     redis_cmd = (
         f"docker run -d --net=host "
-        f"--name redis_container "
+        f"--name {self.action_record_id}_{self.action_type}_redis_container "
         f"--restart unless-stopped "
         f"{redis_image} "
         f"redis-server --bind 0.0.0.0 --appendonly yes --requirepass {redis_password}"
@@ -1556,7 +1503,6 @@ def redis_setup_execute(self: ActionInstance):
     # bg-redis management container with --net=host (Port: 8082)
     cmd = (
         f"docker run --net=host "
-         f"--cidfile ./{self.action_record_id}.cid "
         f"-e REDIS_URL={shlex.quote(env_vars['REDIS_URL'])} "
         f"-e REDIS_PASSWORD={shlex.quote(env_vars['REDIS_PASSWORD'])} "
         f"-e MATRICE_ACCESS_KEY_ID={shlex.quote(self.matrice_access_key_id)} "
@@ -1921,8 +1867,7 @@ def inference_tracker_setup_execute(self: ActionInstance):
     # This is the existing Docker run command
     worker_cmd = (
         f"docker run -d --pull=always --net=host "
-         f"--cidfile ./{self.action_record_id}.cid "
-        f"--name inference-tracker-worker "
+         f"--name {self.action_record_id}_{self.action_type} "
         f"-v matrice_myvol:/matrice_data "
         f'-e ENV="{os.environ.get("ENV", "prod")}" '
         f'-e MATRICE_SECRET_ACCESS_KEY="{self.matrice_secret_access_key}" '
@@ -1963,8 +1908,7 @@ def video_storage_setup_execute(self: ActionInstance):
     # This is the existing Docker run command
     worker_cmd = (
         f"docker run -d --pull=always --net=host "
-         f"--cidfile ./{self.action_record_id}.cid "
-        f"--name media_server "
+        f"--name {self.action_record_id}_{self.action_type} "
         f"-v matrice_myvol:/matrice_data "
         f'-e ENV="{os.environ.get("ENV", "prod")}" '
         f'-e MATRICE_SECRET_ACCESS_KEY="{self.matrice_secret_access_key}" '

matrice_compute/instance_manager.py

@@ -18,6 +18,7 @@ from matrice_compute.resources_tracker import (
     MachineResourcesTracker,
     ActionsResourcesTracker,
     KafkaResourceMonitor,
+    ContainerResourceMonitor,
 )
 from matrice_compute.scaling import Scaling
 from matrice_compute.shutdown_manager import ShutdownManager
@@ -111,6 +112,19 @@ class InstanceManager:
             logging.warning("Failed to initialize Kafka resource monitor: %s", e)
             self.kafka_resource_monitor = None
 
+        # Initialize Container resource monitor using the same internal Kafka as scaling
+        try:
+            kafka_bootstrap = self.scaling.get_kafka_bootstrap_servers()
+            self.container_resource_monitor = ContainerResourceMonitor(
+                instance_id=os.environ.get("INSTANCE_ID"),
+                kafka_bootstrap=kafka_bootstrap,
+                interval_seconds=30
+            )
+            logging.info("InstanceManager initialized with Container resource monitor using internal Kafka: %s", kafka_bootstrap)
+        except (ValueError, Exception) as e:
+            logging.warning("Failed to initialize Container resource monitor: %s", e)
+            self.container_resource_monitor = None
+
         # Initialize Compute Operations Handler for event-driven operations
         # Uses EventListener from matrice_common for simplified Kafka consumption
         try:
@@ -436,6 +450,14 @@ class InstanceManager:
             except Exception as exc:
                 logging.error("Failed to start Kafka resource monitor: %s", str(exc))
 
+        # Start Container resource monitor in background thread
+        if self.container_resource_monitor:
+            try:
+                self.container_resource_monitor.start()
+                logging.info("Started Container resource monitor")
+            except Exception as exc:
+                logging.error("Failed to start Container resource monitor: %s", str(exc))
+
         # Start Compute Operations Handler in background thread
         if self.compute_operations_handler:
             try:
@@ -468,3 +490,40 @@ class InstanceManager:
             instance_manager_thread,
             actions_manager_thread,
         )
+
+    def stop(self):
+        """Stop all background threads and cleanup resources."""
+        logging.info("Stopping InstanceManager...")
+        
+        # Stop Container resource monitor
+        if hasattr(self, 'container_resource_monitor') and self.container_resource_monitor:
+            try:
+                self.container_resource_monitor.stop()
+                logging.info("Stopped Container resource monitor")
+            except Exception as exc:
+                logging.error("Failed to stop Container resource monitor: %s", str(exc))
+        
+        # Stop Kafka resource monitor
+        if hasattr(self, 'kafka_resource_monitor') and self.kafka_resource_monitor:
+            try:
+                self.kafka_resource_monitor.stop()
+                logging.info("Stopped Kafka resource monitor")
+            except Exception as exc:
+                logging.error("Failed to stop Kafka resource monitor: %s", str(exc))
+        
+        # Stop compute operations handler
+        if hasattr(self, 'compute_operations_handler') and self.compute_operations_handler:
+            try:
+                self.compute_operations_handler.stop()
+                logging.info("Stopped Compute Operations Handler")
+            except Exception as exc:
+                logging.error("Failed to stop Compute Operations Handler: %s", str(exc))
+        
+        # Stop container status monitor
+        try:
+            self.stop_container_status_monitor()
+            logging.info("Stopped Container Status Monitor")
+        except Exception as exc:
+            logging.error("Failed to stop Container Status Monitor: %s", str(exc))
+        
+        logging.info("InstanceManager stopped")

matrice_compute/resources_tracker.py

@@ -1240,6 +1240,239 @@ class MachineResourcesTracker:
             )
 
 
+class ContainerResourceMonitor:
+    """
+    Monitors individual container resource utilization and publishes to Kafka.
+    This thread runs independently and reports CPU, memory, and GPU usage for all running containers.
+    """
+
+    def __init__(
+        self,
+        instance_id: Optional[str] = None,
+        kafka_bootstrap: Optional[str] = None,
+        interval_seconds: int = 30,
+    ):
+        """
+        Initialize ContainerResourceMonitor.
+
+        Args:
+            instance_id: Instance identifier for Kafka topic. Defaults to INSTANCE_ID env var.
+            kafka_bootstrap: Kafka bootstrap servers. Required - should be obtained from Scaling.get_kafka_bootstrap_servers().
+            interval_seconds: Interval between container checks in seconds. Defaults to 30.
+        """
+        self.instance_id = instance_id or os.getenv("INSTANCE_ID")
+        if not self.instance_id:
+            raise ValueError("instance_id must be provided or INSTANCE_ID env var must be set")
+
+        if not kafka_bootstrap:
+            raise ValueError("kafka_bootstrap must be provided - use Scaling.get_kafka_bootstrap_servers() to get internal Kafka config")
+
+        self.kafka_bootstrap = kafka_bootstrap
+        self.interval_seconds = interval_seconds
+        self.topic_name = "instance_resource_utilisation"
+
+        self._stop_event = threading.Event()
+        self._monitor_thread: Optional[threading.Thread] = None
+        self._producer = None
+        self._is_running = False
+        self._docker_client = None
+        self._resources_tracker = ResourcesTracker()
+
+    def _get_all_running_containers(self) -> List[docker.models.containers.Container]:
+        """
+        Get all running Docker containers.
+
+        Returns:
+            List[docker.models.containers.Container]: List of running containers
+        """
+        try:
+            if not self._docker_client:
+                self._docker_client = docker.from_env()
+            
+            containers = self._docker_client.containers.list(filters={"status": "running"})
+            return containers
+        except Exception as e:
+            logging.debug("Error getting running containers: %s", e)
+            return []
+
+    def _collect_container_resources(self, container: docker.models.containers.Container) -> Dict:
+        """
+        Collect resource usage for a single container.
+
+        Args:
+            container: Docker container instance
+
+        Returns:
+            Dict: Container resource data
+        """
+        try:
+            container_id = container.id
+            container_name = container.name
+            
+            # Get CPU and memory usage
+            cpu_util, memory_mb = self._resources_tracker.get_container_cpu_and_memory(container)
+            
+            # Get GPU usage (utilization and memory)
+            gpu_util, gpu_memory_mb = self._resources_tracker.get_container_gpu_info(container_id)
+            
+            return {
+                "container_id": container_id,
+                "container_name": container_name,
+                "cpu_utilization_percent": round(cpu_util, 2),
+                "memory_usage_mb": round(memory_mb, 2),
+                "gpu_utilization_percent": round(gpu_util, 2),
+                "gpu_memory_usage_mb": gpu_memory_mb,
+                "timestamp": datetime.now(timezone.utc).isoformat()
+            }
+        except Exception as e:
+            logging.debug("Error collecting resources for container %s: %s", container.name, e)
+            return None
+
+    def _monitor_worker(self):
+        """
+        Worker function that runs in a separate thread to monitor containers and publish to Kafka.
+        """
+        try:
+            from kafka import KafkaProducer
+
+            self._producer = KafkaProducer(
+                bootstrap_servers=self.kafka_bootstrap,
+                value_serializer=lambda v: json.dumps(v).encode("utf-8"),
+                retries=5,
+            )
+            logging.info("Container resource monitor started. Publishing to topic: %s", self.topic_name)
+
+        except ImportError:
+            logging.error("kafka-python not installed. Install with: pip install kafka-python")
+            return
+        except Exception as e:
+            logging.error("Failed to initialize Kafka producer for container monitor: %s", e)
+            return
+
+        while not self._stop_event.is_set():
+            try:
+                # Get all running containers
+                containers = self._get_all_running_containers()
+                
+                if not containers:
+                    logging.debug("No running containers found")
+                else:
+                    container_data = []
+                    
+                    # Collect resources for each container
+                    for container in containers:
+                        resource_data = self._collect_container_resources(container)
+                        if resource_data:
+                            container_data.append(resource_data)
+                    
+                    # Create the payload with instance information and all container data
+                    payload = {
+                        "instance_id": self.instance_id,
+                        "container_count": len(container_data),
+                        "containers": container_data,
+                        "timestamp": datetime.now(timezone.utc).isoformat()
+                    }
+
+                    # Send to Kafka topic
+                    self._producer.send(self.topic_name, payload)
+                    self._producer.flush()
+
+                    logging.debug("Published container resource stats for %d containers", len(container_data))
+
+            except Exception as e:
+                logging.error("Error in container resource monitor loop: %s", e)
+
+            # Wait for interval or until stop event is set
+            if self._stop_event.wait(self.interval_seconds):
+                break
+
+        # Cleanup
+        if self._producer:
+            try:
+                self._producer.close()
+            except Exception as e:
+                logging.debug("Error closing Kafka producer: %s", e)
+
+        if self._docker_client:
+            try:
+                self._docker_client.close()
+            except Exception as e:
+                logging.debug("Error closing Docker client: %s", e)
+
+        logging.info("Container resource monitor stopped.")
+
+    @log_errors(raise_exception=False, log_error=True)
+    def start(self):
+        """
+        Start the container resource monitoring thread.
+
+        Returns:
+            bool: True if started successfully, False otherwise.
+        """
+        if self._is_running:
+            logging.warning("Container resource monitor is already running.")
+            return False
+
+        self._stop_event.clear()
+        self._monitor_thread = threading.Thread(
+            target=self._monitor_worker,
+            daemon=True,
+            name="ContainerResourceMonitor"
+        )
+        self._monitor_thread.start()
+        self._is_running = True
+
+        logging.info("Started container resource monitor thread.")
+        return True
+
+    @log_errors(raise_exception=False, log_error=True)
+    def stop(self, timeout: int = 10):
+        """
+        Stop the container resource monitoring thread gracefully.
+
+        Args:
+            timeout: Maximum time to wait for thread to stop in seconds.
+
+        Returns:
+            bool: True if stopped successfully, False otherwise.
+        """
+        if not self._is_running:
+            logging.warning("Container resource monitor is not running.")
+            return False
+
+        logging.info("Stopping container resource monitor...")
+        self._stop_event.set()
+
+        if self._monitor_thread and self._monitor_thread.is_alive():
+            self._monitor_thread.join(timeout=timeout)
+
+            if self._monitor_thread.is_alive():
+                logging.error("Container resource monitor thread did not stop within timeout.")
+                return False
+
+        self._is_running = False
+        logging.info("Container resource monitor stopped successfully.")
+        return True
+
+    def is_running(self) -> bool:
+        """
+        Check if the container resource monitor is currently running.
+
+        Returns:
+            bool: True if running, False otherwise.
+        """
+        return self._is_running
+
+    def __enter__(self):
+        """Context manager entry."""
+        self.start()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """Context manager exit."""
+        self.stop()
+
+
 class KafkaResourceMonitor:
     """
     Monitors system resources and publishes them to Kafka in a separate thread.

{matrice_compute-0.1.41.dist-info → matrice_compute-0.1.42.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: matrice_compute
-Version: 0.1.41
+Version: 0.1.42
 Summary: Common server utilities for Matrice.ai services
 Author-email: "Matrice.ai" <dipendra@matrice.ai>
 License-Expression: MIT

{matrice_compute-0.1.41.dist-info → matrice_compute-0.1.42.dist-info}/RECORD

@@ -1,18 +1,18 @@
 matrice_compute/__init__.py,sha256=YZhx7rQlD1TAlhBMbsU3_Xp-tpLyTAxWZDcQvqmwR2g,723
-matrice_compute/action_instance.py,sha256=7T25pP05WkHHcfGM1TpT_lpLhukVJR68Dsoa9Seffjc,75946
+matrice_compute/action_instance.py,sha256=N74jgcNm_l6zk9w8shPlgR9EWFjICXWkfOP4MKwig2o,73979
 matrice_compute/actions_manager.py,sha256=a_TulMnu462xc0t_A-Mpug5zhQTmtpjiv7mhiC_IAVw,18280
 matrice_compute/actions_scaledown_manager.py,sha256=pJ0nduNwHWZ10GnqJNx0Ok7cVWabQ_M8E2Vb9pH3A_k,2002
 matrice_compute/compute_operations_handler.py,sha256=amcMhmXtv2irE6qK8Vbgec_8uFqjWmVVp0VWq-73_MU,17781
-matrice_compute/instance_manager.py,sha256=9u3QRTP-MkAWmrSQMMbCKc0TfK584teAg1wWIaqMZdE,19291
+matrice_compute/instance_manager.py,sha256=W0BN1mkfcqCP1jxb6JjhNPUHM-iTmrDu7WoyfTTKGdY,22098
 matrice_compute/instance_utils.py,sha256=N4yPDvNukFEEBngR0lEt4x_XT5hur1q0P-spM2xQIlU,42025
 matrice_compute/prechecks.py,sha256=W9YmNF3RcLhOf4U8WBlExvFqDw1aGWSNTlJtA73lbDQ,17196
 matrice_compute/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-matrice_compute/resources_tracker.py,sha256=AG_lnxoSi1TIDD0atBybntGyvyenwmP7sGCf4shBL4c,59276
+matrice_compute/resources_tracker.py,sha256=1HDCRZLDbYAHGhJojM_Hee8vSpVGbFB96WUUc_rqudY,67870
 matrice_compute/scaling.py,sha256=UQDI8wN9JEKafvUVPF0Pk9XmhKlbMkeu16AZyyOuSE8,55147
 matrice_compute/shutdown_manager.py,sha256=rnP9Qes6JJKDnebmBC9rqkH__X9a8TMjhWQPWoOQKFs,13232
 matrice_compute/task_utils.py,sha256=3qIutiQdYPyGRxH9ZwLbqdg8sZcnp6jp08pszWCRFl0,2820
-matrice_compute-0.1.41.dist-info/licenses/LICENSE.txt,sha256=_uQUZpgO0mRYL5-fPoEvLSbNnLPv6OmbeEDCHXhK6Qc,1066
-matrice_compute-0.1.41.dist-info/METADATA,sha256=SDamr59U-ZhWg9gfBCwjWfQNMKEnJKFARqH_5wzriDE,1038
-matrice_compute-0.1.41.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-matrice_compute-0.1.41.dist-info/top_level.txt,sha256=63Plr3L1GzBUWZO5JZaFkiv8IcB10xUPU-9w3i6ptvE,16
-matrice_compute-0.1.41.dist-info/RECORD,,
+matrice_compute-0.1.42.dist-info/licenses/LICENSE.txt,sha256=_uQUZpgO0mRYL5-fPoEvLSbNnLPv6OmbeEDCHXhK6Qc,1066
+matrice_compute-0.1.42.dist-info/METADATA,sha256=jSxrKx79IwTZZzRkTuqcHaNwixeEpxT6zG4W2F52UJI,1038
+matrice_compute-0.1.42.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+matrice_compute-0.1.42.dist-info/top_level.txt,sha256=63Plr3L1GzBUWZO5JZaFkiv8IcB10xUPU-9w3i6ptvE,16
+matrice_compute-0.1.42.dist-info/RECORD,,