matrice-compute 0.1.30__py3-none-any.whl → 0.1.32__py3-none-any.whl

matrice_compute/__init__.py

@@ -1,6 +1,7 @@
 """Module providing __init__ functionality."""
 
 import subprocess
+import logging
 
 from matrice_common.utils import dependencies_check
 
@@ -17,4 +18,7 @@ subprocess.run( # Re-upgrade docker to avoid missing DOCKER_HOST connection erro
 
 from matrice_compute.instance_manager import InstanceManager  # noqa: E402
 
+logging.getLogger("kafka").setLevel(logging.INFO)
+logging.getLogger("confluent_kafka").setLevel(logging.INFO)
+
 __all__ = ["InstanceManager"]

matrice_compute/action_instance.py

@@ -296,7 +296,7 @@ class ActionInstance:
                         getattr(self, "action_record_id", "unknown"),
                     )
                 else:
-                    logging.debug(
+                    logging.info(
                         "No additional logs to send for action %s",
                         getattr(self, "action_record_id", "unknown"),
                     )
@@ -411,6 +411,7 @@ class ActionInstance:
         destination_workspace_path: str = "/usr/src/workspace",
         docker_workdir: str = "",
         extra_pkgs: list = [],
+        container_name: str = "",
     ):
         """Build base Docker command with common options.
 
@@ -425,6 +426,7 @@ class ActionInstance:
             destination_workspace_path (str): Container workspace path
             docker_workdir (str): Docker working directory
             extra_pkgs (list): List of extra packages to install
+            container_name (str): Docker container name (format: {action_type}_{action_id})
         Returns:
             str: Base Docker command
         """
@@ -489,13 +491,16 @@ class ActionInstance:
             ]
         )
 
+        # Build container name option if provided
+        name_option = f"--name {container_name}" if container_name else ""
+
         cmd_parts = [
-            f"docker run {use_gpu} ",
+            f"docker run -d {use_gpu} ",
+            name_option,
             network_config,
             *[f"-e {key}={shlex.quote(str(value))}" for key, value in env_vars.items()],
             *volumes,
             # Container configuration and startup commands
-            f"--cidfile ./{self.action_record_id}.cid ",
             f"--shm-size=30G --pull=always {shlex.quote(self.docker_container)}",
             f'/bin/bash -c "cd {docker_workdir} && '
             f"{env_exports} && "
@@ -883,6 +888,34 @@ class ActionInstance:
                 job_params=action_details["jobParams"],
             )
 
+    @staticmethod
+    def container_exists(container_id: str) -> bool:
+        """Check if a Docker container exists.
+
+        Args:
+            container_id (str): Container ID or name to check
+
+        Returns:
+            bool: True if container exists, False otherwise
+        """
+        if not container_id:
+            return False
+        try:
+            result = subprocess.run(
+                ["docker", "inspect", container_id],
+                capture_output=True,
+                text=True,
+                timeout=10
+            )
+            return result.returncode == 0
+        except Exception as e:
+            logging.warning(
+                "Error checking if container %s exists: %s",
+                container_id,
+                str(e)
+            )
+            return False
+
     @log_errors(raise_exception=True)
     def start_process(self, cmd, log_name):
         """Start the process and initialize logging.
@@ -897,60 +930,45 @@ class ActionInstance:
         self.cmd = cmd
         self.log_path = f"{self.get_log_path()}/{log_name}_{self.action_record_id}.txt"
 
-        with open(self.log_path, "wb") as out:
-            self.process = subprocess.Popen(
-                shlex.split(self.cmd),
-                stdout=out,
-                stderr=out,
-                env={**os.environ},
-                start_new_session=True,
-            )
+        # Run docker with -d flag to get container ID from stdout
+        process = subprocess.Popen(
+            shlex.split(self.cmd),
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+            env={**os.environ},
+        )
         
-        self.container_id = None
-
-        cid_file_path = f"./{self.action_record_id}.cid"
-        max_retries = 5
-        retry_delay = 1  # seconds
-        for attempt in range(max_retries):
-            try:
-                with open(cid_file_path, "r") as cid_file:
-                    container_id = cid_file.read().strip()
-                    self.container_id = container_id
-                    logging.info(
-                        "Started process for action %s with container ID: %s",
-                        self.action_record_id,
-                        self.container_id,
-                    )
-                    break
-            except FileNotFoundError:
-                logging.warning(
-                    "CID file not found for action %s, attempt %d/%d",
-                    self.action_record_id,
-                    attempt + 1,
-                    max_retries,
-                )
-                time.sleep(retry_delay)
-            except Exception as e:
-                logging.error(
-                    "Error reading CID file for action %s: %s",
-                    self.action_record_id,
-                    str(e),
-                )
-                time.sleep(retry_delay)
-        else:
+        stdout, stderr = process.communicate(timeout=120)
+        
+        if process.returncode != 0:
             logging.error(
-                "Failed to read CID file for action %s after %d attempts",
+                "Docker run failed for action %s: %s",
                 self.action_record_id,
-                max_retries,
+                stderr,
             )
-            raise Exception("Failed to start process: CID file not found")
+            raise RuntimeError(f"Docker run failed: {stderr}")
         
-        # report container id to scaling service
+        self.container_id = stdout.strip()
+        logging.info(
+            "Started container for action %s with ID: %s",
+            self.action_record_id,
+            self.container_id,
+        )
+        
+        # Start following container logs in background
+        self.process = subprocess.Popen(
+            ["docker", "logs", "-f", self.container_id],
+            stdout=open(self.log_path, "wb"),
+            stderr=subprocess.STDOUT,
+            start_new_session=True,
+        )
+        
+        # Report container id to scaling service
         self.scaling.update_action_container_id(
             action_record_id=self.action_record_id,
             container_id=self.container_id,
         )
-    
 
     @log_errors(raise_exception=False)
     def start_logger(self):
@@ -1111,7 +1129,8 @@ def data_preparation_execute(
             "Started pulling Docker image with PID: %s",
             process.pid,
         )
-    cmd = f'{self.get_base_docker_cmd(work_fs, destination_workspace_path="/usr/src/app/workspace", docker_workdir="/usr/src/app/workspace", extra_pkgs=["matrice_dataset"])} python3 /usr/src/app/data_preparation.py {self.action_record_id} "'
+    container_name = f"data_prep_{self.action_record_id}"
+    cmd = f'{self.get_base_docker_cmd(work_fs, destination_workspace_path="/usr/src/app/workspace", docker_workdir="/usr/src/app/workspace", extra_pkgs=["matrice_dataset"], container_name=container_name)} python3 /usr/src/app/data_preparation.py {self.action_record_id} "'
     logging.info("cmd is: %s", cmd)
     self.start(cmd, "data_preparation_log")
 
@@ -1140,7 +1159,8 @@ def data_processing_execute(self: ActionInstance):
         service="bg-job-scheduler",
         job_params=action["jobParams"],
     )
-    cmd = f'{self.get_base_docker_cmd(work_fs, extra_pkgs=["matrice_dataset"])} python3 /usr/src/app/main.py {self.action_record_id} "'
+    container_name = f"data_processing_{self.action_record_id}"
+    cmd = f'{self.get_base_docker_cmd(work_fs, extra_pkgs=["matrice_dataset"], container_name=container_name)} python3 /usr/src/app/main.py {self.action_record_id} "'
     logging.info("cmd: %s", cmd)
     self.start(cmd, "data_processing_log")
 
@@ -1153,7 +1173,8 @@ def data_split_execute(self: ActionInstance):
     if not action_details:
         return
     self.setup_action_requirements(action_details, work_fs, model_family="")
-    cmd = f'{self.get_base_docker_cmd(work_fs, extra_pkgs=["matrice_dataset"])} python3 /usr/src/app/data_split.py {self.action_record_id} "'
+    container_name = f"data_split_{self.action_record_id}"
+    cmd = f'{self.get_base_docker_cmd(work_fs, extra_pkgs=["matrice_dataset"], container_name=container_name)} python3 /usr/src/app/data_split.py {self.action_record_id} "'
     logging.info("cmd: %s", cmd)
     self.start(cmd, "data_split")
 
@@ -1168,7 +1189,8 @@ def dataset_annotation_execute(
     if not action_details:
         return
     self.setup_action_requirements(action_details, work_fs)
-    cmd = f'{self.get_base_docker_cmd(work_fs, extra_pkgs=["matrice_dataset"])} python3 /usr/src/app/dataset_annotation.py {self.action_record_id} "'
+    container_name = f"dataset_annotation_{self.action_record_id}"
+    cmd = f'{self.get_base_docker_cmd(work_fs, extra_pkgs=["matrice_dataset"], container_name=container_name)} python3 /usr/src/app/dataset_annotation.py {self.action_record_id} "'
     logging.info("cmd: %s", cmd)
     self.start(cmd, "dataset_annotation")
 
@@ -1183,7 +1205,8 @@ def dataset_augmentation_execute(
     if not action_details:
         return
     self.setup_action_requirements(action_details, work_fs)
-    cmd = f'{self.get_base_docker_cmd(work_fs, extra_pkgs=["matrice_dataset"])} python3 /usr/src/app/data_augmentation.py {self.action_record_id} "'
+    container_name = f"dataset_augmentation_{self.action_record_id}"
+    cmd = f'{self.get_base_docker_cmd(work_fs, extra_pkgs=["matrice_dataset"], container_name=container_name)} python3 /usr/src/app/data_augmentation.py {self.action_record_id} "'
     logging.info("cmd: %s", cmd)
     self.start(cmd, "dataset_augmentation")
 
@@ -1199,7 +1222,8 @@ def augmentation_server_creation_execute(
     if not action_details:
         return
     self.setup_action_requirements(action_details, work_fs)
-    cmd = f'{self.get_base_docker_cmd(work_fs, extra_pkgs=["matrice_dataset"])} python3 /usr/src/app/aug_server.py {self.action_record_id} {external_port} "'
+    container_name = f"augmentation_setup_{self.action_record_id}"
+    cmd = f'{self.get_base_docker_cmd(work_fs, extra_pkgs=["matrice_dataset"], container_name=container_name)} python3 /usr/src/app/aug_server.py {self.action_record_id} {external_port} "'
     logging.info("cmd: %s", cmd)
     self.start(cmd, "augmentation_setup")
 
@@ -1220,25 +1244,41 @@ def database_setup_execute(self: ActionInstance):
 
     project_id = action_details["_idProject"]
 
-    if action_details["actionDetails"].get("containerId"):
-        logging.info(
-            "Using existing container ID for inference tracker: %s",
-            action_details["actionDetails"]["containerId"],
-        )
-        self.docker_container = action_details["actionDetails"]["containerId"]
-        cmd = "docker restart " + self.docker_container 
-        self.start(cmd, "qdrant_setup")
+    # Define container names with action_record_id for uniqueness
+    mongodb_container_name = f"database_setup_{self.action_record_id}"
+    qdrant_container_name = f"qdrant_{self.action_record_id}"
 
-        #qdrant restart
-        qdrant_cmd = "docker restart qdrant"
-        self.start(qdrant_cmd, 'qdrant_setup')
+    existing_container_id = action_details["actionDetails"].get("containerId")
+    if existing_container_id:
+        # Check if both containers actually exist before trying to restart
+        mongodb_container_exists = ActionInstance.container_exists(existing_container_id)
+        qdrant_container_exists = ActionInstance.container_exists(qdrant_container_name)
 
-        return
+        if mongodb_container_exists and qdrant_container_exists:
+            logging.info(
+                "Using existing container ID for database setup: %s",
+                existing_container_id,
+            )
+            self.docker_container = existing_container_id
+            cmd = "docker restart " + self.docker_container
+            self.start(cmd, "qdrant_setup")
+
+            # qdrant restart
+            qdrant_cmd = f"docker restart {qdrant_container_name}"
+            self.start(qdrant_cmd, "qdrant_setup")
+            return
+        else:
+            logging.warning(
+                "Container(s) not found (mongodb=%s, qdrant=%s). Creating new containers.",
+                mongodb_container_exists,
+                qdrant_container_exists
+            )
+            # Fall through to create new containers
 
     # MongoDB container with --net=host (Port: 27020:27017)
     cmd = (
         f"docker run --pull=always --net=host "
-        f"--name mongodbdatabase "
+        f"--name {mongodb_container_name} "
         f"-v matrice_myvol:/matrice_data "
         f"--cidfile ./{self.action_record_id}.cid "
         f"-e ACTION_RECORD_ID={self.action_record_id} "
@@ -1253,7 +1293,7 @@ def database_setup_execute(self: ActionInstance):
     # Qdrant container with --net=host (Port: 6334)
     qdrant_cmd = (
         f"docker run --pull=always --net=host "
-        f"--name qdrant "
+        f"--name {qdrant_container_name} "
         f"-v matrice_myvol:/matrice_data "
         f"{'qdrant/qdrant:latest'} "
     )
@@ -1279,23 +1319,32 @@ def facial_recognition_setup_execute(self: ActionInstance):
 
     self.setup_action_requirements(action_details)
 
-    if action_details["actionDetails"].get("containerId"):
-        logging.info(
-            "Using existing container ID for facial recognition worker: %s",
-            action_details["actionDetails"]["containerId"],
-        )
-        self.docker_container = action_details["actionDetails"]["containerId"]
-        cmd = "docker restart " + self.docker_container 
-        self.start(cmd, "facial_recognition_setup")
-        return
+    existing_container_id = action_details["actionDetails"].get("containerId")
+    if existing_container_id:
+        # Check if container actually exists before trying to restart
+        if ActionInstance.container_exists(existing_container_id):
+            logging.info(
+                "Using existing container ID for facial recognition worker: %s",
+                existing_container_id,
+            )
+            self.docker_container = existing_container_id
+            cmd = "docker restart " + self.docker_container
+            self.start(cmd, "facial_recognition_setup")
+            return
+        else:
+            logging.warning(
+                "Container %s not found. Creating new container.",
+                existing_container_id
+            )
+            # Fall through to create new container
 
     # Facial recognition worker container with --net=host (Port: 8081)
+    container_name = f"facial_recognition_{self.action_record_id}"
     worker_cmd = (
         f"docker run -d --pull=always --net=host "
-        f"--name worker "
-         f"--cidfile ./{self.action_record_id}.cid "
-        f"-v matrice_myvol:/matrice_data "
+        f"--name {container_name} "
         f"--cidfile ./{self.action_record_id}.cid "
+        f"-v matrice_myvol:/matrice_data "
         f'-e ENV="{os.environ.get("ENV", "prod")}" '
         f'-e MATRICE_SECRET_ACCESS_KEY="{self.matrice_secret_access_key}" '
         f'-e MATRICE_ACCESS_KEY_ID="{self.matrice_access_key_id}" '
@@ -1321,20 +1370,30 @@ def lpr_setup_execute(self: ActionInstance):
 
     self.setup_action_requirements(action_details)
 
-    if action_details["actionDetails"].get("containerId"):
-        logging.info(
-            "Using existing container ID for LPR worker: %s",
-            action_details["actionDetails"]["containerId"],
-        )
-        self.docker_container = action_details["actionDetails"]["containerId"]
-        cmd = "docker restart " + self.docker_container 
-        self.start(cmd, "lpr_setup")
-        return
+    existing_container_id = action_details["actionDetails"].get("containerId")
+    if existing_container_id:
+        # Check if container actually exists before trying to restart
+        if ActionInstance.container_exists(existing_container_id):
+            logging.info(
+                "Using existing container ID for LPR worker: %s",
+                existing_container_id,
+            )
+            self.docker_container = existing_container_id
+            cmd = "docker restart " + self.docker_container
+            self.start(cmd, "lpr_setup")
+            return
+        else:
+            logging.warning(
+                "Container %s not found. Creating new container.",
+                existing_container_id
+            )
+            # Fall through to create new container
 
     # LPR worker container with --net=host (Port: 8082)
+    container_name = f"lpr_{self.action_record_id}"
     worker_cmd = (
         f"docker run -d --net=host --pull=always "
-        f"--name lpr-worker "
+        f"--name {container_name} "
         f"--cidfile ./{self.action_record_id}.cid "
         f"-v matrice_myvol:/matrice_data "
         f'-e ENV="{os.environ.get("ENV", "prod")}" '
@@ -1372,20 +1431,30 @@ def inference_ws_server_execute(self: ActionInstance):
     
     logging.info(f"Inference WebSocket server will use IP: {ws_host} on port 8102 (use_host_network={use_host_network})")
 
-    if action_details["actionDetails"].get("containerId"):
-        logging.info(
-            "Using existing container ID for inference WebSocket server: %s",
-            action_details["actionDetails"]["containerId"],
-        )
-        self.docker_container = action_details["actionDetails"]["containerId"]
-        cmd = "docker restart " + self.docker_container 
-        self.start(cmd, "inference_ws_server")
-        return
+    existing_container_id = action_details["actionDetails"].get("containerId")
+    if existing_container_id:
+        # Check if container actually exists before trying to restart
+        if ActionInstance.container_exists(existing_container_id):
+            logging.info(
+                "Using existing container ID for inference WebSocket server: %s",
+                existing_container_id,
+            )
+            self.docker_container = existing_container_id
+            cmd = "docker restart " + self.docker_container
+            self.start(cmd, "inference_ws_server")
+            return
+        else:
+            logging.warning(
+                "Container %s not found. Creating new container.",
+                existing_container_id
+            )
+            # Fall through to create new container
 
     # Inference WebSocket server with --net=host (Port: 8102)
+    container_name = f"inference_ws_{self.action_record_id}"
     worker_cmd = (
         f"docker run -d --pull=always --net=host "
-        f"--name inference "
+        f"--name {container_name} "
         f"--cidfile ./{self.action_record_id}.cid "
         f'-e ENV="{os.environ.get("ENV", "prod")}" '
         f'-e MATRICE_SECRET_ACCESS_KEY="{self.matrice_secret_access_key}" '
@@ -1420,20 +1489,30 @@ def fe_fs_streaming_execute(self: ActionInstance):
     
     logging.info(f"Frontend streaming will connect to WebSocket at: {ws_url}")
 
-    if action_details["actionDetails"].get("containerId"):
-        logging.info(
-            "Using existing container ID for frontend streaming: %s",
-            action_details["actionDetails"]["containerId"],
-        )
-        self.docker_container = action_details["actionDetails"]["containerId"]
-        cmd = "docker restart " + self.docker_container 
-        self.start(cmd, "fe_fs_streaming")
-        return
-    
+    existing_container_id = action_details["actionDetails"].get("containerId")
+    if existing_container_id:
+        # Check if container actually exists before trying to restart
+        if ActionInstance.container_exists(existing_container_id):
+            logging.info(
+                "Using existing container ID for frontend streaming: %s",
+                existing_container_id,
+            )
+            self.docker_container = existing_container_id
+            cmd = "docker restart " + self.docker_container
+            self.start(cmd, "fe_fs_streaming")
+            return
+        else:
+            logging.warning(
+                "Container %s not found. Creating new container.",
+                existing_container_id
+            )
+            # Fall through to create new container
+
     # Frontend streaming with --net=host (Port: 3000)
+    container_name = f"fe_streaming_{self.action_record_id}"
     worker_cmd = (
         f"docker run -d --pull=always --net=host "
-        f"--name fe_streaming "
+        f"--name {container_name} "
         f"--cidfile ./{self.action_record_id}.cid "
         f"-v matrice_myvol:/matrice_data "
         f'-e ENV="{os.environ.get("ENV", "prod")}" '
@@ -1465,20 +1544,30 @@ def fe_analytics_service_execute(self: ActionInstance):
 
     project_id = action_details["_idProject"]
 
-    if action_details["actionDetails"].get("containerId"):
-        logging.info(
-            "Using existing container ID for frontend analytics service: %s",
-            action_details["actionDetails"]["containerId"],
-        )
-        self.docker_container = action_details["actionDetails"]["containerId"]
-        cmd = "docker restart " + self.docker_container 
-        self.start(cmd, "fe_analytics_service")
-        return
-    
+    existing_container_id = action_details["actionDetails"].get("containerId")
+    if existing_container_id:
+        # Check if container actually exists before trying to restart
+        if ActionInstance.container_exists(existing_container_id):
+            logging.info(
+                "Using existing container ID for frontend analytics service: %s",
+                existing_container_id,
+            )
+            self.docker_container = existing_container_id
+            cmd = "docker restart " + self.docker_container
+            self.start(cmd, "fe_analytics_service")
+            return
+        else:
+            logging.warning(
+                "Container %s not found. Creating new container.",
+                existing_container_id
+            )
+            # Fall through to create new container
+
     # Frontend analytics service with --net=host (Port: 3001)
+    container_name = f"fe_analytics_{self.action_record_id}"
     worker_cmd = (
         f"docker run -d --pull=always --net=host "
-        f"--name fe-analytics "
+        f"--name {container_name} "
         f"--cidfile ./{self.action_record_id}.cid "
         f'-e NEXT_PUBLIC_DEPLOYMENT_ENV="{os.environ.get("ENV", "prod")}" '
         f'-e MATRICE_SECRET_ACCESS_KEY="{self.matrice_secret_access_key}" '
@@ -1510,7 +1599,8 @@ def synthetic_dataset_generation_execute(self: ActionInstance):
     else:
         return
     use_gpu = self.get_gpu_config(action_details)
-    cmd = f'{self.get_base_docker_cmd(work_fs=work_fs, use_gpu=use_gpu, extra_env_vars=extra_env_vars, extra_pkgs=["matrice_dataset"])} python3 /usr/src/app/synthetic_dataset_generation.py {self.action_record_id} "'
+    container_name = f"dataset_generation_{self.action_record_id}"
+    cmd = f'{self.get_base_docker_cmd(work_fs=work_fs, use_gpu=use_gpu, extra_env_vars=extra_env_vars, extra_pkgs=["matrice_dataset"], container_name=container_name)} python3 /usr/src/app/synthetic_dataset_generation.py {self.action_record_id} "'
     logging.info("cmd is: %s", cmd)
     self.start(cmd, "dataset_generation")
 
@@ -1531,7 +1621,8 @@ def synthetic_data_setup_execute(self: ActionInstance):
     else:
         return
     use_gpu = self.get_gpu_config(action_details)
-    cmd = f'{self.get_base_docker_cmd(work_fs=work_fs, use_gpu=use_gpu, extra_env_vars=extra_env_vars, extra_pkgs=["matrice_dataset"])} python3 /usr/src/app/data_generation.py {self.action_record_id} {external_port} "'
+    container_name = f"synthetic_data_setup_{self.action_record_id}"
+    cmd = f'{self.get_base_docker_cmd(work_fs=work_fs, use_gpu=use_gpu, extra_env_vars=extra_env_vars, extra_pkgs=["matrice_dataset"], container_name=container_name)} python3 /usr/src/app/data_generation.py {self.action_record_id} {external_port} "'
     logging.info("cmd is: %s", cmd)
     self.start(cmd, "synthetic_data_setup")
 
@@ -1568,26 +1659,40 @@ def redis_setup_execute(self: ActionInstance):
 
     redis_image = action_details["actionDetails"].get("redis_image", "redis:latest")
 
+    # Define container names with action_record_id for uniqueness
+    redis_container_name = f"redis_{self.action_record_id}"
 
-    if action_details["actionDetails"].get("containerId"):
-        logging.info(
-            "Using existing container ID for redis management: %s",
-            action_details["actionDetails"]["containerId"],
-        )
-        self.docker_container = action_details["actionDetails"]["containerId"]
-        cmd = "docker restart " + self.docker_container 
-        self.start(cmd, "redis_setup")
+    existing_container_id = action_details["actionDetails"].get("containerId")
+    if existing_container_id:
+        # Check if both containers actually exist before trying to restart
+        management_container_exists = ActionInstance.container_exists(existing_container_id)
+        redis_container_exists = ActionInstance.container_exists(redis_container_name)
 
-        # Redis container restart
-        redis_restart_cmd = "docker restart redis_container"
-        self.start(redis_restart_cmd, "redis")
+        if management_container_exists and redis_container_exists:
+            logging.info(
+                "Using existing container ID for redis management: %s",
+                existing_container_id,
+            )
+            self.docker_container = existing_container_id
+            cmd = "docker restart " + self.docker_container
+            self.start(cmd, "redis_setup")
+
+            # Redis container restart
+            redis_restart_cmd = f"docker restart {redis_container_name}"
+            self.start(redis_restart_cmd, "redis")
+            return
+        else:
+            logging.warning(
+                "Container(s) not found (management=%s, redis=%s). Creating new containers.",
+                management_container_exists,
+                redis_container_exists
+            )
+            # Fall through to create new containers
 
-        return
-    
     # Redis container with --net=host (Port: 6379)
     redis_cmd = (
         f"docker run -d --net=host "
-        f"--name redis_container "
+        f"--name {redis_container_name} "
         f"--restart unless-stopped "
         f"{redis_image} "
         f"redis-server --bind 0.0.0.0 "
@@ -1657,7 +1762,8 @@ def deploy_aggregator_execute(
     if not action_details:
         return
     self.setup_action_requirements(action_details, work_fs)
-    cmd = f'{self.get_base_docker_cmd(work_fs)} python3 /usr/src/app/deploy_aggregator.py {self.action_record_id} "'
+    container_name = f"deploy_aggregator_{self.action_record_id}"
+    cmd = f'{self.get_base_docker_cmd(work_fs, container_name=container_name)} python3 /usr/src/app/deploy_aggregator.py {self.action_record_id} "'
     logging.info("cmd: %s", cmd)
     self.start(cmd, "deploy_aggregator")
 
@@ -1705,7 +1811,8 @@ def model_deploy_execute(self: ActionInstance):
         "TRITON_PORTS": triton_ports
     }
 
-    cmd = f'{self.get_base_docker_cmd(work_fs, use_gpu, mount_docker_sock=True, action_id=action_id, extra_env_vars=extra_env_vars, extra_pkgs=["matrice_inference", "matrice_analytics"])} python3 deploy.py {self.action_record_id} {external_port}"'
+    container_name = f"model_deploy_{self.action_record_id}"
+    cmd = f'{self.get_base_docker_cmd(work_fs, use_gpu, mount_docker_sock=True, action_id=action_id, extra_env_vars=extra_env_vars, extra_pkgs=["matrice_inference", "matrice_analytics"], container_name=container_name)} python3 deploy.py {self.action_record_id} {external_port}"'
     logging.info("cmd is: %s", cmd)
     self.start(cmd, "deploy_log")
 
@@ -1728,17 +1835,27 @@ def model_train_execute(self: ActionInstance):
         action_id=action_id,
     )
 
-    if action_details["actionDetails"].get("containerId"):
-        logging.info(
-            "Using existing container ID for training: %s",
-            action_details["actionDetails"]["containerId"],
-        )
-        self.docker_container = action_details["actionDetails"]["containerId"]
-        cmd = "docker restart " + self.docker_container 
-        self.start(cmd, "train_log")
-        return
-    
-    cmd = f'{self.get_base_docker_cmd(work_fs, use_gpu, action_id=action_id, model_key=model_key)} python3 train.py {self.action_record_id} "'
+    existing_container_id = action_details["actionDetails"].get("containerId")
+    if existing_container_id:
+        # Check if container actually exists before trying to restart
+        if ActionInstance.container_exists(existing_container_id):
+            logging.info(
+                "Using existing container ID for training: %s",
+                existing_container_id,
+            )
+            self.docker_container = existing_container_id
+            cmd = "docker restart " + self.docker_container
+            self.start(cmd, "train_log")
+            return
+        else:
+            logging.warning(
+                "Container %s not found. Creating new container.",
+                existing_container_id
+            )
+            # Fall through to create new container
+
+    container_name = f"model_train_{self.action_record_id}"
+    cmd = f'{self.get_base_docker_cmd(work_fs, use_gpu, action_id=action_id, model_key=model_key, container_name=container_name)} python3 train.py {self.action_record_id} "'
     logging.info("cmd is: %s", cmd)
     self.start(cmd, "train_log")
 
@@ -1759,17 +1876,27 @@ def model_eval_execute(self: ActionInstance):
         model_family=model_family,
         action_id=action_id,
     )
-    if action_details["actionDetails"].get("containerId"):
-        logging.info(
-            "Using existing container ID for training: %s",
-            action_details["actionDetails"]["containerId"],
-        )
-        self.docker_container = action_details["actionDetails"]["containerId"]
-        cmd = "docker restart " + self.docker_container 
-        self.start(cmd, "eval_log")
-        return
-    
-    cmd = f'{self.get_base_docker_cmd(work_fs, use_gpu, action_id=action_id)} python3 eval.py {self.action_record_id} "'
+    existing_container_id = action_details["actionDetails"].get("containerId")
+    if existing_container_id:
+        # Check if container actually exists before trying to restart
+        if ActionInstance.container_exists(existing_container_id):
+            logging.info(
+                "Using existing container ID for evaluation: %s",
+                existing_container_id,
+            )
+            self.docker_container = existing_container_id
+            cmd = "docker restart " + self.docker_container
+            self.start(cmd, "eval_log")
+            return
+        else:
+            logging.warning(
+                "Container %s not found. Creating new container.",
+                existing_container_id
+            )
+            # Fall through to create new container
+
+    container_name = f"model_eval_{self.action_record_id}"
+    cmd = f'{self.get_base_docker_cmd(work_fs, use_gpu, action_id=action_id, container_name=container_name)} python3 eval.py {self.action_record_id} "'
     logging.info("cmd is: %s", cmd)
     self.start(cmd, "eval_log")
 
@@ -1793,17 +1920,27 @@ def model_export_execute(self: ActionInstance):
         model_family=model_family,
         action_id=action_id,
     )
-    if action_details["actionDetails"].get("containerId"):
-        logging.info(
-            "Using existing container ID for training: %s",
-            action_details["actionDetails"]["containerId"],
-        )
-        self.docker_container = action_details["actionDetails"]["containerId"]
-        cmd = "docker restart " + self.docker_container 
-        self.start(cmd, "export_log")
-        return
-    
-    cmd = f'{self.get_base_docker_cmd(work_fs, use_gpu, action_id=action_id)} python3 export.py {self.action_record_id} "'
+    existing_container_id = action_details["actionDetails"].get("containerId")
+    if existing_container_id:
+        # Check if container actually exists before trying to restart
+        if ActionInstance.container_exists(existing_container_id):
+            logging.info(
+                "Using existing container ID for export: %s",
+                existing_container_id,
+            )
+            self.docker_container = existing_container_id
+            cmd = "docker restart " + self.docker_container
+            self.start(cmd, "export_log")
+            return
+        else:
+            logging.warning(
+                "Container %s not found. Creating new container.",
+                existing_container_id
+            )
+            # Fall through to create new container
+
+    container_name = f"model_export_{self.action_record_id}"
+    cmd = f'{self.get_base_docker_cmd(work_fs, use_gpu, action_id=action_id, container_name=container_name)} python3 export.py {self.action_record_id} "'
     logging.info("cmd is: %s", cmd)
     self.start(cmd, "export_log")
 
@@ -1819,7 +1956,8 @@ def image_build_execute(self: ActionInstance):
     action_id = action_details["_id"]
     internal_api_key = self.get_internal_api_key(action_id)
     extra_env_vars = {"MATRICE_INTERNAL_API_KEY": internal_api_key}
-    cmd = f'{self.get_base_docker_cmd(mount_docker_sock=True, extra_env_vars=extra_env_vars)} python3 main.py {model_family_id} {action_id}"'
+    container_name = f"image_build_{self.action_record_id}"
+    cmd = f'{self.get_base_docker_cmd(mount_docker_sock=True, extra_env_vars=extra_env_vars, container_name=container_name)} python3 main.py {model_family_id} {action_id}"'
     logging.info("cmd is: %s", cmd)
     self.start(cmd, "image_build_log")
 
@@ -1831,7 +1969,8 @@ def resource_clone_execute(self: ActionInstance):
     if not action_details:
         return
     self.setup_action_requirements(action_details)
-    cmd = f'{self.get_base_docker_cmd()} python3 main.py {self.action_record_id} "'
+    container_name = f"resource_clone_{self.action_record_id}"
+    cmd = f'{self.get_base_docker_cmd(container_name=container_name)} python3 main.py {self.action_record_id} "'
     logging.info("cmd is: %s", cmd)
     self.start(cmd, "resource_clone")
 
@@ -1847,17 +1986,27 @@ def streaming_gateway_execute(self: ActionInstance):
         self.docker_container = (
             f"aiforeveryone/streaming-gateway:{os.environ.get('ENV', 'prod')}"
         )
-    if action_details["actionDetails"].get("containerId"):
-        logging.info(
-            "Using existing container ID for training: %s",
-            action_details["actionDetails"]["containerId"],
-        )
-        self.docker_container = action_details["actionDetails"]["containerId"]
-        cmd = "docker restart " + self.docker_container 
-        self.start(cmd, "streaming_gateway")
-        return
-    
-    cmd = f'{self.get_base_docker_cmd(extra_pkgs=["matrice_streaming"])} python3 /usr/src/app/streaming_gateway.py {self.action_record_id} "'
+    existing_container_id = action_details["actionDetails"].get("containerId")
+    if existing_container_id:
+        # Check if container actually exists before trying to restart
+        if ActionInstance.container_exists(existing_container_id):
+            logging.info(
+                "Using existing container ID for streaming gateway: %s",
+                existing_container_id,
+            )
+            self.docker_container = existing_container_id
+            cmd = "docker restart " + self.docker_container
+            self.start(cmd, "streaming_gateway")
+            return
+        else:
+            logging.warning(
+                "Container %s not found. Creating new container.",
+                existing_container_id
+            )
+            # Fall through to create new container
+
+    container_name = f"streaming_gateway_{self.action_record_id}"
+    cmd = f'{self.get_base_docker_cmd(extra_pkgs=["matrice_streaming"], container_name=container_name)} python3 /usr/src/app/streaming_gateway.py {self.action_record_id} "'
     logging.info("cmd is: %s", cmd)
     self.start(cmd, "streaming_gateway")
 
@@ -1951,16 +2100,24 @@ def kafka_setup_execute(self: ActionInstance):
     else:
         pkgs = f"matrice_common matrice"
 
-    if action_details["actionDetails"].get("containerId"):
-        logging.info(
-            "Using existing container ID for training: %s",
-            action_details["actionDetails"]["containerId"],
-        )
-        self.docker_container = action_details["actionDetails"]["containerId"]
-        cmd = "docker restart " + self.docker_container 
-        self.start(cmd, "kafka_setup")
-        return
-
+    existing_container_id = action_details["actionDetails"].get("containerId")
+    if existing_container_id:
+        # Check if container actually exists before trying to restart
+        if ActionInstance.container_exists(existing_container_id):
+            logging.info(
+                "Using existing container ID for kafka: %s",
+                existing_container_id,
+            )
+            self.docker_container = existing_container_id
+            cmd = "docker restart " + self.docker_container
+            self.start(cmd, "kafka_setup")
+            return
+        else:
+            logging.warning(
+                "Container %s not found. Creating new container.",
+                existing_container_id
+            )
+            # Fall through to create new container
 
     # Kafka container with --net=host (Ports: 9092, 9093)
     cmd = (
@@ -1997,21 +2154,31 @@ def inference_tracker_setup_execute(self: ActionInstance):
     
     self.setup_action_requirements(action_details)
 
-    if action_details["actionDetails"].get("containerId"):
-        logging.info(
-            "Using existing container ID for inference tracker: %s",
-            action_details["actionDetails"]["containerId"],
-        )
-        self.docker_container = action_details["actionDetails"]["containerId"]
-        cmd = "docker restart " + self.docker_container 
-        self.start(cmd, "inference_tracker_setup")
-        return
-    
+    existing_container_id = action_details["actionDetails"].get("containerId")
+    if existing_container_id:
+        # Check if container actually exists before trying to restart
+        if ActionInstance.container_exists(existing_container_id):
+            logging.info(
+                "Using existing container ID for inference tracker: %s",
+                existing_container_id,
+            )
+            self.docker_container = existing_container_id
+            cmd = "docker restart " + self.docker_container
+            self.start(cmd, "inference_tracker_setup")
+            return
+        else:
+            logging.warning(
+                "Container %s not found. Creating new container.",
+                existing_container_id
+            )
+            # Fall through to create new container
+
     # This is the existing Docker run command
+    container_name = f"inference_tracker_{self.action_record_id}"
     worker_cmd = (
         f"docker run -d --pull=always --net=host "
-         f"--cidfile ./{self.action_record_id}.cid "
-        f"--name inference-tracker-worker "
+        f"--cidfile ./{self.action_record_id}.cid "
+        f"--name {container_name} "
         f"-v matrice_myvol:/matrice_data "
         f'-e ENV="{os.environ.get("ENV", "prod")}" '
         f'-e MATRICE_SECRET_ACCESS_KEY="{self.matrice_secret_access_key}" '

matrice_compute/actions_manager.py

@@ -224,7 +224,7 @@ class ActionsManager:
                 action_ids
             )
         else:
-            logging.debug("No actions currently running")
+            logging.info("No actions currently running")
 
         return self.current_actions
 

matrice_compute/instance_manager.py

@@ -404,7 +404,7 @@ class InstanceManager:
                 if self.container_kafka_producer:
                     try:
                         self.container_kafka_producer.send(topic_name, status_message)
-                        logging.debug("Container status monitor: Sent status for %d containers", len(containers))
+                        logging.info("Container status monitor: Sent status for %d containers", len(containers))
                     except Exception as e:
                         logging.error("Container status monitor: Failed to send to Kafka: %s", str(e))
                 

matrice_compute/scaling.py

@@ -295,7 +295,7 @@ class Scaling:
                 logging.warning(f"Kafka returned error for {api}, falling back to REST")
 
         # Kafka failed or disabled, try REST
-        logging.info(f"Using REST API for {api}")
+        logging.debug(f"Using REST API for {api}")
         try:
             rest_response = rest_fallback_func()
 

matrice_compute/shutdown_manager.py

@@ -185,7 +185,7 @@ class ShutdownManager:
                             time.sleep(2)
                             return True
                     except Exception as e:
-                        logging.debug("Aggressive command failed: %s", str(e))
+                        logging.info("Aggressive command failed: %s", str(e))
         except Exception as e:
             logging.error("Error in aggressive shutdown methods: %s", str(e))
         return False
@@ -271,7 +271,7 @@ class ShutdownManager:
         """
         # CRITICAL: Check if this is a reserved instance that should not be shut down
         # if self.reserved_instance:
-        #     logging.debug("Reserved instance detected, skipping shutdown check")
+        #     logging.info("Reserved instance detected, skipping shutdown check")
         #     return
 
         # Update idle time tracking

{matrice_compute-0.1.30.dist-info → matrice_compute-0.1.32.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: matrice_compute
-Version: 0.1.30
+Version: 0.1.32
 Summary: Common server utilities for Matrice.ai services
 Author-email: "Matrice.ai" <dipendra@matrice.ai>
 License-Expression: MIT

matrice_compute-0.1.32.dist-info/RECORD

@@ -0,0 +1,18 @@
+matrice_compute/__init__.py,sha256=YZhx7rQlD1TAlhBMbsU3_Xp-tpLyTAxWZDcQvqmwR2g,723
+matrice_compute/action_instance.py,sha256=j6_3OG82HT7WcdWMy6VjEWwYxELfLhGJ1Y6ZaoRgWig,85420
+matrice_compute/actions_manager.py,sha256=14DKWfdJ145oyA0x5YVaj4ylnKE5Kd6xJZ5xzk0Jres,18147
+matrice_compute/actions_scaledown_manager.py,sha256=pJ0nduNwHWZ10GnqJNx0Ok7cVWabQ_M8E2Vb9pH3A_k,2002
+matrice_compute/compute_operations_handler.py,sha256=amcMhmXtv2irE6qK8Vbgec_8uFqjWmVVp0VWq-73_MU,17781
+matrice_compute/instance_manager.py,sha256=9u3QRTP-MkAWmrSQMMbCKc0TfK584teAg1wWIaqMZdE,19291
+matrice_compute/instance_utils.py,sha256=N4yPDvNukFEEBngR0lEt4x_XT5hur1q0P-spM2xQIlU,42025
+matrice_compute/prechecks.py,sha256=W9YmNF3RcLhOf4U8WBlExvFqDw1aGWSNTlJtA73lbDQ,17196
+matrice_compute/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+matrice_compute/resources_tracker.py,sha256=1jSLrIFlOh-vgyNzFrUrE2Ak2JAGCIfV7wcyEPJ0f2c,32246
+matrice_compute/scaling.py,sha256=UQDI8wN9JEKafvUVPF0Pk9XmhKlbMkeu16AZyyOuSE8,55147
+matrice_compute/shutdown_manager.py,sha256=rnP9Qes6JJKDnebmBC9rqkH__X9a8TMjhWQPWoOQKFs,13232
+matrice_compute/task_utils.py,sha256=3qIutiQdYPyGRxH9ZwLbqdg8sZcnp6jp08pszWCRFl0,2820
+matrice_compute-0.1.32.dist-info/licenses/LICENSE.txt,sha256=_uQUZpgO0mRYL5-fPoEvLSbNnLPv6OmbeEDCHXhK6Qc,1066
+matrice_compute-0.1.32.dist-info/METADATA,sha256=DMQ2-4mfoiU0aUvxsTVe7lcvhrZ5_uiIvzkIun_6sP4,1038
+matrice_compute-0.1.32.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+matrice_compute-0.1.32.dist-info/top_level.txt,sha256=63Plr3L1GzBUWZO5JZaFkiv8IcB10xUPU-9w3i6ptvE,16
+matrice_compute-0.1.32.dist-info/RECORD,,

matrice_compute-0.1.30.dist-info/RECORD

@@ -1,18 +0,0 @@
-matrice_compute/__init__.py,sha256=ZzQcFsT005VCgq9VZUh565f4upOooEb_FwZ6RgweNZs,597
-matrice_compute/action_instance.py,sha256=qe_5AYptyLyFbFLyNUW9e762vAZnIRtyHTtnu_8GxPM,77359
-matrice_compute/actions_manager.py,sha256=Iex5uw0PLRR4pvIAZDxc2CypucbanKDbJ3SK8mMGXK8,18148
-matrice_compute/actions_scaledown_manager.py,sha256=pJ0nduNwHWZ10GnqJNx0Ok7cVWabQ_M8E2Vb9pH3A_k,2002
-matrice_compute/compute_operations_handler.py,sha256=amcMhmXtv2irE6qK8Vbgec_8uFqjWmVVp0VWq-73_MU,17781
-matrice_compute/instance_manager.py,sha256=kPZYfiq3Oevs5r1xzwvDzE27zeWF9oBBxh9KhpHJuG4,19292
-matrice_compute/instance_utils.py,sha256=N4yPDvNukFEEBngR0lEt4x_XT5hur1q0P-spM2xQIlU,42025
-matrice_compute/prechecks.py,sha256=W9YmNF3RcLhOf4U8WBlExvFqDw1aGWSNTlJtA73lbDQ,17196
-matrice_compute/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-matrice_compute/resources_tracker.py,sha256=1jSLrIFlOh-vgyNzFrUrE2Ak2JAGCIfV7wcyEPJ0f2c,32246
-matrice_compute/scaling.py,sha256=cdEJqdVsPGDeOjkVAG85lubOn-qwDRV5qqmrNl_XpCM,55146
-matrice_compute/shutdown_manager.py,sha256=0MYV_AqygqR9NEntYf7atUC-PbWXyNkm1f-8c2aizgA,13234
-matrice_compute/task_utils.py,sha256=3qIutiQdYPyGRxH9ZwLbqdg8sZcnp6jp08pszWCRFl0,2820
-matrice_compute-0.1.30.dist-info/licenses/LICENSE.txt,sha256=_uQUZpgO0mRYL5-fPoEvLSbNnLPv6OmbeEDCHXhK6Qc,1066
-matrice_compute-0.1.30.dist-info/METADATA,sha256=kTDi915lwYwQJWMEyAT1F5GT5GfUW2SQfon5ki_X9tM,1038
-matrice_compute-0.1.30.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-matrice_compute-0.1.30.dist-info/top_level.txt,sha256=63Plr3L1GzBUWZO5JZaFkiv8IcB10xUPU-9w3i6ptvE,16
-matrice_compute-0.1.30.dist-info/RECORD,,