matrice-compute 0.1.31__py3-none-any.whl → 0.1.33__py3-none-any.whl

matrice_compute/__init__.py

@@ -1,6 +1,7 @@
 """Module providing __init__ functionality."""
 
 import subprocess
+import logging
 
 from matrice_common.utils import dependencies_check
 
@@ -17,4 +18,7 @@ subprocess.run( # Re-upgrade docker to avoid missing DOCKER_HOST connection erro
 
 from matrice_compute.instance_manager import InstanceManager  # noqa: E402
 
+logging.getLogger("kafka").setLevel(logging.INFO)
+logging.getLogger("confluent_kafka").setLevel(logging.INFO)
+
 __all__ = ["InstanceManager"]

matrice_compute/action_instance.py

@@ -296,7 +296,7 @@ class ActionInstance:
                         getattr(self, "action_record_id", "unknown"),
                     )
                 else:
-                    logging.debug(
+                    logging.info(
                         "No additional logs to send for action %s",
                         getattr(self, "action_record_id", "unknown"),
                     )
@@ -411,6 +411,7 @@ class ActionInstance:
         destination_workspace_path: str = "/usr/src/workspace",
         docker_workdir: str = "",
         extra_pkgs: list = [],
+        container_name: str = "",
     ):
         """Build base Docker command with common options.
 
@@ -425,6 +426,7 @@ class ActionInstance:
             destination_workspace_path (str): Container workspace path
             docker_workdir (str): Docker working directory
             extra_pkgs (list): List of extra packages to install
+            container_name (str): Docker container name (format: {action_type}_{action_id})
         Returns:
             str: Base Docker command
         """
@@ -489,8 +491,12 @@ class ActionInstance:
             ]
         )
 
+        # Build container name option if provided
+        name_option = f"--name {container_name}" if container_name else ""
+
         cmd_parts = [
             f"docker run -d {use_gpu} ",
+            name_option,
             network_config,
             *[f"-e {key}={shlex.quote(str(value))}" for key, value in env_vars.items()],
             *volumes,
@@ -882,6 +888,34 @@ class ActionInstance:
                 job_params=action_details["jobParams"],
             )
 
+    @staticmethod
+    def container_exists(container_id: str) -> bool:
+        """Check if a Docker container exists.
+
+        Args:
+            container_id (str): Container ID or name to check
+
+        Returns:
+            bool: True if container exists, False otherwise
+        """
+        if not container_id:
+            return False
+        try:
+            result = subprocess.run(
+                ["docker", "inspect", container_id],
+                capture_output=True,
+                text=True,
+                timeout=10
+            )
+            return result.returncode == 0
+        except Exception as e:
+            logging.warning(
+                "Error checking if container %s exists: %s",
+                container_id,
+                str(e)
+            )
+            return False
+
     @log_errors(raise_exception=True)
     def start_process(self, cmd, log_name):
         """Start the process and initialize logging.
@@ -905,7 +939,16 @@ class ActionInstance:
             env={**os.environ},
         )
         
-        stdout, stderr = process.communicate(timeout=120)
+        # Use a longer timeout for docker run since --pull=always may need to
+        # download large images on first run. Default: 30 minutes (1800 seconds)
+        # Can be configured via DOCKER_START_TIMEOUT_SECONDS environment variable
+        docker_start_timeout = int(os.environ.get("DOCKER_START_TIMEOUT_SECONDS", 1800))
+        logging.info(
+            "Waiting for docker container to start for action %s (timeout: %d seconds)",
+            self.action_record_id,
+            docker_start_timeout,
+        )
+        stdout, stderr = process.communicate(timeout=docker_start_timeout)
         
         if process.returncode != 0:
             logging.error(
@@ -1095,7 +1138,8 @@ def data_preparation_execute(
             "Started pulling Docker image with PID: %s",
             process.pid,
         )
-    cmd = f'{self.get_base_docker_cmd(work_fs, destination_workspace_path="/usr/src/app/workspace", docker_workdir="/usr/src/app/workspace", extra_pkgs=["matrice_dataset"])} python3 /usr/src/app/data_preparation.py {self.action_record_id} "'
+    container_name = f"data_prep_{self.action_record_id}"
+    cmd = f'{self.get_base_docker_cmd(work_fs, destination_workspace_path="/usr/src/app/workspace", docker_workdir="/usr/src/app/workspace", extra_pkgs=["matrice_dataset"], container_name=container_name)} python3 /usr/src/app/data_preparation.py {self.action_record_id} "'
     logging.info("cmd is: %s", cmd)
     self.start(cmd, "data_preparation_log")
 
@@ -1124,7 +1168,8 @@ def data_processing_execute(self: ActionInstance):
         service="bg-job-scheduler",
         job_params=action["jobParams"],
     )
-    cmd = f'{self.get_base_docker_cmd(work_fs, extra_pkgs=["matrice_dataset"])} python3 /usr/src/app/main.py {self.action_record_id} "'
+    container_name = f"data_processing_{self.action_record_id}"
+    cmd = f'{self.get_base_docker_cmd(work_fs, extra_pkgs=["matrice_dataset"], container_name=container_name)} python3 /usr/src/app/main.py {self.action_record_id} "'
     logging.info("cmd: %s", cmd)
     self.start(cmd, "data_processing_log")
 
@@ -1137,7 +1182,8 @@ def data_split_execute(self: ActionInstance):
     if not action_details:
         return
     self.setup_action_requirements(action_details, work_fs, model_family="")
-    cmd = f'{self.get_base_docker_cmd(work_fs, extra_pkgs=["matrice_dataset"])} python3 /usr/src/app/data_split.py {self.action_record_id} "'
+    container_name = f"data_split_{self.action_record_id}"
+    cmd = f'{self.get_base_docker_cmd(work_fs, extra_pkgs=["matrice_dataset"], container_name=container_name)} python3 /usr/src/app/data_split.py {self.action_record_id} "'
     logging.info("cmd: %s", cmd)
     self.start(cmd, "data_split")
 
@@ -1152,7 +1198,8 @@ def dataset_annotation_execute(
     if not action_details:
         return
     self.setup_action_requirements(action_details, work_fs)
-    cmd = f'{self.get_base_docker_cmd(work_fs, extra_pkgs=["matrice_dataset"])} python3 /usr/src/app/dataset_annotation.py {self.action_record_id} "'
+    container_name = f"dataset_annotation_{self.action_record_id}"
+    cmd = f'{self.get_base_docker_cmd(work_fs, extra_pkgs=["matrice_dataset"], container_name=container_name)} python3 /usr/src/app/dataset_annotation.py {self.action_record_id} "'
     logging.info("cmd: %s", cmd)
     self.start(cmd, "dataset_annotation")
 
@@ -1167,7 +1214,8 @@ def dataset_augmentation_execute(
     if not action_details:
         return
     self.setup_action_requirements(action_details, work_fs)
-    cmd = f'{self.get_base_docker_cmd(work_fs, extra_pkgs=["matrice_dataset"])} python3 /usr/src/app/data_augmentation.py {self.action_record_id} "'
+    container_name = f"dataset_augmentation_{self.action_record_id}"
+    cmd = f'{self.get_base_docker_cmd(work_fs, extra_pkgs=["matrice_dataset"], container_name=container_name)} python3 /usr/src/app/data_augmentation.py {self.action_record_id} "'
     logging.info("cmd: %s", cmd)
     self.start(cmd, "dataset_augmentation")
 
@@ -1183,7 +1231,8 @@ def augmentation_server_creation_execute(
     if not action_details:
         return
     self.setup_action_requirements(action_details, work_fs)
-    cmd = f'{self.get_base_docker_cmd(work_fs, extra_pkgs=["matrice_dataset"])} python3 /usr/src/app/aug_server.py {self.action_record_id} {external_port} "'
+    container_name = f"augmentation_setup_{self.action_record_id}"
+    cmd = f'{self.get_base_docker_cmd(work_fs, extra_pkgs=["matrice_dataset"], container_name=container_name)} python3 /usr/src/app/aug_server.py {self.action_record_id} {external_port} "'
     logging.info("cmd: %s", cmd)
     self.start(cmd, "augmentation_setup")
 
@@ -1204,25 +1253,41 @@ def database_setup_execute(self: ActionInstance):
 
     project_id = action_details["_idProject"]
 
-    if action_details["actionDetails"].get("containerId"):
-        logging.info(
-            "Using existing container ID for inference tracker: %s",
-            action_details["actionDetails"]["containerId"],
-        )
-        self.docker_container = action_details["actionDetails"]["containerId"]
-        cmd = "docker restart " + self.docker_container 
-        self.start(cmd, "qdrant_setup")
+    # Define container names with action_record_id for uniqueness
+    mongodb_container_name = f"database_setup_{self.action_record_id}"
+    qdrant_container_name = f"qdrant_{self.action_record_id}"
 
-        #qdrant restart
-        qdrant_cmd = "docker restart qdrant"
-        self.start(qdrant_cmd, 'qdrant_setup')
+    existing_container_id = action_details["actionDetails"].get("containerId")
+    if existing_container_id:
+        # Check if both containers actually exist before trying to restart
+        mongodb_container_exists = ActionInstance.container_exists(existing_container_id)
+        qdrant_container_exists = ActionInstance.container_exists(qdrant_container_name)
 
-        return
+        if mongodb_container_exists and qdrant_container_exists:
+            logging.info(
+                "Using existing container ID for database setup: %s",
+                existing_container_id,
+            )
+            self.docker_container = existing_container_id
+            cmd = "docker restart " + self.docker_container
+            self.start(cmd, "qdrant_setup")
+
+            # qdrant restart
+            qdrant_cmd = f"docker restart {qdrant_container_name}"
+            self.start(qdrant_cmd, "qdrant_setup")
+            return
+        else:
+            logging.warning(
+                "Container(s) not found (mongodb=%s, qdrant=%s). Creating new containers.",
+                mongodb_container_exists,
+                qdrant_container_exists
+            )
+            # Fall through to create new containers
 
     # MongoDB container with --net=host (Port: 27020:27017)
     cmd = (
         f"docker run --pull=always --net=host "
-        f"--name mongodbdatabase "
+        f"--name {mongodb_container_name} "
         f"-v matrice_myvol:/matrice_data "
         f"--cidfile ./{self.action_record_id}.cid "
         f"-e ACTION_RECORD_ID={self.action_record_id} "
@@ -1237,7 +1302,7 @@ def database_setup_execute(self: ActionInstance):
     # Qdrant container with --net=host (Port: 6334)
     qdrant_cmd = (
         f"docker run --pull=always --net=host "
-        f"--name qdrant "
+        f"--name {qdrant_container_name} "
         f"-v matrice_myvol:/matrice_data "
         f"{'qdrant/qdrant:latest'} "
     )
@@ -1263,23 +1328,32 @@ def facial_recognition_setup_execute(self: ActionInstance):
 
     self.setup_action_requirements(action_details)
 
-    if action_details["actionDetails"].get("containerId"):
-        logging.info(
-            "Using existing container ID for facial recognition worker: %s",
-            action_details["actionDetails"]["containerId"],
-        )
-        self.docker_container = action_details["actionDetails"]["containerId"]
-        cmd = "docker restart " + self.docker_container 
-        self.start(cmd, "facial_recognition_setup")
-        return
+    existing_container_id = action_details["actionDetails"].get("containerId")
+    if existing_container_id:
+        # Check if container actually exists before trying to restart
+        if ActionInstance.container_exists(existing_container_id):
+            logging.info(
+                "Using existing container ID for facial recognition worker: %s",
+                existing_container_id,
+            )
+            self.docker_container = existing_container_id
+            cmd = "docker restart " + self.docker_container
+            self.start(cmd, "facial_recognition_setup")
+            return
+        else:
+            logging.warning(
+                "Container %s not found. Creating new container.",
+                existing_container_id
+            )
+            # Fall through to create new container
 
     # Facial recognition worker container with --net=host (Port: 8081)
+    container_name = f"facial_recognition_{self.action_record_id}"
     worker_cmd = (
         f"docker run -d --pull=always --net=host "
-        f"--name worker "
-         f"--cidfile ./{self.action_record_id}.cid "
-        f"-v matrice_myvol:/matrice_data "
+        f"--name {container_name} "
         f"--cidfile ./{self.action_record_id}.cid "
+        f"-v matrice_myvol:/matrice_data "
         f'-e ENV="{os.environ.get("ENV", "prod")}" '
         f'-e MATRICE_SECRET_ACCESS_KEY="{self.matrice_secret_access_key}" '
         f'-e MATRICE_ACCESS_KEY_ID="{self.matrice_access_key_id}" '
@@ -1305,20 +1379,30 @@ def lpr_setup_execute(self: ActionInstance):
 
     self.setup_action_requirements(action_details)
 
-    if action_details["actionDetails"].get("containerId"):
-        logging.info(
-            "Using existing container ID for LPR worker: %s",
-            action_details["actionDetails"]["containerId"],
-        )
-        self.docker_container = action_details["actionDetails"]["containerId"]
-        cmd = "docker restart " + self.docker_container 
-        self.start(cmd, "lpr_setup")
-        return
+    existing_container_id = action_details["actionDetails"].get("containerId")
+    if existing_container_id:
+        # Check if container actually exists before trying to restart
+        if ActionInstance.container_exists(existing_container_id):
+            logging.info(
+                "Using existing container ID for LPR worker: %s",
+                existing_container_id,
+            )
+            self.docker_container = existing_container_id
+            cmd = "docker restart " + self.docker_container
+            self.start(cmd, "lpr_setup")
+            return
+        else:
+            logging.warning(
+                "Container %s not found. Creating new container.",
+                existing_container_id
+            )
+            # Fall through to create new container
 
     # LPR worker container with --net=host (Port: 8082)
+    container_name = f"lpr_{self.action_record_id}"
     worker_cmd = (
         f"docker run -d --net=host --pull=always "
-        f"--name lpr-worker "
+        f"--name {container_name} "
         f"--cidfile ./{self.action_record_id}.cid "
         f"-v matrice_myvol:/matrice_data "
         f'-e ENV="{os.environ.get("ENV", "prod")}" '
@@ -1356,20 +1440,30 @@ def inference_ws_server_execute(self: ActionInstance):
     
     logging.info(f"Inference WebSocket server will use IP: {ws_host} on port 8102 (use_host_network={use_host_network})")
 
-    if action_details["actionDetails"].get("containerId"):
-        logging.info(
-            "Using existing container ID for inference WebSocket server: %s",
-            action_details["actionDetails"]["containerId"],
-        )
-        self.docker_container = action_details["actionDetails"]["containerId"]
-        cmd = "docker restart " + self.docker_container 
-        self.start(cmd, "inference_ws_server")
-        return
+    existing_container_id = action_details["actionDetails"].get("containerId")
+    if existing_container_id:
+        # Check if container actually exists before trying to restart
+        if ActionInstance.container_exists(existing_container_id):
+            logging.info(
+                "Using existing container ID for inference WebSocket server: %s",
+                existing_container_id,
+            )
+            self.docker_container = existing_container_id
+            cmd = "docker restart " + self.docker_container
+            self.start(cmd, "inference_ws_server")
+            return
+        else:
+            logging.warning(
+                "Container %s not found. Creating new container.",
+                existing_container_id
+            )
+            # Fall through to create new container
 
     # Inference WebSocket server with --net=host (Port: 8102)
+    container_name = f"inference_ws_{self.action_record_id}"
     worker_cmd = (
         f"docker run -d --pull=always --net=host "
-        f"--name inference "
+        f"--name {container_name} "
         f"--cidfile ./{self.action_record_id}.cid "
         f'-e ENV="{os.environ.get("ENV", "prod")}" '
         f'-e MATRICE_SECRET_ACCESS_KEY="{self.matrice_secret_access_key}" '
@@ -1404,20 +1498,30 @@ def fe_fs_streaming_execute(self: ActionInstance):
     
     logging.info(f"Frontend streaming will connect to WebSocket at: {ws_url}")
 
-    if action_details["actionDetails"].get("containerId"):
-        logging.info(
-            "Using existing container ID for frontend streaming: %s",
-            action_details["actionDetails"]["containerId"],
-        )
-        self.docker_container = action_details["actionDetails"]["containerId"]
-        cmd = "docker restart " + self.docker_container 
-        self.start(cmd, "fe_fs_streaming")
-        return
-    
+    existing_container_id = action_details["actionDetails"].get("containerId")
+    if existing_container_id:
+        # Check if container actually exists before trying to restart
+        if ActionInstance.container_exists(existing_container_id):
+            logging.info(
+                "Using existing container ID for frontend streaming: %s",
+                existing_container_id,
+            )
+            self.docker_container = existing_container_id
+            cmd = "docker restart " + self.docker_container
+            self.start(cmd, "fe_fs_streaming")
+            return
+        else:
+            logging.warning(
+                "Container %s not found. Creating new container.",
+                existing_container_id
+            )
+            # Fall through to create new container
+
     # Frontend streaming with --net=host (Port: 3000)
+    container_name = f"fe_streaming_{self.action_record_id}"
     worker_cmd = (
         f"docker run -d --pull=always --net=host "
-        f"--name fe_streaming "
+        f"--name {container_name} "
         f"--cidfile ./{self.action_record_id}.cid "
         f"-v matrice_myvol:/matrice_data "
         f'-e ENV="{os.environ.get("ENV", "prod")}" '
@@ -1449,20 +1553,30 @@ def fe_analytics_service_execute(self: ActionInstance):
 
     project_id = action_details["_idProject"]
 
-    if action_details["actionDetails"].get("containerId"):
-        logging.info(
-            "Using existing container ID for frontend analytics service: %s",
-            action_details["actionDetails"]["containerId"],
-        )
-        self.docker_container = action_details["actionDetails"]["containerId"]
-        cmd = "docker restart " + self.docker_container 
-        self.start(cmd, "fe_analytics_service")
-        return
-    
+    existing_container_id = action_details["actionDetails"].get("containerId")
+    if existing_container_id:
+        # Check if container actually exists before trying to restart
+        if ActionInstance.container_exists(existing_container_id):
+            logging.info(
+                "Using existing container ID for frontend analytics service: %s",
+                existing_container_id,
+            )
+            self.docker_container = existing_container_id
+            cmd = "docker restart " + self.docker_container
+            self.start(cmd, "fe_analytics_service")
+            return
+        else:
+            logging.warning(
+                "Container %s not found. Creating new container.",
+                existing_container_id
+            )
+            # Fall through to create new container
+
     # Frontend analytics service with --net=host (Port: 3001)
+    container_name = f"fe_analytics_{self.action_record_id}"
     worker_cmd = (
         f"docker run -d --pull=always --net=host "
-        f"--name fe-analytics "
+        f"--name {container_name} "
         f"--cidfile ./{self.action_record_id}.cid "
         f'-e NEXT_PUBLIC_DEPLOYMENT_ENV="{os.environ.get("ENV", "prod")}" '
         f'-e MATRICE_SECRET_ACCESS_KEY="{self.matrice_secret_access_key}" '
@@ -1494,7 +1608,8 @@ def synthetic_dataset_generation_execute(self: ActionInstance):
     else:
         return
     use_gpu = self.get_gpu_config(action_details)
-    cmd = f'{self.get_base_docker_cmd(work_fs=work_fs, use_gpu=use_gpu, extra_env_vars=extra_env_vars, extra_pkgs=["matrice_dataset"])} python3 /usr/src/app/synthetic_dataset_generation.py {self.action_record_id} "'
+    container_name = f"dataset_generation_{self.action_record_id}"
+    cmd = f'{self.get_base_docker_cmd(work_fs=work_fs, use_gpu=use_gpu, extra_env_vars=extra_env_vars, extra_pkgs=["matrice_dataset"], container_name=container_name)} python3 /usr/src/app/synthetic_dataset_generation.py {self.action_record_id} "'
     logging.info("cmd is: %s", cmd)
     self.start(cmd, "dataset_generation")
 
@@ -1515,7 +1630,8 @@ def synthetic_data_setup_execute(self: ActionInstance):
     else:
         return
     use_gpu = self.get_gpu_config(action_details)
-    cmd = f'{self.get_base_docker_cmd(work_fs=work_fs, use_gpu=use_gpu, extra_env_vars=extra_env_vars, extra_pkgs=["matrice_dataset"])} python3 /usr/src/app/data_generation.py {self.action_record_id} {external_port} "'
+    container_name = f"synthetic_data_setup_{self.action_record_id}"
+    cmd = f'{self.get_base_docker_cmd(work_fs=work_fs, use_gpu=use_gpu, extra_env_vars=extra_env_vars, extra_pkgs=["matrice_dataset"], container_name=container_name)} python3 /usr/src/app/data_generation.py {self.action_record_id} {external_port} "'
     logging.info("cmd is: %s", cmd)
     self.start(cmd, "synthetic_data_setup")
 
@@ -1552,26 +1668,40 @@ def redis_setup_execute(self: ActionInstance):
 
     redis_image = action_details["actionDetails"].get("redis_image", "redis:latest")
 
+    # Define container names with action_record_id for uniqueness
+    redis_container_name = f"redis_{self.action_record_id}"
 
-    if action_details["actionDetails"].get("containerId"):
-        logging.info(
-            "Using existing container ID for redis management: %s",
-            action_details["actionDetails"]["containerId"],
-        )
-        self.docker_container = action_details["actionDetails"]["containerId"]
-        cmd = "docker restart " + self.docker_container 
-        self.start(cmd, "redis_setup")
+    existing_container_id = action_details["actionDetails"].get("containerId")
+    if existing_container_id:
+        # Check if both containers actually exist before trying to restart
+        management_container_exists = ActionInstance.container_exists(existing_container_id)
+        redis_container_exists = ActionInstance.container_exists(redis_container_name)
 
-        # Redis container restart
-        redis_restart_cmd = "docker restart redis_container"
-        self.start(redis_restart_cmd, "redis")
+        if management_container_exists and redis_container_exists:
+            logging.info(
+                "Using existing container ID for redis management: %s",
+                existing_container_id,
+            )
+            self.docker_container = existing_container_id
+            cmd = "docker restart " + self.docker_container
+            self.start(cmd, "redis_setup")
+
+            # Redis container restart
+            redis_restart_cmd = f"docker restart {redis_container_name}"
+            self.start(redis_restart_cmd, "redis")
+            return
+        else:
+            logging.warning(
+                "Container(s) not found (management=%s, redis=%s). Creating new containers.",
+                management_container_exists,
+                redis_container_exists
+            )
+            # Fall through to create new containers
 
-        return
-    
     # Redis container with --net=host (Port: 6379)
     redis_cmd = (
         f"docker run -d --net=host "
-        f"--name redis_container "
+        f"--name {redis_container_name} "
         f"--restart unless-stopped "
         f"{redis_image} "
         f"redis-server --bind 0.0.0.0 "
@@ -1641,7 +1771,8 @@ def deploy_aggregator_execute(
     if not action_details:
         return
     self.setup_action_requirements(action_details, work_fs)
-    cmd = f'{self.get_base_docker_cmd(work_fs)} python3 /usr/src/app/deploy_aggregator.py {self.action_record_id} "'
+    container_name = f"deploy_aggregator_{self.action_record_id}"
+    cmd = f'{self.get_base_docker_cmd(work_fs, container_name=container_name)} python3 /usr/src/app/deploy_aggregator.py {self.action_record_id} "'
     logging.info("cmd: %s", cmd)
     self.start(cmd, "deploy_aggregator")
 
@@ -1689,7 +1820,8 @@ def model_deploy_execute(self: ActionInstance):
         "TRITON_PORTS": triton_ports
     }
 
-    cmd = f'{self.get_base_docker_cmd(work_fs, use_gpu, mount_docker_sock=True, action_id=action_id, extra_env_vars=extra_env_vars, extra_pkgs=["matrice_inference", "matrice_analytics"])} python3 deploy.py {self.action_record_id} {external_port}"'
+    container_name = f"model_deploy_{self.action_record_id}"
+    cmd = f'{self.get_base_docker_cmd(work_fs, use_gpu, mount_docker_sock=True, action_id=action_id, extra_env_vars=extra_env_vars, extra_pkgs=["matrice_inference", "matrice_analytics"], container_name=container_name)} python3 deploy.py {self.action_record_id} {external_port}"'
     logging.info("cmd is: %s", cmd)
     self.start(cmd, "deploy_log")
 
@@ -1712,17 +1844,27 @@ def model_train_execute(self: ActionInstance):
         action_id=action_id,
     )
 
-    if action_details["actionDetails"].get("containerId"):
-        logging.info(
-            "Using existing container ID for training: %s",
-            action_details["actionDetails"]["containerId"],
-        )
-        self.docker_container = action_details["actionDetails"]["containerId"]
-        cmd = "docker restart " + self.docker_container 
-        self.start(cmd, "train_log")
-        return
-    
-    cmd = f'{self.get_base_docker_cmd(work_fs, use_gpu, action_id=action_id, model_key=model_key)} python3 train.py {self.action_record_id} "'
+    existing_container_id = action_details["actionDetails"].get("containerId")
+    if existing_container_id:
+        # Check if container actually exists before trying to restart
+        if ActionInstance.container_exists(existing_container_id):
+            logging.info(
+                "Using existing container ID for training: %s",
+                existing_container_id,
+            )
+            self.docker_container = existing_container_id
+            cmd = "docker restart " + self.docker_container
+            self.start(cmd, "train_log")
+            return
+        else:
+            logging.warning(
+                "Container %s not found. Creating new container.",
+                existing_container_id
+            )
+            # Fall through to create new container
+
+    container_name = f"model_train_{self.action_record_id}"
+    cmd = f'{self.get_base_docker_cmd(work_fs, use_gpu, action_id=action_id, model_key=model_key, container_name=container_name)} python3 train.py {self.action_record_id} "'
     logging.info("cmd is: %s", cmd)
     self.start(cmd, "train_log")
 
@@ -1743,17 +1885,27 @@ def model_eval_execute(self: ActionInstance):
         model_family=model_family,
         action_id=action_id,
     )
-    if action_details["actionDetails"].get("containerId"):
-        logging.info(
-            "Using existing container ID for training: %s",
-            action_details["actionDetails"]["containerId"],
-        )
-        self.docker_container = action_details["actionDetails"]["containerId"]
-        cmd = "docker restart " + self.docker_container 
-        self.start(cmd, "eval_log")
-        return
-    
-    cmd = f'{self.get_base_docker_cmd(work_fs, use_gpu, action_id=action_id)} python3 eval.py {self.action_record_id} "'
+    existing_container_id = action_details["actionDetails"].get("containerId")
+    if existing_container_id:
+        # Check if container actually exists before trying to restart
+        if ActionInstance.container_exists(existing_container_id):
+            logging.info(
+                "Using existing container ID for evaluation: %s",
+                existing_container_id,
+            )
+            self.docker_container = existing_container_id
+            cmd = "docker restart " + self.docker_container
+            self.start(cmd, "eval_log")
+            return
+        else:
+            logging.warning(
+                "Container %s not found. Creating new container.",
+                existing_container_id
+            )
+            # Fall through to create new container
+
+    container_name = f"model_eval_{self.action_record_id}"
+    cmd = f'{self.get_base_docker_cmd(work_fs, use_gpu, action_id=action_id, container_name=container_name)} python3 eval.py {self.action_record_id} "'
     logging.info("cmd is: %s", cmd)
     self.start(cmd, "eval_log")
 
@@ -1777,17 +1929,27 @@ def model_export_execute(self: ActionInstance):
         model_family=model_family,
         action_id=action_id,
     )
-    if action_details["actionDetails"].get("containerId"):
-        logging.info(
-            "Using existing container ID for training: %s",
-            action_details["actionDetails"]["containerId"],
-        )
-        self.docker_container = action_details["actionDetails"]["containerId"]
-        cmd = "docker restart " + self.docker_container 
-        self.start(cmd, "export_log")
-        return
-    
-    cmd = f'{self.get_base_docker_cmd(work_fs, use_gpu, action_id=action_id)} python3 export.py {self.action_record_id} "'
+    existing_container_id = action_details["actionDetails"].get("containerId")
+    if existing_container_id:
+        # Check if container actually exists before trying to restart
+        if ActionInstance.container_exists(existing_container_id):
+            logging.info(
+                "Using existing container ID for export: %s",
+                existing_container_id,
+            )
+            self.docker_container = existing_container_id
+            cmd = "docker restart " + self.docker_container
+            self.start(cmd, "export_log")
+            return
+        else:
+            logging.warning(
+                "Container %s not found. Creating new container.",
+                existing_container_id
+            )
+            # Fall through to create new container
+
+    container_name = f"model_export_{self.action_record_id}"
+    cmd = f'{self.get_base_docker_cmd(work_fs, use_gpu, action_id=action_id, container_name=container_name)} python3 export.py {self.action_record_id} "'
     logging.info("cmd is: %s", cmd)
     self.start(cmd, "export_log")
 
@@ -1803,7 +1965,8 @@ def image_build_execute(self: ActionInstance):
     action_id = action_details["_id"]
     internal_api_key = self.get_internal_api_key(action_id)
     extra_env_vars = {"MATRICE_INTERNAL_API_KEY": internal_api_key}
-    cmd = f'{self.get_base_docker_cmd(mount_docker_sock=True, extra_env_vars=extra_env_vars)} python3 main.py {model_family_id} {action_id}"'
+    container_name = f"image_build_{self.action_record_id}"
+    cmd = f'{self.get_base_docker_cmd(mount_docker_sock=True, extra_env_vars=extra_env_vars, container_name=container_name)} python3 main.py {model_family_id} {action_id}"'
     logging.info("cmd is: %s", cmd)
     self.start(cmd, "image_build_log")
 
@@ -1815,7 +1978,8 @@ def resource_clone_execute(self: ActionInstance):
     if not action_details:
         return
     self.setup_action_requirements(action_details)
-    cmd = f'{self.get_base_docker_cmd()} python3 main.py {self.action_record_id} "'
+    container_name = f"resource_clone_{self.action_record_id}"
+    cmd = f'{self.get_base_docker_cmd(container_name=container_name)} python3 main.py {self.action_record_id} "'
     logging.info("cmd is: %s", cmd)
     self.start(cmd, "resource_clone")
 
@@ -1831,17 +1995,27 @@ def streaming_gateway_execute(self: ActionInstance):
         self.docker_container = (
             f"aiforeveryone/streaming-gateway:{os.environ.get('ENV', 'prod')}"
         )
-    if action_details["actionDetails"].get("containerId"):
-        logging.info(
-            "Using existing container ID for training: %s",
-            action_details["actionDetails"]["containerId"],
-        )
-        self.docker_container = action_details["actionDetails"]["containerId"]
-        cmd = "docker restart " + self.docker_container 
-        self.start(cmd, "streaming_gateway")
-        return
-    
-    cmd = f'{self.get_base_docker_cmd(extra_pkgs=["matrice_streaming"])} python3 /usr/src/app/streaming_gateway.py {self.action_record_id} "'
+    existing_container_id = action_details["actionDetails"].get("containerId")
+    if existing_container_id:
+        # Check if container actually exists before trying to restart
+        if ActionInstance.container_exists(existing_container_id):
+            logging.info(
+                "Using existing container ID for streaming gateway: %s",
+                existing_container_id,
+            )
+            self.docker_container = existing_container_id
+            cmd = "docker restart " + self.docker_container
+            self.start(cmd, "streaming_gateway")
+            return
+        else:
+            logging.warning(
+                "Container %s not found. Creating new container.",
+                existing_container_id
+            )
+            # Fall through to create new container
+
+    container_name = f"streaming_gateway_{self.action_record_id}"
+    cmd = f'{self.get_base_docker_cmd(extra_pkgs=["matrice_streaming"], container_name=container_name)} python3 /usr/src/app/streaming_gateway.py {self.action_record_id} "'
     logging.info("cmd is: %s", cmd)
     self.start(cmd, "streaming_gateway")
 
@@ -1935,16 +2109,24 @@ def kafka_setup_execute(self: ActionInstance):
     else:
         pkgs = f"matrice_common matrice"
 
-    if action_details["actionDetails"].get("containerId"):
-        logging.info(
-            "Using existing container ID for training: %s",
-            action_details["actionDetails"]["containerId"],
-        )
-        self.docker_container = action_details["actionDetails"]["containerId"]
-        cmd = "docker restart " + self.docker_container 
-        self.start(cmd, "kafka_setup")
-        return
-
+    existing_container_id = action_details["actionDetails"].get("containerId")
+    if existing_container_id:
+        # Check if container actually exists before trying to restart
+        if ActionInstance.container_exists(existing_container_id):
+            logging.info(
+                "Using existing container ID for kafka: %s",
+                existing_container_id,
+            )
+            self.docker_container = existing_container_id
+            cmd = "docker restart " + self.docker_container
+            self.start(cmd, "kafka_setup")
+            return
+        else:
+            logging.warning(
+                "Container %s not found. Creating new container.",
+                existing_container_id
+            )
+            # Fall through to create new container
 
     # Kafka container with --net=host (Ports: 9092, 9093)
     cmd = (
@@ -1981,21 +2163,31 @@ def inference_tracker_setup_execute(self: ActionInstance):
     
     self.setup_action_requirements(action_details)
 
-    if action_details["actionDetails"].get("containerId"):
-        logging.info(
-            "Using existing container ID for inference tracker: %s",
-            action_details["actionDetails"]["containerId"],
-        )
-        self.docker_container = action_details["actionDetails"]["containerId"]
-        cmd = "docker restart " + self.docker_container 
-        self.start(cmd, "inference_tracker_setup")
-        return
-    
+    existing_container_id = action_details["actionDetails"].get("containerId")
+    if existing_container_id:
+        # Check if container actually exists before trying to restart
+        if ActionInstance.container_exists(existing_container_id):
+            logging.info(
+                "Using existing container ID for inference tracker: %s",
+                existing_container_id,
+            )
+            self.docker_container = existing_container_id
+            cmd = "docker restart " + self.docker_container
+            self.start(cmd, "inference_tracker_setup")
+            return
+        else:
+            logging.warning(
+                "Container %s not found. Creating new container.",
+                existing_container_id
+            )
+            # Fall through to create new container
+
     # This is the existing Docker run command
+    container_name = f"inference_tracker_{self.action_record_id}"
     worker_cmd = (
         f"docker run -d --pull=always --net=host "
-         f"--cidfile ./{self.action_record_id}.cid "
-        f"--name inference-tracker-worker "
+        f"--cidfile ./{self.action_record_id}.cid "
+        f"--name {container_name} "
         f"-v matrice_myvol:/matrice_data "
         f'-e ENV="{os.environ.get("ENV", "prod")}" '
         f'-e MATRICE_SECRET_ACCESS_KEY="{self.matrice_secret_access_key}" '

matrice_compute/actions_manager.py

@@ -43,7 +43,11 @@ class ActionsManager:
         """
         actions = []
         logging.info("Polling backend for new jobs")
-        fetched_actions, error, _ = self.scaling.assign_jobs(has_gpu())
+        result = self.scaling.assign_jobs(has_gpu())
+        if result is None:
+            logging.error("assign_jobs returned None")
+            return actions
+        fetched_actions, error, _ = result
         if error:
             logging.error("Error assigning jobs: %s", error)
             return actions
@@ -224,7 +228,7 @@ class ActionsManager:
                 action_ids
             )
         else:
-            logging.debug("No actions currently running")
+            logging.info("No actions currently running")
 
         return self.current_actions
 

matrice_compute/instance_manager.py

@@ -404,7 +404,7 @@ class InstanceManager:
                 if self.container_kafka_producer:
                     try:
                         self.container_kafka_producer.send(topic_name, status_message)
-                        logging.debug("Container status monitor: Sent status for %d containers", len(containers))
+                        logging.info("Container status monitor: Sent status for %d containers", len(containers))
                     except Exception as e:
                         logging.error("Container status monitor: Failed to send to Kafka: %s", str(e))
                 

matrice_compute/scaling.py

@@ -295,7 +295,7 @@ class Scaling:
                 logging.warning(f"Kafka returned error for {api}, falling back to REST")
 
         # Kafka failed or disabled, try REST
-        logging.info(f"Using REST API for {api}")
+        logging.debug(f"Using REST API for {api}")
         try:
             rest_response = rest_fallback_func()
 

matrice_compute/shutdown_manager.py

@@ -185,7 +185,7 @@ class ShutdownManager:
                             time.sleep(2)
                             return True
                     except Exception as e:
-                        logging.debug("Aggressive command failed: %s", str(e))
+                        logging.info("Aggressive command failed: %s", str(e))
         except Exception as e:
             logging.error("Error in aggressive shutdown methods: %s", str(e))
         return False
@@ -271,7 +271,7 @@ class ShutdownManager:
         """
         # CRITICAL: Check if this is a reserved instance that should not be shut down
         # if self.reserved_instance:
-        #     logging.debug("Reserved instance detected, skipping shutdown check")
+        #     logging.info("Reserved instance detected, skipping shutdown check")
         #     return
 
         # Update idle time tracking

{matrice_compute-0.1.31.dist-info → matrice_compute-0.1.33.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: matrice_compute
-Version: 0.1.31
+Version: 0.1.33
 Summary: Common server utilities for Matrice.ai services
 Author-email: "Matrice.ai" <dipendra@matrice.ai>
 License-Expression: MIT

matrice_compute-0.1.33.dist-info/RECORD

@@ -0,0 +1,18 @@
+matrice_compute/__init__.py,sha256=YZhx7rQlD1TAlhBMbsU3_Xp-tpLyTAxWZDcQvqmwR2g,723
+matrice_compute/action_instance.py,sha256=jWv-YlB1_YiaEU-oazGBMMwSHFh2TJ5NU88R7NxCtUM,85957
+matrice_compute/actions_manager.py,sha256=a_TulMnu462xc0t_A-Mpug5zhQTmtpjiv7mhiC_IAVw,18280
+matrice_compute/actions_scaledown_manager.py,sha256=pJ0nduNwHWZ10GnqJNx0Ok7cVWabQ_M8E2Vb9pH3A_k,2002
+matrice_compute/compute_operations_handler.py,sha256=amcMhmXtv2irE6qK8Vbgec_8uFqjWmVVp0VWq-73_MU,17781
+matrice_compute/instance_manager.py,sha256=9u3QRTP-MkAWmrSQMMbCKc0TfK584teAg1wWIaqMZdE,19291
+matrice_compute/instance_utils.py,sha256=N4yPDvNukFEEBngR0lEt4x_XT5hur1q0P-spM2xQIlU,42025
+matrice_compute/prechecks.py,sha256=W9YmNF3RcLhOf4U8WBlExvFqDw1aGWSNTlJtA73lbDQ,17196
+matrice_compute/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+matrice_compute/resources_tracker.py,sha256=1jSLrIFlOh-vgyNzFrUrE2Ak2JAGCIfV7wcyEPJ0f2c,32246
+matrice_compute/scaling.py,sha256=UQDI8wN9JEKafvUVPF0Pk9XmhKlbMkeu16AZyyOuSE8,55147
+matrice_compute/shutdown_manager.py,sha256=rnP9Qes6JJKDnebmBC9rqkH__X9a8TMjhWQPWoOQKFs,13232
+matrice_compute/task_utils.py,sha256=3qIutiQdYPyGRxH9ZwLbqdg8sZcnp6jp08pszWCRFl0,2820
+matrice_compute-0.1.33.dist-info/licenses/LICENSE.txt,sha256=_uQUZpgO0mRYL5-fPoEvLSbNnLPv6OmbeEDCHXhK6Qc,1066
+matrice_compute-0.1.33.dist-info/METADATA,sha256=iJWPWtfLyDOfNVvSxmH8BMLcXkFlX1Z7Lm6Sih-JSfY,1038
+matrice_compute-0.1.33.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+matrice_compute-0.1.33.dist-info/top_level.txt,sha256=63Plr3L1GzBUWZO5JZaFkiv8IcB10xUPU-9w3i6ptvE,16
+matrice_compute-0.1.33.dist-info/RECORD,,

matrice_compute-0.1.31.dist-info/RECORD

@@ -1,18 +0,0 @@
-matrice_compute/__init__.py,sha256=ZzQcFsT005VCgq9VZUh565f4upOooEb_FwZ6RgweNZs,597
-matrice_compute/action_instance.py,sha256=NpI7uCaLJ5GKdW-2JBGCjTwijb8XBrRc7GKRC4uhQF4,76650
-matrice_compute/actions_manager.py,sha256=Iex5uw0PLRR4pvIAZDxc2CypucbanKDbJ3SK8mMGXK8,18148
-matrice_compute/actions_scaledown_manager.py,sha256=pJ0nduNwHWZ10GnqJNx0Ok7cVWabQ_M8E2Vb9pH3A_k,2002
-matrice_compute/compute_operations_handler.py,sha256=amcMhmXtv2irE6qK8Vbgec_8uFqjWmVVp0VWq-73_MU,17781
-matrice_compute/instance_manager.py,sha256=kPZYfiq3Oevs5r1xzwvDzE27zeWF9oBBxh9KhpHJuG4,19292
-matrice_compute/instance_utils.py,sha256=N4yPDvNukFEEBngR0lEt4x_XT5hur1q0P-spM2xQIlU,42025
-matrice_compute/prechecks.py,sha256=W9YmNF3RcLhOf4U8WBlExvFqDw1aGWSNTlJtA73lbDQ,17196
-matrice_compute/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-matrice_compute/resources_tracker.py,sha256=1jSLrIFlOh-vgyNzFrUrE2Ak2JAGCIfV7wcyEPJ0f2c,32246
-matrice_compute/scaling.py,sha256=cdEJqdVsPGDeOjkVAG85lubOn-qwDRV5qqmrNl_XpCM,55146
-matrice_compute/shutdown_manager.py,sha256=0MYV_AqygqR9NEntYf7atUC-PbWXyNkm1f-8c2aizgA,13234
-matrice_compute/task_utils.py,sha256=3qIutiQdYPyGRxH9ZwLbqdg8sZcnp6jp08pszWCRFl0,2820
-matrice_compute-0.1.31.dist-info/licenses/LICENSE.txt,sha256=_uQUZpgO0mRYL5-fPoEvLSbNnLPv6OmbeEDCHXhK6Qc,1066
-matrice_compute-0.1.31.dist-info/METADATA,sha256=nhJU2AA0SxaSWMZXKjYtAthzjbjdEmmD3agMYqukQx8,1038
-matrice_compute-0.1.31.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-matrice_compute-0.1.31.dist-info/top_level.txt,sha256=63Plr3L1GzBUWZO5JZaFkiv8IcB10xUPU-9w3i6ptvE,16
-matrice_compute-0.1.31.dist-info/RECORD,,