matrice-compute 0.1.12__py3-none-any.whl → 0.1.14__py3-none-any.whl

matrice_compute/__init__.py

@@ -1,9 +1,20 @@
 """Module providing __init__ functionality."""
 
+import subprocess
 
 from matrice_common.utils import dependencies_check
 
-dependencies_check(["docker", "psutil", "cryptography", "notebook", "aiohttp", "kafka-python"])
+dependencies_check(
+    ["docker", "psutil", "cryptography", "notebook", "aiohttp", "kafka-python"]
+)
+
+subprocess.run( # Re-upgrade docker to avoid missing DOCKER_HOST connection error
+    ["pip", "install", "--upgrade", "docker"],
+    check=True,
+    stdout=subprocess.DEVNULL,   # suppress normal output
+    stderr=subprocess.DEVNULL    # suppress warnings/progress
+)
+
 from matrice_compute.instance_manager import InstanceManager  # noqa: E402
 
 __all__ = ["InstanceManager"]

matrice_compute/action_instance.py

@@ -74,7 +74,8 @@ class ActionInstance:
             "streaming_gateway": streaming_gateway_execute,
             "facial_recognition_setup": facial_recognition_setup_execute,
             "fe_fs_streaming": fe_fs_streaming_execute,
-            "inference_ws_server": inference_ws_server_execute
+            "inference_ws_server": inference_ws_server_execute,
+            "lpr_setup": lpr_setup_execute
         }
         if self.action_type not in self.actions_map:
             raise ValueError(f"Unknown action type: {self.action_type}")
@@ -574,75 +575,75 @@ class ActionInstance:
             )
             raise
 
-    @log_errors(raise_exception=False)
-    def create_redis_container(self, redis_image=None, redis_password=None):
-        """Create and start a Redis container using Docker.
-
-        Args:
-            redis_image (str, optional): Redis Docker image to use. Defaults to 'redis:latest'
-
-        Returns:
-            tuple: (container_info, error, message)
-        """
-        if redis_image is None:
-            redis_image = "redis:latest"
-
-        network_name = f"redis_network_{int(time.time())}"
-        subprocess.run(f"docker network create {network_name}", shell=True, check=True)
-
-        try:
-            # Get an available port for Redis
-            external_port = "6379"
-
-            # Generate a unique container name and password
-            container_name = f"redis_container_{int(time.time())}"
-
-            # Build the docker command to create Redis container with password
-            cmd = (
-                f"docker run -d "
-                f"--network {network_name} "
-                f"--name {container_name} "
-                f"-p {external_port}:6379 "
-                f"--restart unless-stopped "
-                f"{redis_image} "
-                f"redis-server --bind 0.0.0.0 --appendonly yes --requirepass {redis_password}"
-            )
-
-            logging.info("Creating Redis container with command: %s", cmd)
-
-            # Execute the command
-            result = subprocess.run(
-                cmd, shell=True, capture_output=True, text=True, timeout=60
-            )
-
-            if result.returncode == 0:
-                container_id = result.stdout.strip()
-                container_info = {
-                    "container_id": container_id,
-                    "container_name": container_name,
-                    "network_name": network_name,
-                    "external_port": external_port,
-                    "internal_port": 6379,
-                    "password": redis_password,
-                    "image": redis_image,
-                    "status": "running",
-                }
-
-                logging.info("Redis container created successfully: %s", container_info)
-                return container_info, None, "Redis container created successfully"
-            else:
-                error_message = f"Failed to create Redis container: {result.stderr}"
-                logging.error(error_message)
-                return None, "ContainerCreationError", error_message
-
-        except subprocess.TimeoutExpired:
-            error_message = "Timeout while creating Redis container"
-            logging.error(error_message)
-            return None, "TimeoutError", error_message
-        except Exception as e:
-            error_message = f"Unexpected error creating Redis container: {str(e)}"
-            logging.error(error_message)
-            return None, "UnexpectedError", error_message
+    # @log_errors(raise_exception=False)
+    # def create_redis_container(self, redis_image=None, redis_password=None):
+    #     """Create and start a Redis container using Docker.
+
+    #     Args:
+    #         redis_image (str, optional): Redis Docker image to use. Defaults to 'redis:latest'
+
+    #     Returns:
+    #         tuple: (container_info, error, message)
+    #     """
+    #     if redis_image is None:
+    #         redis_image = "redis:latest"
+
+    #     network_name = f"redis_network_{int(time.time())}"
+    #     subprocess.run(f"docker network create {network_name}", shell=True, check=True)
+
+    #     try:
+    #         # Get an available port for Redis
+    #         external_port = "6379"
+
+    #         # Generate a unique container name and password
+    #         container_name = f"redis_container_{int(time.time())}"
+
+    #         # Build the docker command to create Redis container with password
+    #         cmd = (
+    #             f"docker run -d "
+    #             f"--network {network_name} "
+    #             f"--name {container_name} "
+    #             f"-p {external_port}:6379 "
+    #             f"--restart unless-stopped "
+    #             f"{redis_image} "
+    #             f"redis-server --bind 0.0.0.0 --appendonly yes --requirepass {redis_password}"
+    #         )
+
+    #         logging.info("Creating Redis container with command: %s", cmd)
+
+    #         # Execute the command
+    #         result = subprocess.run(
+    #             cmd, shell=True, capture_output=True, text=True, timeout=60
+    #         )
+
+    #         if result.returncode == 0:
+    #             container_id = result.stdout.strip()
+    #             container_info = {
+    #                 "container_id": container_id,
+    #                 "container_name": container_name,
+    #                 "network_name": network_name,
+    #                 "external_port": external_port,
+    #                 "internal_port": 6379,
+    #                 "password": redis_password,
+    #                 "image": redis_image,
+    #                 "status": "running",
+    #             }
+
+    #             logging.info("Redis container created successfully: %s", container_info)
+    #             return container_info, None, "Redis container created successfully"
+    #         else:
+    #             error_message = f"Failed to create Redis container: {result.stderr}"
+    #             logging.error(error_message)
+    #             return None, "ContainerCreationError", error_message
+
+    #     except subprocess.TimeoutExpired:
+    #         error_message = "Timeout while creating Redis container"
+    #         logging.error(error_message)
+    #         return None, "TimeoutError", error_message
+    #     except Exception as e:
+    #         error_message = f"Unexpected error creating Redis container: {str(e)}"
+    #         logging.error(error_message)
+    #         return None, "UnexpectedError", error_message
 
     @log_errors(raise_exception=False, log_error=False)
     def send_logs_continuously(self):
@@ -1052,7 +1053,7 @@ def database_setup_execute(self: ActionInstance):
         f"-e MATRICE_ACCESS_KEY_ID={self.matrice_access_key_id} "
         f"-e MATRICE_SECRET_ACCESS_KEY={self.matrice_secret_access_key} "
         f"-e PROJECT_ID={project_id} "
-        f"-e ENV=dev "
+        f'-e ENV="{os.environ.get("ENV", "prod")}" '
         f"{image} "
     )
     print("Docker command", cmd)
@@ -1101,29 +1102,64 @@ def facial_recognition_setup_execute(self: ActionInstance):
     self.start(worker_cmd, "facial_recognition_setup")
 
 @log_errors(raise_exception=False)
-def inference_ws_server_execute(self: ActionInstance):
+def lpr_setup_execute(self: ActionInstance):
     """
-    Creates and start inference pipline.
+    Creates and setup the database for license plate server.
     """
     action_details = self.get_action_details()
 
     if not action_details:
         return
-    image = action_details["actionDetails"].get("docker")
+    image = self.docker_container
+    external_port = self.scaling.get_open_port()
 
     self.setup_action_requirements(action_details)
 
     # Add worker container run command
     worker_cmd = (
         f"docker run -d --pull=always "
-        f"--name inference "
-        f"-p 8102:8102 "
+        f"--name lpr-worker "
+        f"-p {external_port}:8082 "
         f'-e ENV="{os.environ.get("ENV", "prod")}" '
         f'-e MATRICE_SECRET_ACCESS_KEY="{self.matrice_secret_access_key}" '
         f'-e MATRICE_ACCESS_KEY_ID="{self.matrice_access_key_id}" '
+        f'-e ACTION_ID="{self.action_record_id}" '
+        f'-e PORT={external_port} '
         f"{image}"
     )
-    print("inference docker run command:", worker_cmd)
+    print("Worker docker run command:", worker_cmd)
+
+    # Docker Command run
+    self.start(worker_cmd, "lpr_setup")
+
+@log_errors(raise_exception=False)
+def inference_ws_server_execute(self: ActionInstance):
+    """
+    Creates and start inference pipeline.
+    Inference WebSocket server runs on port 8102 (localhost only with --net=host).
+    """
+    action_details = self.get_action_details()
+
+    if not action_details:
+        return
+    image = action_details["actionDetails"].get("docker")
+    
+
+    self.setup_action_requirements(action_details)
+
+    # Inference WebSocket server with --net=host (Port: 8102)
+    worker_cmd = (
+        f"docker run -d --pull=always --net=host "
+        f"--name inference "
+        f'-e ENV="{os.environ.get("ENV", "prod")}" '
+        f'-e MATRICE_SECRET_ACCESS_KEY="{self.matrice_secret_access_key}" '
+        f'-e MATRICE_ACCESS_KEY_ID="{self.matrice_access_key_id}" '
+        f"{image} "
+        f"./app "
+        f"{self.action_record_id} "
+
+    )
+    logging.info("Starting inference WebSocket server (Port: 8102): %s", worker_cmd)
 
     # Docker Command run
     self.start(worker_cmd, "inference_ws_server")
@@ -1132,7 +1168,8 @@ def inference_ws_server_execute(self: ActionInstance):
 @log_errors(raise_exception=False)
 def fe_fs_streaming_execute(self: ActionInstance):
     """
-    Creates and setup the frontend for fs streaming
+    Creates and setup the frontend for fs streaming.
+    Frontend streaming runs on port 3000 (localhost only with --net=host).
     """
     action_details = self.get_action_details()
 
@@ -1142,17 +1179,16 @@ def fe_fs_streaming_execute(self: ActionInstance):
 
     self.setup_action_requirements(action_details)
 
-    # Add worker container run command
+    # Frontend streaming with --net=host (Port: 3000)
     worker_cmd = (
-        f"docker run -d --pull=always "
+        f"docker run -d --pull=always --net=host "
         f"--name fe_streaming "
-        f"-p 3000:3000 "
         f'-e ENV="{os.environ.get("ENV", "prod")}" '
         f'-e MATRICE_SECRET_ACCESS_KEY="{self.matrice_secret_access_key}" '
         f'-e MATRICE_ACCESS_KEY_ID="{self.matrice_access_key_id}" '
         f"{image}"
     )
-    print("fe_fs_stremaing docker run command:", worker_cmd)
+    logging.info("Starting frontend streaming (Port: 3000): %s", worker_cmd)
 
     # Docker Command run
     self.start(worker_cmd, "fe_fs_streaming")
@@ -1204,8 +1240,8 @@ def synthetic_data_setup_execute(self: ActionInstance):
 def redis_setup_execute(self: ActionInstance):
     """
     Creates and starts a Redis container using Docker.
+    Redis runs on port 6379 (localhost only with --net=host).
     """
-    external_port = self.scaling.get_open_port()
     work_fs = get_max_file_system()
 
     action_details = self.get_action_details()
@@ -1217,18 +1253,6 @@ def redis_setup_execute(self: ActionInstance):
         "password", f"redis_pass_{int(time.time())}"
     )
 
-    container_info, error, message = self.create_redis_container(
-        action_details["actionDetails"].get("redis_image", "redis:latest"),
-        redis_password=redis_password,
-    )
-    if error:
-        logging.error(
-            "Error creating Redis container: %s",
-            message,
-        )
-        return
-    logging.info("Redis container created successfully: %s", container_info)
-
     # Initialize redis container
     self.setup_action_requirements(
         action_details,
@@ -1237,17 +1261,39 @@ def redis_setup_execute(self: ActionInstance):
         action_id=action_id,
     )
 
+    redis_image = action_details["actionDetails"].get("redis_image", "redis:latest")
+    
+    # Redis container with --net=host (Port: 6379)
+    redis_cmd = (
+        f"docker run -d --net=host "
+        f"--name redis_container_{int(time.time())} "
+        f"--restart unless-stopped "
+        f"{redis_image} "
+        f"redis-server --bind 0.0.0.0 --appendonly yes --requirepass {redis_password}"
+    )
+    
+    logging.info("Starting Redis container (Port: 6379): %s", redis_cmd)
+    
+    # Start Redis container first
+    redis_process = subprocess.Popen(
+        redis_cmd,
+        shell=True,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+    logging.info("Redis container started successfully on localhost:6379")
+    
+    # Wait for Redis to be ready
+    time.sleep(5)
+
     env_vars = {
-        "REDIS_URL": f"{container_info['container_name']}:{container_info['external_port']}",
-        "REDIS_PASSWORD": container_info["password"],
+        "REDIS_URL": f"localhost:6379",
+        "REDIS_PASSWORD": redis_password,
     }
 
-    network_config = f" --network {container_info['network_name']} -p 8082:8082"
-
-    # Make the docker file here
+    # bg-redis management container with --net=host (Port: 8082)
     cmd = (
-        f"docker run "
-        f"{network_config} "
+        f"docker run --net=host "
         f"-e REDIS_URL={shlex.quote(env_vars['REDIS_URL'])} "
         f"-e REDIS_PASSWORD={shlex.quote(env_vars['REDIS_PASSWORD'])} "
         f"-e MATRICE_ACCESS_KEY_ID={shlex.quote(self.matrice_access_key_id)} "
@@ -1259,7 +1305,7 @@ def redis_setup_execute(self: ActionInstance):
         f"{self.action_record_id} "
     )
 
-    logging.info("cmd is: %s", cmd)
+    logging.info("Starting bg-redis management (Port: 8082): %s", cmd)
 
     self.start(cmd, "redis_setup")
 

matrice_compute/instance_manager.py

@@ -153,7 +153,8 @@ class InstanceManager:
             key,
             value,
         ) in manual_instance_info.items():
-            os.environ[key] = value
+            if value is not None:
+                os.environ[key] = str(value)
         if not (os.environ.get("SERVICE_PROVIDER") and os.environ.get("INSTANCE_ID")):
             raise Exception(
                 "SERVICE_PROVIDER and INSTANCE_ID must be set as environment variables or passed as arguments"

matrice_compute/instance_utils.py

@@ -366,7 +366,13 @@ def get_max_file_system() -> str:
             max_available_filesystem,
             max_disk["available"],
         )
-    if max_available_filesystem in ["/", ""]:
+    # Check if filesystem is writable, or if it's root/empty
+    if max_available_filesystem in ["/", ""] or not os.access(max_available_filesystem, os.W_OK):
+        if max_available_filesystem not in ["/", ""]:
+            logging.warning(
+                "Filesystem %s is not writable, falling back to home directory",
+                max_available_filesystem,
+            )
         home_dir = os.path.expanduser("~")
         if not os.environ.get("WORKSPACE_DIR"):
             logging.error("WORKSPACE_DIR environment variable not set")

matrice_compute/resources_tracker.py

@@ -247,10 +247,14 @@ class ActionsResourcesTracker:
         self.max_actions_usage = {}
         self.resources_tracker = ResourcesTracker()
         self.client = docker.from_env()
+        self.logged_stopped_containers = []
 
     @log_errors(raise_exception=False, log_error=True)
     def update_actions_resources(self) -> None:
-        """Process both running and exited containers"""
+        """Process both running and exited containers.
+        
+        Note: Does not remove containers to keep logs. Only tracks resource usage.
+        """
         exited_containers = self.client.containers.list(
             filters={"status": "exited"},
             all=True,
@@ -259,8 +263,12 @@ class ActionsResourcesTracker:
         if exited_containers:
             for container in exited_containers:
                 try:
+                    if container.id in self.logged_stopped_containers:
+                        continue
                     self._update_container_action_status(container, "completed")
-                    container.remove()
+                    self.logged_stopped_containers.append(container.id)
+                    # COMMENTED OUT: Do not remove containers to keep logs
+                    # container.remove()
                 except Exception as err:
                     logging.error(
                         "Error processing exited container %s: %s",
@@ -310,7 +318,7 @@ class ActionsResourcesTracker:
         args_24 = [arg for arg in remove_quotation_marks(inspect_data["Args"]) if len(arg) == 24 and "pypi" not in arg]
         action_record_id = args_24[-1] if args_24 else None
         if not action_record_id:
-            logging.warning("No valid action_id found for the container. Container ID: %s, Args: %s", container.id, inspect_data["Args"])
+            logging.debug("No valid action_id found for the container. Container ID: %s, Args: %s", container.id, inspect_data["Args"])
         duration = calculate_time_difference(start_time, finish_time)
         (
             current_gpu_utilization,
@@ -320,6 +328,8 @@ class ActionsResourcesTracker:
         ) = self.get_current_action_usage(container, status)
         sub_containers = self.get_sub_containers_by_label("action_id", action_record_id)
         for sub_container in sub_containers:
+            if sub_container.id in self.logged_stopped_containers:
+                continue
             (
                 sub_container_gpu_utilization,
                 sub_container_gpu_memory,
@@ -330,10 +340,12 @@ class ActionsResourcesTracker:
             current_gpu_memory += sub_container_gpu_memory
             current_cpu_utilization += sub_container_cpu_utilization
             current_memory_utilization += sub_container_memory_utilization
+            # COMMENTED OUT: Do not stop/remove sub-containers to keep logs
             if status == "completed":
                 try:
                     sub_container.stop()
-                    sub_container.remove(force=True)
+                    self.logged_stopped_containers.append(sub_container.id)
+            #         sub_container.remove(force=True)
                 except Exception as err:
                     logging.error(
                         "Error removing sub-container %s: %s",

matrice_compute/scaling.py

@@ -3,12 +3,13 @@
 import os
 import logging
 from matrice_common.utils import log_errors
-from kafka import KafkaProducer, KafkaConsumer
+# from kafka import KafkaProducer, KafkaConsumer
 import uuid
 import json
 import time
 import base64
 
+# TODO: update /scaling to /compute
 
 class Scaling:
 
@@ -37,32 +38,34 @@ class Scaling:
             "Initialized Scaling with instance_id: %s",
             instance_id,
         )
-        self.kafka_config = {
-            "bootstrap_servers": self.get_kafka_bootstrap_servers(),
-            "api_request_topic": "action_requests",
-            "api_response_topic": "action_responses",
-            "scaling_request_topic": "compute_requests",
-            "scaling_response_topic": "compute_responses"
-        }
-        self.kafka_producer = KafkaProducer(
-            bootstrap_servers=self.kafka_config["bootstrap_servers"],
-            value_serializer=lambda v: json.dumps(v).encode("utf-8"),)
+        # KAFKA TEMPORARILY DISABLED - Using REST API directly
+        # self.kafka_config = {
+        #     "bootstrap_servers": self.get_kafka_bootstrap_servers(),
+        #     "api_request_topic": "action_requests",
+        #     "api_response_topic": "action_responses",
+        #     "scaling_request_topic": "compute_requests",
+        #     "scaling_response_topic": "compute_responses"
+        # }
+        # self.kafka_producer = KafkaProducer(
+        #     bootstrap_servers=self.kafka_config["bootstrap_servers"],
+        #     value_serializer=lambda v: json.dumps(v).encode("utf-8"),)
    
 
 
-    @log_errors(default_return=(None, "Error creating Kafka producer", "Kafka producer creation failed"), log_error=True)
-    def get_kafka_bootstrap_servers(self):
-        """Get Kafka bootstrap servers from API and decode base64 fields."""
-        path = "/v1/actions/get_kafka_info"
-        response = self.rpc.get(path=path)
-        if not response or not response.get("success"):
-            raise ValueError(f"Failed to fetch Kafka config: {response.get('message', 'No response')}")
-        encoded_ip = response["data"]["ip"]
-        encoded_port = response["data"]["port"]
-        ip = base64.b64decode(encoded_ip).decode("utf-8")
-        port = base64.b64decode(encoded_port).decode("utf-8")
-        bootstrap_servers = f"{ip}:{port}"
-        return bootstrap_servers
+    # KAFKA TEMPORARILY DISABLED - Using REST API directly
+    # @log_errors(default_return=(None, "Error creating Kafka producer", "Kafka producer creation failed"), log_error=True)
+    # def get_kafka_bootstrap_servers(self):
+    #     """Get Kafka bootstrap servers from API and decode base64 fields."""
+    #     path = "/v1/actions/get_kafka_info"
+    #     response = self.rpc.get(path=path)
+    #     if not response or not response.get("success"):
+    #         raise ValueError(f"Failed to fetch Kafka config: {response.get('message', 'No response')}")
+    #     encoded_ip = response["data"]["ip"]
+    #     encoded_port = response["data"]["port"]
+    #     ip = base64.b64decode(encoded_ip).decode("utf-8")
+    #     port = base64.b64decode(encoded_port).decode("utf-8")
+    #     bootstrap_servers = f"{ip}:{port}"
+    #     return bootstrap_servers
 
     @log_errors(default_return=(None, "Error processing response", "Response processing failed"), log_error=True)
     def handle_response(self, resp, success_message, error_message):
@@ -183,6 +186,8 @@ class Scaling:
         Returns:
             Tuple of (data, error, message) from API response
         """
+        if not action_record_id:
+            return None, "Action record id is required", "Action record id is required"
         logging.info(
             "Updating action status for action %s",
             action_record_id,
@@ -285,34 +290,44 @@ class Scaling:
 
     @log_errors(log_error=True)
     def get_action_details(self, action_status_id):
-        """Get details for a specific action using Kafka, fallback to REST, then cache."""
+        """Get details for a specific action using REST API.
+        
+        Args:
+            action_status_id: ID of the action status to fetch
+            
+        Returns:
+            Tuple of (data, error, message) from API response
+        """
         logging.info("Getting action details for action %s", action_status_id)
-        api = "get_action_details"
-        payload = {"actionRecordId": action_status_id}
-        # Try Kafka first
-        data, error, message, kafka_response_received = self._send_kafka_request(
-            api=api,
-            payload=payload,
-            request_topic=self.kafka_config["api_request_topic"],
-            response_topic=self.kafka_config["api_response_topic"],
-            timeout=60
-        )
-        if kafka_response_received:
-            return data, error, message
-
-        # Only if Kafka transport failed or timed out, try REST
+        # KAFKA TEMPORARILY DISABLED - Using REST API directly
+        # api = "get_action_details"
+        # payload = {"actionRecordId": action_status_id}
+        # data, error, message, kafka_response_received = self._send_kafka_request(
+        #     api=api,
+        #     payload=payload,
+        #     request_topic=self.kafka_config["api_request_topic"],
+        #     response_topic=self.kafka_config["api_response_topic"],
+        #     timeout=60
+        # )
+        # # Check if Kafka response was received and if it's an error, log and fallback to REST API
+        # if kafka_response_received:
+        #     if error:
+        #         logging.warning("Kafka returned error for get_action_details: %s. Falling back to REST API.", error)
+        #     else:
+        #         return data, error, message
+
+        # Using REST API directly
         try:
             path = f"/v1/actions/action/{action_status_id}/details"
             resp = self.rpc.get(path=path)
             return self.handle_response(
                 resp,
-                "Task details fetched successfully (REST fallback)",
-                "Could not fetch the task details (REST fallback)",
+                "Task details fetched successfully",
+                "Could not fetch the task details",
             )
         except Exception as e:
-            logging.error("REST fallback failed: %s", e)
-            self._cache_failed_request(api, payload)
-            return None, f"Failed via Kafka and REST: {e}", "Cached for retry"
+            logging.error("REST API failed (get_action_details): %s", e)
+            return None, f"Failed via REST: {e}", "REST API failed"
             
 
     @log_errors(log_error=True)
@@ -327,11 +342,26 @@ class Scaling:
         service="",
         job_params=None,
     ):
-        """Update an action using Kafka, fallback to REST, then cache."""
+        """Update an action using REST API.
+        
+        Args:
+            id: Action ID
+            step_code: Step code
+            action_type: Type of action
+            status: Status of the action
+            sub_action: Sub-action details
+            status_description: Description of the status
+            service: Service name
+            job_params: Job parameters dictionary
+            
+        Returns:
+            Tuple of (data, error, message) from API response
+        """
         if job_params is None:
             job_params = {}
         logging.info("Updating action %s", id)
-        api = "update_action"
+        # KAFKA TEMPORARILY DISABLED - Using REST API directly
+        # api = "update_action"
         payload = {
             "_id": id,
             "stepCode": step_code,
@@ -342,63 +372,85 @@ class Scaling:
             "serviceName": service,
             "jobParams": job_params,
         }
-        data, error, message, kafka_response_received = self._send_kafka_request(
-            api=api,
-            payload=payload,
-            request_topic=self.kafka_config["api_request_topic"],
-            response_topic=self.kafka_config["api_response_topic"],
-            timeout=60
-        )
-        if kafka_response_received:
-            return data, error, message
+        # data, error, message, kafka_response_received = self._send_kafka_request(
+        #     api=api,
+        #     payload=payload,
+        #     request_topic=self.kafka_config["api_request_topic"],
+        #     response_topic=self.kafka_config["api_response_topic"],
+        #     timeout=60
+        # )
+        # # Check if Kafka response was received and if it's an error, log and fallback to REST API
+        # if kafka_response_received:
+        #     if error:
+        #         logging.warning("Kafka returned error for update_action: %s. Falling back to REST API.", error)
+        #     else:
+        #         return data, error, message
+
+        # Using REST API directly
         try:
             path = "/v1/actions"
             resp = self.rpc.put(path=path, payload=payload)
             return self.handle_response(
                 resp,
-                "Error logged successfully (REST fallback)",
-                "Could not log the errors (REST fallback)",
+                "Error logged successfully",
+                "Could not log the errors",
             )
         except Exception as e:
-            logging.error("REST fallback failed (update_action): %s", e)
-            self._cache_failed_request(api, payload)
-            return None, f"Failed via Kafka and REST: {e}", "Cached for retry"
+            logging.error("REST API failed (update_action): %s", e)
+            return None, f"Failed via REST: {e}", "REST API failed"
            
 
     @log_errors(log_error=True)
     def assign_jobs(self, is_gpu):
-        """Assign jobs to the instance using Kafka, fallback to REST, then cache."""
-        logging.info("Assigning jobs for instance %s (GPU: %s)", self.instance_id, is_gpu)
-        api = "assign_jobs"
-        payload = {
-            "instanceID": self.instance_id,
-            "isGPUInstance": is_gpu,
-        }
-
-        data, error, message, kafka_response_received = self._send_kafka_request(
-            api=api,
-            payload=payload,
-            request_topic=self.kafka_config["api_request_topic"],
-            response_topic=self.kafka_config["api_response_topic"],
-            timeout=60
-        )
-
-        if kafka_response_received:
-            return data, error, message
-       
-        # Fallback to REST
+        """Assign jobs to the instance using REST API.
+        
+        Args:
+            is_gpu: Boolean or any value indicating if this is a GPU instance.
+                    Will be converted to proper boolean.
+        
+        Returns:
+            Tuple of (data, error, message) from API response
+        """
+        # Convert is_gpu to proper boolean
+        is_gpu_bool = bool(is_gpu)
+        logging.info("Assigning jobs for instance %s (GPU: %s)", self.instance_id, is_gpu_bool)
+        
+        # KAFKA TEMPORARILY DISABLED - Using REST API directly
+        # api = "assign_jobs"
+        # payload = {
+        #     "instanceID": self.instance_id,
+        #     "isGPUInstance": is_gpu_bool,
+        # }
+
+        # data, error, message, kafka_response_received = self._send_kafka_request(
+        #     api=api,
+        #     payload=payload,
+        #     request_topic=self.kafka_config["api_request_topic"],
+        #     response_topic=self.kafka_config["api_response_topic"],
+        #     timeout=60
+        # )
+
+        # # Check if Kafka response was received and if it's an error, log and fallback to REST API
+        # if kafka_response_received:
+        #     if error:
+        #         logging.warning("Kafka returned error for assign_jobs: %s. Falling back to REST API.", error)
+        #     else:
+        #         return data, error, message
+
+        # Using REST API directly
         try:
-            path = f"/v1/actions/assign_jobs/{str(is_gpu)}/{self.instance_id}"
+            # Convert boolean to lowercase string for API endpoint
+            is_gpu_str = str(is_gpu_bool).lower()
+            path = f"/v1/actions/assign_jobs/{is_gpu_str}/{self.instance_id}"
             resp = self.rpc.get(path=path)
             return self.handle_response(
                 resp,
-                "Pinged successfully (REST fallback)",
-                "Could not ping the scaling jobs (REST fallback)",
+                "Pinged successfully",
+                "Could not ping the scaling jobs",
             )
         except Exception as e:
-            logging.error("REST fallback failed (assign_jobs): %s", e)
-            self._cache_failed_request(api, payload)
-            return None, f"Failed via Kafka and REST: {e}", "Cached for retry"
+            logging.error("REST API failed (assign_jobs): %s", e)
+            return None, f"Failed via REST: {e}", "REST API failed"
           
 
     @log_errors(log_error=True)
@@ -409,7 +461,17 @@ class Scaling:
         availableMemory=0,
         availableGPUMemory=0,
     ):
-        """Update available resources for the instance using Kafka, fallback to REST, then cache."""
+        """Update available resources for the instance using REST API.
+        
+        Args:
+            availableCPU: Available CPU resources
+            availableGPU: Available GPU resources
+            availableMemory: Available memory
+            availableGPUMemory: Available GPU memory
+            
+        Returns:
+            Tuple of (data, error, message) from API response
+        """
         logging.info("Updating available resources for instance %s", self.instance_id)
         payload = {
             "instance_id": self.instance_id,
@@ -418,63 +480,84 @@ class Scaling:
             "availableGPUMemory": availableGPUMemory,
             "availableGPU": availableGPU,
         }
-        api = "update_available_resources"
-        correlation_id = str(uuid.uuid4())
-       
-
-        data, error, message, kafka_response_received = self._send_kafka_request(
-            api=api,
-            payload=payload,
-            request_topic=self.kafka_config["scaling_request_topic"],
-            response_topic=self.kafka_config["scaling_response_topic"],
-            timeout=60
-        )
-
-        if kafka_response_received:
-            return data, error, message
+        # KAFKA TEMPORARILY DISABLED - Using REST API directly
+        # api = "update_available_resources"
+        # correlation_id = str(uuid.uuid4())
+
+        # data, error, message, kafka_response_received = self._send_kafka_request(
+        #     api=api,
+        #     payload=payload,
+        #     request_topic=self.kafka_config["scaling_request_topic"],
+        #     response_topic=self.kafka_config["scaling_response_topic"],
+        #     timeout=60
+        # )
+
+        # # Check if Kafka response was received
+        # # Response format: {'correlationId': 'id', 'status': 'success'/'error', 'data': ..., 'error': 'error message'}
+        # if kafka_response_received:
+        #     if error:
+        #         logging.warning("Kafka returned error for update_available_resources: %s. Falling back to REST API.", error)
+        #     else:
+        #         return data, error, message
+
+        # Using REST API directly
         try:
-            path = f"/v1/scaling/update_available_resources/{self.instance_id}"
+            path = f"/v1/compute/update_available_resources/{self.instance_id}"
             resp = self.rpc.put(path=path, payload=payload)
             return self.handle_response(
                 resp,
-                "Resources updated successfully (REST fallback)",
-                "Could not update the resources (REST fallback)",
+                "Resources updated successfully",
+                "Could not update the resources",
             )
         except Exception as e:
-            logging.error("REST fallback failed (update_available_resources): %s", e)
-            self._cache_failed_request(api, payload)
-            return None, f"Failed to update available resources via Kafka and REST: {e}", "Cached for retry"
+            logging.error("REST API failed (update_available_resources): %s", e)
+            return None, f"Failed to update available resources via REST: {e}", "REST API failed"
 
     @log_errors(log_error=True)
     def update_action_docker_logs(self, action_record_id, log_content):
-        """Update docker logs for an action using Kafka, fallback to REST, then cache."""
+        """Update docker logs for an action using REST API.
+        
+        Args:
+            action_record_id: ID of the action record
+            log_content: Content of the logs to update
+            
+        Returns:
+            Tuple of (data, error, message) from API response
+        """
         logging.info("Updating docker logs for action %s", action_record_id)
-        api = "update_action_docker_logs"
+        # KAFKA TEMPORARILY DISABLED - Using REST API directly
+        # api = "update_action_docker_logs"
         payload = {
             "actionRecordId": action_record_id,
             "logContent": log_content,
-           
         }
-        data, error, message, kafka_response_received = self._send_kafka_request(
-                api=api,
-                payload=payload,
-                request_topic=self.kafka_config["api_request_topic"],
-                response_topic=self.kafka_config["api_response_topic"],
-                timeout=60
-            )
-
-        if kafka_response_received:
-            return data, error, message
+        # data, error, message, kafka_response_received = self._send_kafka_request(
+        #     api=api,
+        #     payload=payload,
+        #     request_topic=self.kafka_config["api_request_topic"],
+        #     response_topic=self.kafka_config["api_response_topic"],
+        #     timeout=60
+        # )
+
+        # # Check if Kafka response was received and if it's an error, log and fallback to REST API
+        # if kafka_response_received:
+        #     if error:
+        #         logging.warning("Kafka returned error for update_action_docker_logs: %s. Falling back to REST API.", error)
+        #     else:
+        #         return data, error, message
+
+        # Using REST API directly
         try:
             path = "/v1/actions/update_action_docker_logs"
             resp = self.rpc.put(path=path, payload=payload)
             return self.handle_response(
                 resp,
-                "Docker logs updated successfully (REST fallback)",
-                "Could not update the docker logs (REST fallback)",
+                "Docker logs updated successfully",
+                "Could not update the docker logs",
             )
         except Exception as e:
-            logging.error("REST fallback failed (update_action_docker_logs): %s", e)
+            logging.error("REST API failed (update_action_docker_logs): %s", e)
+            return None, f"Failed via REST: {e}", "REST API failed"
           
 
     @log_errors(log_error=True)
@@ -533,7 +616,8 @@ class Scaling:
             if port in self.used_ports:
                 continue
             self.used_ports.add(port)
-            os.environ["USED_PORTS"] = ",".join(str(p) for p in self.used_ports)
+            ports_value = ",".join(str(p) for p in self.used_ports)
+            os.environ["USED_PORTS"] = str(ports_value)
             logging.info("Found available port: %s", port)
             return port
         logging.error(
@@ -563,7 +647,7 @@ class Scaling:
         Returns:
             Tuple of (data, error, message) from API response
         """
-        path = f"/v1/scaling/get_models_secret_keys?secret_name={secret_name}"
+        path = f"/v1/compute/get_models_secret_keys?secret_name={secret_name}"
         resp = self.rpc.get(path=path)
         return self.handle_response(
             resp,
@@ -790,98 +874,99 @@ class Scaling:
             "Could not fetch internal keys",
         )
 
-    @log_errors(log_error=True)
-    def handle_kafka_response(self, msg, success_message, error_message):
-        """
-        Helper to process Kafka response messages in a consistent way.
-        """
-        if msg.get("status") == "success":
-            data = msg.get("data")
-            error = None
-            message = success_message
-            logging.info(message)
-        else:
-            data = msg.get("data")
-            error = msg.get("error", "Unknown error")
-            message = error_message
-            logging.error("%s: %s", message, error)
-        return data, error, message
-
-    def _send_kafka_request(self, api, payload, request_topic, response_topic, timeout=60):
-        """
-        Helper to send a request to Kafka and wait for a response.
-        Returns (data, error, message, kafka_response_received) where kafka_response_received is True if a response was received (even if error), False if transport error/timeout.
-        """
-        correlation_id = str(uuid.uuid4())
-        request_message = {
-            "correlationId": correlation_id,
-            "api": api,
-            "payload": payload,
-        }
-
-        consumer = KafkaConsumer(
-            response_topic,
-            bootstrap_servers=self.kafka_config["bootstrap_servers"],
-            group_id=None,
-            value_deserializer=lambda m: json.loads(m.decode("utf-8")),
-            auto_offset_reset='latest',
-            enable_auto_commit=True,
-        )
-
-        try:
-            if hasattr(self.session.rpc, 'AUTH_TOKEN'):
-                self.session.rpc.AUTH_TOKEN.set_bearer_token()
-                auth_token = self.session.rpc.AUTH_TOKEN.bearer_token
-                auth_token = auth_token.replace("Bearer ", "")
-                headers = [("Authorization", bytes(f"{auth_token}", "utf-8"))]
-            else:
-                headers = None
-            self.kafka_producer.send(request_topic, request_message, headers=headers)
-            # self.kafka_producer.flush()
-            logging.info("Sent %s request to Kafka topic %s", api, request_topic)
-        except Exception as e:
-            logging.error("Kafka producer error: %s", e)
-            return None, f"Kafka producer error: {e}", "Kafka send failed", False
-        try:
-            start = time.time()
-            while time.time() - start < timeout:
-                # Poll for messages with a short timeout to avoid blocking forever
-                message_batch = consumer.poll(timeout_ms=1000)
-                if message_batch:
-                    for topic_partition, messages in message_batch.items():
-                        for message in messages:
-                            print("trying to fetch message")
-                            msg = message.value
-                            if msg.get("correlationId") == correlation_id:
-                                consumer.close()
-                                # Always treat a received response as final, even if error
-                                return self.handle_kafka_response(
-                                    msg,
-                                    f"Fetched via Kafka for {api}",
-                                    f"Kafka error response for {api}"
-                                ) + (True,)
-                else:
-                    print(f"No messages received, waiting... ({time.time() - start:.1f}s/{timeout}s)")
-            
-            consumer.close()
-            logging.warning("Kafka response timeout for %s after %d seconds", api, timeout)
-            return None, "Kafka response timeout", "Kafka response timeout", False
-        except Exception as e:
-            logging.error("Kafka consumer error: %s", e)
-            return None, f"Kafka consumer error: {e}", "Kafka consumer error", False
-
-    def _cache_failed_request(self, api, payload):
-        """Cache the failed request for retry. Here, we use a simple file cache as a placeholder."""
-        try:
-            cache_file = os.path.join(os.path.dirname(__file__), 'request_cache.json')
-            if os.path.exists(cache_file):
-                with open(cache_file, 'r') as f:
-                    cache = json.load(f)
-            else:
-                cache = []
-            cache.append({"api": api, "payload": payload, "ts": time.time()})
-            with open(cache_file, 'w') as f:
-                json.dump(cache, f)
-            logging.info("Cached failed request for api %s", api)
-        except Exception as e:
-            logging.error("Failed to cache request: %s", e)
+    # KAFKA TEMPORARILY DISABLED - Using REST API directly
+    # @log_errors(log_error=True)
+    # def handle_kafka_response(self, msg, success_message, error_message):
+    #     """
+    #     Helper to process Kafka response messages in a consistent way.
+    #     """
+    #     if msg.get("status") == "success":
+    #         data = msg.get("data")
+    #         error = None
+    #         message = success_message
+    #         logging.info(message)
+    #     else:
+    #         data = msg.get("data")
+    #         error = msg.get("error", "Unknown error")
+    #         message = error_message
+    #         logging.error("%s: %s", message, error)
+    #     return data, error, message
+
+    # def _send_kafka_request(self, api, payload, request_topic, response_topic, timeout=60):
+    #     """
+    #     Helper to send a request to Kafka and wait for a response.
+    #     Returns (data, error, message, kafka_response_received) where kafka_response_received is True if a response was received (even if error), False if transport error/timeout.
+    #     """
+    #     correlation_id = str(uuid.uuid4())
+    #     request_message = {
+    #         "correlationId": correlation_id,
+    #         "api": api,
+    #         "payload": payload,
+    #     }
+
+    #     consumer = KafkaConsumer(
+    #         response_topic,
+    #         bootstrap_servers=self.kafka_config["bootstrap_servers"],
+    #         group_id=None,
+    #         value_deserializer=lambda m: json.loads(m.decode("utf-8")),
+    #         auto_offset_reset='latest',
+    #         enable_auto_commit=True,
+    #     )
+
+    #     try:
+    #         if hasattr(self.session.rpc, 'AUTH_TOKEN'):
+    #             self.session.rpc.AUTH_TOKEN.set_bearer_token()
+    #             auth_token = self.session.rpc.AUTH_TOKEN.bearer_token
+    #             auth_token = auth_token.replace("Bearer ", "")
+    #             headers = [("Authorization", bytes(f"{auth_token}", "utf-8"))]
+    #         else:
+    #             headers = None
+    #         self.kafka_producer.send(request_topic, request_message, headers=headers)
+    #         # self.kafka_producer.flush()
+    #         logging.info("Sent %s request to Kafka topic %s", api, request_topic)
+    #     except Exception as e:
+    #         logging.error("Kafka producer error: %s", e)
+    #         return None, f"Kafka producer error: {e}", "Kafka send failed", False
+    #     try:
+    #         start = time.time()
+    #         while time.time() - start < timeout:
+    #             # Poll for messages with a short timeout to avoid blocking forever
+    #             message_batch = consumer.poll(timeout_ms=1000)
+    #             if message_batch:
+    #                 for topic_partition, messages in message_batch.items():
+    #                     for message in messages:
+    #                         print("trying to fetch message")
+    #                         msg = message.value
+    #                         if msg.get("correlationId") == correlation_id:
+    #                             consumer.close()
+    #                             # Always treat a received response as final, even if error
+    #                             return self.handle_kafka_response(
+    #                                 msg,
+    #                                 f"Fetched via Kafka for {api}",
+    #                                 f"Kafka error response for {api}"
+    #                             ) + (True,)
+    #             else:
+    #                 print(f"No messages received, waiting... ({time.time() - start:.1f}s/{timeout}s)")
+    #
+    #         consumer.close()
+    #         logging.warning("Kafka response timeout for %s after %d seconds", api, timeout)
+    #         return None, "Kafka response timeout", "Kafka response timeout", False
+    #     except Exception as e:
+    #         logging.error("Kafka consumer error: %s", e)
+    #         return None, f"Kafka consumer error: {e}", "Kafka consumer error", False
+
+    # def _cache_failed_request(self, api, payload):
+    #     """Cache the failed request for retry. Here, we use a simple file cache as a placeholder."""
+    #     try:
+    #         cache_file = os.path.join(os.path.dirname(__file__), 'request_cache.json')
+    #         if os.path.exists(cache_file):
+    #             with open(cache_file, 'r') as f:
+    #                 cache = json.load(f)
+    #         else:
+    #             cache = []
+    #         cache.append({"api": api, "payload": payload, "ts": time.time()})
+    #         with open(cache_file, 'w') as f:
+    #             json.dump(cache, f)
+    #         logging.info("Cached failed request for api %s", api)
+    #     except Exception as e:
+    #         logging.error("Failed to cache request: %s", e)

matrice_compute/task_utils.py

@@ -29,8 +29,8 @@ def setup_workspace_and_run_task(
     workspace_dir = f"{work_fs}/{action_id}"
     codebase_zip_path = f"{workspace_dir}/file.zip"
     requirements_txt_path = f"{workspace_dir}/requirements.txt"
-    if os.path.exists(workspace_dir):
-        return
+    # if os.path.exists(workspace_dir): # don't skip if workspace already exists, override it
+    #     return
     os.makedirs(workspace_dir, exist_ok=True)
     
     # Download codebase ZIP file

{matrice_compute-0.1.12.dist-info → matrice_compute-0.1.14.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: matrice_compute
-Version: 0.1.12
+Version: 0.1.14
 Summary: Common server utilities for Matrice.ai services
 Author-email: "Matrice.ai" <dipendra@matrice.ai>
 License-Expression: MIT

matrice_compute-0.1.14.dist-info/RECORD

@@ -0,0 +1,17 @@
+matrice_compute/__init__.py,sha256=ZzQcFsT005VCgq9VZUh565f4upOooEb_FwZ6RgweNZs,597
+matrice_compute/action_instance.py,sha256=aYNpRySPatxFltn_ekVmCd5h69I992_YerUTZwGWyHA,59763
+matrice_compute/actions_manager.py,sha256=5U-xM6tl_Z6x96bi-c7AJM9ru80LqTN8f5Oce8dAu_A,7780
+matrice_compute/actions_scaledown_manager.py,sha256=pJ0nduNwHWZ10GnqJNx0Ok7cVWabQ_M8E2Vb9pH3A_k,2002
+matrice_compute/instance_manager.py,sha256=8USyX09ZxLvnVNIrjRogbyUeMCfgWnasuRqYkkVF4tQ,10146
+matrice_compute/instance_utils.py,sha256=7jnWurSpq8PQxPGlSTc0qmpNdD5jIL8pjYKdjhVhS60,22310
+matrice_compute/prechecks.py,sha256=W9YmNF3RcLhOf4U8WBlExvFqDw1aGWSNTlJtA73lbDQ,17196
+matrice_compute/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+matrice_compute/resources_tracker.py,sha256=2hLKVxYihROtQ6fO4V_BplTgvkN8qH2H9_qxpOIpZkc,18521
+matrice_compute/scaling.py,sha256=3F8SWvy9wWczpJ6dbY5RrXWw5ByZlIzAPJklir3KIFI,35359
+matrice_compute/shutdown_manager.py,sha256=0MYV_AqygqR9NEntYf7atUC-PbWXyNkm1f-8c2aizgA,13234
+matrice_compute/task_utils.py,sha256=3qIutiQdYPyGRxH9ZwLbqdg8sZcnp6jp08pszWCRFl0,2820
+matrice_compute-0.1.14.dist-info/licenses/LICENSE.txt,sha256=_uQUZpgO0mRYL5-fPoEvLSbNnLPv6OmbeEDCHXhK6Qc,1066
+matrice_compute-0.1.14.dist-info/METADATA,sha256=u8ZIOoIX3uMEA4Lgaiuh73xsoPSdcHTZXAJuIBpn6KE,1038
+matrice_compute-0.1.14.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+matrice_compute-0.1.14.dist-info/top_level.txt,sha256=63Plr3L1GzBUWZO5JZaFkiv8IcB10xUPU-9w3i6ptvE,16
+matrice_compute-0.1.14.dist-info/RECORD,,

matrice_compute-0.1.12.dist-info/RECORD

@@ -1,17 +0,0 @@
-matrice_compute/__init__.py,sha256=HG5yzsY2dcQ0sGKwxMj-Sv2zDhbSC00slAdkcfS9nng,304
-matrice_compute/action_instance.py,sha256=6tyZehK7SfIu6NjXp4wFeYMY0BINShmXtoCXyimDKN0,58002
-matrice_compute/actions_manager.py,sha256=5U-xM6tl_Z6x96bi-c7AJM9ru80LqTN8f5Oce8dAu_A,7780
-matrice_compute/actions_scaledown_manager.py,sha256=pJ0nduNwHWZ10GnqJNx0Ok7cVWabQ_M8E2Vb9pH3A_k,2002
-matrice_compute/instance_manager.py,sha256=LhPOqrzmrs-QdorqgDOuBDHjpUkLPgCZovBdCBiVmVw,10103
-matrice_compute/instance_utils.py,sha256=tIFVUi8HJPy4GY-jtfVx2zIgmXNta7s3jCIRzBga1hI,21977
-matrice_compute/prechecks.py,sha256=W9YmNF3RcLhOf4U8WBlExvFqDw1aGWSNTlJtA73lbDQ,17196
-matrice_compute/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-matrice_compute/resources_tracker.py,sha256=My26LPglDHcQcTkxxiXwpfdqkpEAt3clrqJ-k1fAl1M,17878
-matrice_compute/scaling.py,sha256=8HfbKMsR7EI0rrLfKl_gz6FMO2Q4sLXELxGc3DcLwz8,31743
-matrice_compute/shutdown_manager.py,sha256=0MYV_AqygqR9NEntYf7atUC-PbWXyNkm1f-8c2aizgA,13234
-matrice_compute/task_utils.py,sha256=ML9uTrYQiWgEMJitYxoGlVOa9KUXNKV_WqnousOTK6k,2762
-matrice_compute-0.1.12.dist-info/licenses/LICENSE.txt,sha256=_uQUZpgO0mRYL5-fPoEvLSbNnLPv6OmbeEDCHXhK6Qc,1066
-matrice_compute-0.1.12.dist-info/METADATA,sha256=__gJ0pkG07q5s8kOmdk8ItFWV3pfcHa-EUKDEe48xrY,1038
-matrice_compute-0.1.12.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-matrice_compute-0.1.12.dist-info/top_level.txt,sha256=63Plr3L1GzBUWZO5JZaFkiv8IcB10xUPU-9w3i6ptvE,16
-matrice_compute-0.1.12.dist-info/RECORD,,