matrice-compute 0.1.25__py3-none-any.whl → 0.1.26__py3-none-any.whl

matrice_compute/actions_manager.py

@@ -27,6 +27,7 @@ class ActionsManager:
             scaling (Scaling): Scaling service instance
         """
         self.current_actions: dict[str, ActionInstance] = {}
+        self.stopped_actions: dict[str, ActionInstance] = {}  # Track stopped actions separately
         self.scaling = scaling
         self.memory_threshold = 0.9
         self.poll_interval = 10
@@ -111,75 +112,110 @@ class ActionsManager:
     def process_actions(self) -> None:
         """Process fetched actions."""
         for action in self.fetch_actions():
+            action_id = action["_id"]
+            
+            # Skip if action is already running in current_actions
+            if action_id in self.current_actions:
+                logging.info("Action %s already in current_actions, skipping", action_id)
+                continue
+            
+            # If action exists in stopped_actions, remove it before starting fresh
+            if action_id in self.stopped_actions:
+                logging.info("Action %s found in stopped_actions, removing before restart", action_id)
+                del self.stopped_actions[action_id]
+            
+            # Process and add to current_actions
             action_instance = self.process_action(action)
             if action_instance:
-                self.current_actions[action["_id"]] = action_instance
+                # Ensure action is not in stopped_actions (defensive check)
+                if action_id in self.stopped_actions:
+                    del self.stopped_actions[action_id]
+                self.current_actions[action_id] = action_instance
 
     @log_errors(raise_exception=False)
-    def purge_unwanted(self) -> None:
-        """Purge completed or failed actions.
-        
-        This method checks all actions in the current_actions dictionary and removes any that:
-        1. Are explicitly reported as not running by the is_running() method
-        2. Have invalid or corrupted process objects
+    def update_actions_status(self) -> None:
+        """Update tracking of running vs stopped actions.
+
+        This method checks all actions and moves stopped ones to stopped_actions dict
+        without deleting them. This prevents interference with compute operations
+        handler while maintaining accurate status reporting.
         """
-        purged_count = 0
-        
-        # Check each action and purge if needed
+        moved_to_stopped = 0
+
+        # Check each action and update its status
         for action_id, instance in list(self.current_actions.items()):
-            should_purge = False
-            purge_reason = ""
-            
-            # Check if process is reported as not running
-            if not instance.is_running():
-                should_purge = True
-                purge_reason = "process reported as not running"
-            
+            is_running = False
+            status_reason = ""
+
+            # Check if process is running
+            if hasattr(instance, 'is_running'):
+                try:
+                    is_running = instance.is_running()
+                except Exception as e:
+                    logging.error("Error checking is_running for action %s: %s", action_id, str(e))
+                    is_running = False
+                    status_reason = f"error checking status: {str(e)}"
+
             # Check for process object validity
-            elif not hasattr(instance, 'process') or instance.process is None:
-                should_purge = True
-                purge_reason = "invalid process object"
-            
-            # Purge if any condition was met
-            if should_purge:
+            if not is_running and not status_reason:
+                if not hasattr(instance, 'process') or instance.process is None:
+                    status_reason = "no process object"
+                else:
+                    status_reason = "process not running"
+
+            # Move to stopped_actions if not running (but don't delete)
+            if not is_running:
                 logging.info(
-                    "Action %s is being purged: %s",
+                    "Action %s moved to stopped_actions: %s",
                     action_id,
-                    purge_reason
+                    status_reason
                 )
-                
-                # Remove from tracking dictionaries
-                del self.current_actions[action_id]
-                purged_count += 1
+                # Ensure action is removed from current_actions before adding to stopped_actions
+                if action_id in self.current_actions:
+                    del self.current_actions[action_id]
+                # Ensure action is not duplicated in stopped_actions
+                if action_id not in self.stopped_actions:
+                    self.stopped_actions[action_id] = instance
+                moved_to_stopped += 1
 
-                # Try to explicitly stop the action if possible
-                try:
-                    if hasattr(instance, 'stop'):
-                        instance.stop()
-                except Exception as e:
-                    logging.error(f"Error stopping action {action_id}: {str(e)}")
-        
-        if purged_count > 0:
+        # Log current state
+        running_ids = list(self.current_actions.keys())
+        stopped_ids = list(self.stopped_actions.keys())
+
+        if self.current_actions or self.stopped_actions:
             logging.info(
-                "Purged %d completed actions, %d actions remain in queue",
-                purged_count,
-                len(self.current_actions)
+                "Actions status: %d running %s, %d stopped %s",
+                len(self.current_actions),
+                running_ids if running_ids else "[]",
+                len(self.stopped_actions),
+                stopped_ids if stopped_ids else "[]"
             )
 
+    @log_errors(raise_exception=False)
+    def purge_unwanted(self) -> None:
+        """Purge completed or failed actions.
+
+        NOTE: This now calls update_actions_status() which moves stopped actions
+        to a separate dict instead of deleting them. This prevents interference
+        with compute operations handler while maintaining accurate status.
+        """
+        self.update_actions_status()
+
     @log_errors(default_return={}, raise_exception=False)
     def get_current_actions(self) -> dict:
-        """Get the current actions.
+        """Get the current running actions.
 
         This method:
-        1. Purges any completed actions using purge_unwanted()
-        2. Double-checks remaining actions to ensure they are truly running
+        1. Updates action status tracking via update_actions_status()
+        2. Returns only the running actions (current_actions dict)
         3. Provides detailed logging about current actions state
 
         Returns:
-            dict: Current active actions
+            dict: Current running actions only
         """
-        # Always purge unwanted actions first
-        self.purge_unwanted()
+        # Update status tracking (moves stopped to stopped_actions)
+        self.update_actions_status()
+
         if self.current_actions:
             action_ids = list(self.current_actions.keys())
             logging.info(
@@ -189,9 +225,213 @@ class ActionsManager:
             )
         else:
             logging.debug("No actions currently running")
-            return {}
+
         return self.current_actions
 
+    @log_errors(default_return={}, raise_exception=False)
+    def get_all_actions(self) -> dict:
+        """Get all tracked actions (both running and stopped).
+
+        Returns:
+            dict: All tracked actions with their status
+        """
+        all_actions = {}
+        for action_id, instance in self.current_actions.items():
+            all_actions[action_id] = {"instance": instance, "status": "running"}
+        for action_id, instance in self.stopped_actions.items():
+            all_actions[action_id] = {"instance": instance, "status": "stopped"}
+        return all_actions
+
+    @log_errors(default_return={}, raise_exception=False)
+    def get_stopped_actions(self) -> dict:
+        """Get stopped actions.
+
+        Returns:
+            dict: Stopped actions
+        """
+        return self.stopped_actions
+
+    @log_errors(default_return={}, raise_exception=False)
+    def stop_action(self, action_record_id: str) -> dict:
+        """Stop a specific action by its record ID.
+
+        Args:
+            action_record_id (str): The action record ID to stop
+
+        Returns:
+            dict: Result dictionary with status information
+        """
+        logging.info("Attempting to stop action: %s", action_record_id)
+
+        # Check if action exists in current (running) actions
+        action_instance = None
+        action_source = None
+
+        if action_record_id in self.current_actions:
+            action_instance = self.current_actions[action_record_id]
+            action_source = "current_actions"
+        elif action_record_id in self.stopped_actions:
+            # Action already in stopped_actions
+            logging.info("Action %s already in stopped_actions", action_record_id)
+            return {
+                "success": True,
+                "reason": "already_stopped",
+                "action_id": action_record_id
+            }
+        else:
+            logging.warning("Action %s not found in current or stopped actions", action_record_id)
+            return {
+                "success": False,
+                "reason": "action_not_found",
+                "action_id": action_record_id
+            }
+
+        # Check if action is actually running
+        if not action_instance.is_running():
+            logging.info("Action %s is not running, moving to stopped_actions", action_record_id)
+            # Move to stopped_actions instead of deleting
+            # Ensure action is removed from current_actions first
+            if action_record_id in self.current_actions:
+                del self.current_actions[action_record_id]
+            # Ensure action is not duplicated in stopped_actions
+            if action_record_id not in self.stopped_actions:
+                self.stopped_actions[action_record_id] = action_instance
+            return {
+                "success": True,
+                "reason": "already_stopped",
+                "action_id": action_record_id
+            }
+
+        # Stop the action
+        try:
+            logging.info("Stopping action %s", action_record_id)
+            action_instance.stop()
+
+            # Update action status to stopped
+            self.scaling.update_action_status(
+                service_provider=os.environ["SERVICE_PROVIDER"],
+                action_record_id=action_record_id,
+                status="stopped",
+                isRunning=False,
+                action_duration=0,
+            )
+
+            # Move to stopped_actions instead of deleting
+            # Ensure action is removed from current_actions first
+            if action_record_id in self.current_actions:
+                del self.current_actions[action_record_id]
+            # Ensure action is not duplicated in stopped_actions
+            if action_record_id not in self.stopped_actions:
+                self.stopped_actions[action_record_id] = action_instance
+
+            logging.info("Successfully stopped action: %s", action_record_id)
+            return {
+                "success": True,
+                "action_id": action_record_id,
+                "stopped_at": time.time()
+            }
+
+        except Exception as e:
+            logging.error("Error stopping action %s: %s", action_record_id, str(e))
+            return {
+                "success": False,
+                "reason": "stop_failed",
+                "error": str(e),
+                "action_id": action_record_id
+            }
+
+    @log_errors(default_return={}, raise_exception=False)
+    def restart_action(self, action_record_id: str) -> dict:
+        """Restart a specific action by its record ID.
+
+        This method stops the action if it's running, then fetches fresh action
+        details from the backend and starts it again.
+
+        Args:
+            action_record_id (str): The action record ID to restart
+
+        Returns:
+            dict: Result dictionary with status information
+        """
+        logging.info("Attempting to restart action: %s", action_record_id)
+
+        # Step 1: Stop the action if it exists in current_actions or stopped_actions
+        stop_result = {"success": True, "reason": "not_running"}
+        if action_record_id in self.current_actions:
+            logging.info("Stopping existing action %s before restart", action_record_id)
+            stop_result = self.stop_action(action_record_id)
+
+            if not stop_result.get("success"):
+                logging.error("Failed to stop action %s for restart", action_record_id)
+                return {
+                    "success": False,
+                    "reason": "stop_failed_before_restart",
+                    "stop_result": stop_result,
+                    "action_id": action_record_id
+                }
+
+            # Wait a moment for cleanup
+            time.sleep(2)
+        elif action_record_id in self.stopped_actions:
+            logging.info("Action %s found in stopped_actions, will restart", action_record_id)
+            stop_result = {"success": True, "reason": "was_stopped"}
+
+        # Step 2: Fetch fresh action details from backend
+        try:
+            logging.info("Fetching action details for restart: %s", action_record_id)
+
+            # Get action details via API
+            action_details, error, _ = self.scaling.get_action_details(action_record_id)
+
+            if error or not action_details:
+                logging.error("Failed to fetch action details for %s: %s",
+                            action_record_id, error)
+                return {
+                    "success": False,
+                    "reason": "fetch_failed",
+                    "error": error,
+                    "action_id": action_record_id
+                }
+
+            # Step 3: Process (start) the action
+            logging.info("Starting action %s after restart", action_record_id)
+            action_instance = self.process_action(action_details)
+
+            if action_instance:
+                # Ensure action is removed from stopped_actions if present
+                if action_record_id in self.stopped_actions:
+                    del self.stopped_actions[action_record_id]
+                # Ensure action is removed from current_actions if present (defensive check)
+                if action_record_id in self.current_actions:
+                    logging.warning("Action %s already in current_actions during restart, replacing", action_record_id)
+                    del self.current_actions[action_record_id]
+                # Add to current_actions
+                self.current_actions[action_record_id] = action_instance
+
+                logging.info("Successfully restarted action: %s", action_record_id)
+                return {
+                    "success": True,
+                    "action_id": action_record_id,
+                    "restarted_at": time.time(),
+                    "stop_result": stop_result
+                }
+            else:
+                logging.error("Failed to start action %s after restart", action_record_id)
+                return {
+                    "success": False,
+                    "reason": "start_failed_after_restart",
+                    "action_id": action_record_id
+                }
+
+        except Exception as e:
+            logging.error("Error restarting action %s: %s", action_record_id, str(e))
+            return {
+                "success": False,
+                "reason": "restart_failed",
+                "error": str(e),
+                "action_id": action_record_id
+            }
+
     @log_errors(raise_exception=True)
     def start_actions_manager(self) -> None:
         """Start the actions manager main loop."""

matrice_compute/compute_operations_handler.py

@@ -0,0 +1,490 @@
+"""
+Compute Operations Handler - Kafka Event-Driven Operations Manager
+
+This module handles compute instance operations (start/stop/restart) triggered from
+the frontend dashboard via Kafka events. It consumes events from the 'compute_operations'
+topic and performs the actual operations on compute instances and their actions.
+
+Uses EventListener from matrice_common for simplified Kafka consumption.
+
+Event Structure:
+{
+    "instance_id": "string",
+    "action_record_id": "string",  # Can be ObjectID("000000000000000000000000") or all zeros for instance-level operations
+    "operation": "start|stop|restart",
+    "account_number": 12345,
+    "requested_by": "user@example.com",
+    "request_id": "uuid-string",
+    "timestamp": "2025-11-21T10:30:00.123Z"
+}
+"""
+
+import logging
+import re
+import time
+from typing import Dict, Any, Optional
+import sys
+import traceback
+import os
+import subprocess
+
+from matrice_common.stream.event_listener import EventListener
+
+# Configure logging
+logger = logging.getLogger(__name__)
+
+
+class ComputeOperationsHandler:
+    """
+    Handles Kafka-based compute operations for instance and action management.
+
+    This class uses EventListener from matrice_common to listen for operation
+    events from the 'compute_operations' Kafka topic. It delegates operations
+    to the ActionsManager for execution and updates status via API calls.
+    """
+
+    KAFKA_TOPIC = "compute_operations"
+
+    def __init__(self, actions_manager, session, scaling, instance_id: str):
+        """
+        Initialize the Compute Operations Handler.
+
+        Args:
+            actions_manager: Reference to the ActionsManager instance
+            session: Session object for authentication and Kafka configuration
+            scaling: Scaling service instance for API status updates
+            instance_id: This compute instance's ID for filtering events
+        """
+        self.actions_manager = actions_manager
+        self.session = session
+        self.scaling = scaling
+        self.instance_id = instance_id
+        self.event_listener: Optional[EventListener] = None
+        self.running = False
+
+        logger.info(f"Initializing ComputeOperationsHandler for instance ID: {instance_id}")
+
+    def start(self) -> bool:
+        """
+        Start the operations handler using EventListener.
+
+        Returns:
+            bool: True if started successfully, False otherwise
+        """
+        if self.running:
+            logger.warning("ComputeOperationsHandler is already running")
+            return False
+
+        try:
+            self.event_listener = EventListener(
+                session=self.session,
+                topics=[self.KAFKA_TOPIC],
+                event_handler=self._handle_operation_event,
+                filter_field='instance_id',
+                filter_value=self.instance_id,
+                consumer_group_id=f"compute_ops_{self.instance_id}"
+            )
+            self.running = self.event_listener.start()
+
+            if self.running:
+                logger.info("ComputeOperationsHandler started successfully")
+            else:
+                logger.error("ComputeOperationsHandler failed to start")
+
+            return self.running
+
+        except Exception as e:
+            logger.error(f"Failed to start ComputeOperationsHandler: {e}")
+            logger.error(traceback.format_exc())
+            return False
+
+    def stop(self):
+        """
+        Stop the operations handler gracefully.
+        """
+        logger.info("Stopping ComputeOperationsHandler...")
+        self.running = False
+
+        if self.event_listener:
+            self.event_listener.stop()
+
+        logger.info("ComputeOperationsHandler stopped")
+
+    def _handle_operation_event(self, event: Dict[str, Any]):
+        """
+        Handle incoming operation event from Kafka.
+
+        This is the callback function passed to EventListener.
+
+        Args:
+            event: The operation event dictionary
+        """
+        logger.info(f"Received operation event: {event}")
+
+        # Validate event structure
+        if not self._validate_event(event):
+            logger.error(f"Invalid event structure: {event}")
+            return
+
+        # Process the operation
+        self._process_operation(event)
+
+    def _is_instance_level_operation(self, action_record_id: str) -> bool:
+        """
+        Check if action_record_id represents an instance-level operation.
+        Instance-level operations are identified by action_record_id containing only zeros,
+        which can come in various formats:
+        - "000000000000000000000000"
+        - "ObjectID(\"000000000000000000000000\")"
+        - "ObjectID('000000000000000000000000')"
+
+        Args:
+            action_record_id: The action record ID to check
+
+        Returns:
+            True if this is an instance-level operation, False otherwise
+        """
+        if not action_record_id:
+            return False
+
+        # Handle ObjectID("...") or ObjectID('...') format from Kafka messages
+        clean_id = action_record_id
+        if 'ObjectID' in action_record_id:
+            match = re.search(r'ObjectID\(["\']([^"\']+)["\']\)', action_record_id)
+            if match:
+                clean_id = match.group(1)
+
+        # Check if the string contains only zeros (any length)
+        return clean_id.replace('0', '') == ''
+
+    def _extract_action_record_id(self, action_record_id: str) -> str:
+        """
+        Extract the actual action record ID from various formats.
+
+        Args:
+            action_record_id: The raw action record ID (may be wrapped in ObjectID)
+
+        Returns:
+            The extracted action record ID string
+        """
+        if not action_record_id:
+            return action_record_id
+
+        # Handle ObjectID("...") or ObjectID('...') format
+        if 'ObjectID' in action_record_id:
+            match = re.search(r'ObjectID\(["\']([^"\']+)["\']\)', action_record_id)
+            if match:
+                return match.group(1)
+
+        return action_record_id
+
+    def _validate_event(self, event: Dict[str, Any]) -> bool:
+        """
+        Validate that the event has all required fields.
+
+        Args:
+            event: The event dictionary to validate
+
+        Returns:
+            True if event is valid, False otherwise
+        """
+        required_fields = [
+            "instance_id",
+            "action_record_id",
+            "operation",
+            "account_number",
+            "requested_by",
+            "request_id",
+            "timestamp"
+        ]
+
+        for field in required_fields:
+            if field not in event:
+                logger.error(f"Missing required field: {field}")
+                return False
+
+        # Validate operation type
+        valid_operations = ["start", "stop", "restart"]
+        if event["operation"] not in valid_operations:
+            logger.error(f"Invalid operation: {event['operation']}. Must be one of {valid_operations}")
+            return False
+
+        return True
+
+    def _process_operation(self, event: Dict[str, Any]):
+        """
+        Process a compute operation event.
+
+        Args:
+            event: The operation event dictionary
+        """
+        operation = event["operation"]
+        raw_action_record_id = event["action_record_id"]
+        action_record_id = self._extract_action_record_id(raw_action_record_id)
+        request_id = event["request_id"]
+        requested_by = event["requested_by"]
+
+        logger.info(f"Processing {operation} operation for action {action_record_id} "
+                   f"(request: {request_id}, user: {requested_by})")
+
+        try:
+            # Check if this is an instance-level operation (action_record_id contains only zeros)
+            is_instance_operation = self._is_instance_level_operation(raw_action_record_id)
+
+            if is_instance_operation:
+                result = self._handle_instance_operation(operation, event)
+            else:
+                result = self._handle_action_operation(operation, action_record_id, event)
+
+            # Update status via API and logging
+            self._update_operation_status(event, action_record_id, "completed", result)
+
+        except Exception as e:
+            error_msg = f"Operation failed: {str(e)}"
+            logger.error(error_msg)
+            logger.error(traceback.format_exc())
+
+            # Update failure status
+            self._update_operation_status(event, action_record_id, "failed", {"error": error_msg})
+
+    def _handle_action_operation(self, operation: str, action_record_id: str,
+                                 event: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Handle operations on a specific action.
+
+        Args:
+            operation: The operation type (start/stop/restart)
+            action_record_id: The action record ID to operate on
+            event: The full event dictionary
+
+        Returns:
+            Result dictionary with operation details
+        """
+        if operation == "start":
+            return self._start_action(action_record_id, event)
+        elif operation == "stop":
+            return self._stop_action(action_record_id, event)
+        elif operation == "restart":
+            return self._restart_action(action_record_id, event)
+        else:
+            raise ValueError(f"Unknown operation: {operation}")
+
+    def _handle_instance_operation(self, operation: str, event: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Handle operations on the entire instance (the Python application itself).
+
+        Args:
+            operation: The operation type (start/stop/restart)
+            event: The full event dictionary
+
+        Returns:
+            Result dictionary with operation details (may not return if app is killed/restarted)
+        """
+        logger.info(f"Executing instance-level {operation} operation on Python application")
+
+        if operation == "stop":
+            # Kill the Python application itself
+            logger.critical("Instance-level STOP: Killing Python application process")
+            try:
+                # Log status before killing
+                logger.warning(
+                    f"Operation {operation} on instance {self.instance_id}: "
+                    f"completed - killing_application (PID: {os.getpid()})"
+                )
+                # Give a moment for logs to be written
+                time.sleep(0.5)
+            except Exception as e:
+                logger.error(f"Failed to log status before kill: {e}")
+
+            # Forcefully exit the application
+            logger.critical(f"Terminating Python application (PID: {os.getpid()})")
+            os._exit(0)  # Forceful exit, doesn't call cleanup handlers
+
+        elif operation == "restart":
+            # Restart the Python application itself
+            logger.critical("Instance-level RESTART: Restarting Python application process")
+            try:
+                # Log status before restarting
+                logger.warning(
+                    f"Operation {operation} on instance {self.instance_id}: "
+                    f"completed - restarting_application (PID: {os.getpid()})"
+                )
+                # Give a moment for logs to be written
+                time.sleep(0.5)
+            except Exception as e:
+                logger.error(f"Failed to log status before restart: {e}")
+
+            # Restart the application
+            logger.critical(f"Restarting Python application (PID: {os.getpid()})")
+            self._restart_application()
+
+        elif operation == "start":
+            # Start doesn't make sense for instance-level
+            logger.warning("Start operation not supported at instance level")
+            return {
+                "operation": operation,
+                "instance_level": True,
+                "status": "not_supported",
+                "message": "Start operation is not supported at instance level"
+            }
+
+        # This should not be reached for stop/restart operations
+        return {
+            "operation": operation,
+            "instance_level": True,
+            "status": "completed"
+        }
+
+    def _restart_application(self):
+        """
+        Restart the Python application by replacing the current process.
+        This uses os.execv() to replace the current process with a new one.
+        """
+        try:
+            python_executable = sys.executable
+            script_args = sys.argv
+
+            logger.info(f"Restarting with: {python_executable} {' '.join(script_args)}")
+
+            # Use os.execv() to replace the current process
+            # This will restart the application with the same arguments
+            os.execv(python_executable, [python_executable] + script_args)
+
+        except Exception as e:
+            logger.error(f"Failed to restart application: {e}")
+            logger.error(traceback.format_exc())
+            # Fallback: try using subprocess to start a new process and exit
+            try:
+                logger.info("Attempting fallback restart method")
+                python_executable = sys.executable
+                script_args = sys.argv
+
+                # Start new process
+                subprocess.Popen([python_executable] + script_args)
+                # Exit current process
+                logger.critical("New process started, exiting current process")
+                os._exit(0)
+            except Exception as fallback_error:
+                logger.error(f"Fallback restart also failed: {fallback_error}")
+                logger.error(traceback.format_exc())
+                # Last resort: just exit
+                os._exit(1)
+
+    def _start_action(self, action_record_id: str, event: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Start a specific action.
+
+        Args:
+            action_record_id: The action record ID to start
+            event: The full event dictionary
+
+        Returns:
+            Result dictionary
+        """
+        logger.info(f"Starting action: {action_record_id}")
+
+        # Check if action is already running
+        current_actions = self.actions_manager.get_current_actions()
+        if action_record_id in current_actions:
+            action_instance = current_actions[action_record_id]
+            if action_instance.is_running():
+                logger.warning(f"Action {action_record_id} is already running")
+                return {
+                    "status": "already_running",
+                    "action_id": action_record_id
+                }
+
+        # Fetch action details from backend and start it
+        # This will be handled by the ActionsManager's normal flow
+        # Force a fetch to pick up this specific action
+        self.actions_manager.fetch_actions()
+
+        return {
+            "status": "started",
+            "action_id": action_record_id
+        }
+
+    def _stop_action(self, action_record_id: str, event: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Stop a specific action.
+
+        Args:
+            action_record_id: The action record ID to stop
+            event: The full event dictionary
+
+        Returns:
+            Result dictionary
+        """
+        logger.info(f"Stopping action: {action_record_id}")
+
+        result = self.actions_manager.stop_action(action_record_id)
+
+        return {
+            "status": "stopped",
+            "action_id": action_record_id,
+            "details": result
+        }
+
+    def _restart_action(self, action_record_id: str, event: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Restart a specific action.
+
+        Args:
+            action_record_id: The action record ID to restart
+            event: The full event dictionary
+
+        Returns:
+            Result dictionary
+        """
+        logger.info(f"Restarting action: {action_record_id}")
+
+        result = self.actions_manager.restart_action(action_record_id)
+
+        return {
+            "status": "restarted",
+            "action_id": action_record_id,
+            "details": result
+        }
+
+    def _update_operation_status(self, event: Dict[str, Any], action_record_id: str,
+                                 status: str, result: Dict[str, Any]):
+        """
+        Update operation status via API and logging.
+
+        Args:
+            event: The original event
+            action_record_id: The extracted action record ID
+            status: Operation status (completed/failed)
+            result: Result details
+        """
+        operation = event["operation"]
+        request_id = event["request_id"]
+
+        # Log status as warning for visibility
+        logger.warning(
+            f"Operation {operation} on {action_record_id}: {status} - "
+            f"request_id={request_id}, result={result}"
+        )
+
+        # Update via API (for action-level operations only)
+        if not self._is_instance_level_operation(event["action_record_id"]):
+            try:
+                # Determine isRunning based on operation and status
+                is_running = False
+                if status == "completed":
+                    if operation == "start":
+                        is_running = True
+                    elif operation == "restart":
+                        is_running = True
+                    elif operation == "stop":
+                        is_running = False
+
+                self.scaling.update_action_status(
+                    service_provider=os.environ.get("SERVICE_PROVIDER", ""),
+                    action_record_id=action_record_id,
+                    status=status,
+                    isRunning=is_running,
+                )
+                logger.info(f"API status updated for action {action_record_id}: {status}")
+            except Exception as e:
+                logger.error(f"Failed to update API status for action {action_record_id}: {e}")

matrice_compute/instance_manager.py

@@ -7,6 +7,7 @@ import threading
 import time
 from matrice_compute.actions_manager import ActionsManager
 from matrice_compute.actions_scaledown_manager import ActionsScaleDownManager
+from matrice_compute.compute_operations_handler import ComputeOperationsHandler
 from matrice_compute.instance_utils import (
     get_instance_info,
     get_decrypted_access_key_pair,
@@ -90,6 +91,22 @@ class InstanceManager:
         logging.info("InstanceManager initialized with machine resources tracker")
         self.actions_resources_tracker = ActionsResourcesTracker(self.scaling)
         logging.info("InstanceManager initialized with actions resources tracker")
+
+        # Initialize Compute Operations Handler for event-driven operations
+        # Uses EventListener from matrice_common for simplified Kafka consumption
+        try:
+            instance_id = os.environ.get("INSTANCE_ID")
+            self.compute_operations_handler = ComputeOperationsHandler(
+                actions_manager=self.actions_manager,
+                session=self.session,
+                scaling=self.scaling,
+                instance_id=instance_id
+            )
+            logging.info("InstanceManager initialized with Compute Operations Handler for instance ID: %s", instance_id)
+        except Exception as e:
+            logging.warning("Failed to initialize Compute Operations Handler: %s", e)
+            self.compute_operations_handler = None
+
         self.poll_interval = 10
         # Note: encryption_key is set in _setup_env_credentials
         logging.info("InstanceManager initialized.")
@@ -252,6 +269,14 @@ class InstanceManager:
         Returns:
             tuple: (instance_manager_thread, actions_manager_thread)
         """
+        # Start Compute Operations Handler in background thread
+        if self.compute_operations_handler:
+            try:
+                self.compute_operations_handler.start()
+                logging.info("Started Compute Operations Handler")
+            except Exception as exc:
+                logging.error("Failed to start Compute Operations Handler: %s", str(exc))
+
         # Create and start threads
         instance_manager_thread = threading.Thread(
             target=self.start_instance_manager,

matrice_compute/resources_tracker.py

@@ -402,8 +402,13 @@ class ActionsResourcesTracker:
                 new_args.extend(x.replace('"', "").replace("'", "") for x in arg.split(" "))
             return new_args
 
-        args_24 = [arg for arg in remove_quotation_marks(inspect_data["Args"]) if len(arg) == 24 and "pypi" not in arg]
-        action_record_id = args_24[-1] if args_24 else None
+        def is_valid_objectid(s: str) -> bool:
+            """Check if string is a valid MongoDB ObjectId (24 hex characters)"""
+            s = s.strip()
+            return len(s) == 24 and all(c in '0123456789abcdefABCDEF' for c in s)
+        
+        valid_objectids = [arg for arg in remove_quotation_marks(inspect_data["Args"]) if is_valid_objectid(arg)]
+        action_record_id = valid_objectids[-1] if valid_objectids else None
         if not action_record_id:
             logging.debug("No valid action_id found for the container. Container ID: %s, Args: %s", container.id, inspect_data["Args"])
         duration = calculate_time_difference(start_time, finish_time)

matrice_compute/scaling.py

@@ -2,6 +2,7 @@
 
 import os
 import logging
+import base64
 from matrice_common.utils import log_errors
 
 class Scaling:
@@ -33,6 +34,28 @@ class Scaling:
             "Initialized Scaling with instance_id: %s (REST API only)",
             instance_id
         )
+    
+    @log_errors(default_return=None, log_error=True)
+    def get_kafka_bootstrap_servers(self):
+        """Get Kafka bootstrap servers from API and decode base64 fields.
+
+        Returns:
+            str: Kafka bootstrap servers in format "ip:port"
+
+        Raises:
+            ValueError: If unable to fetch Kafka configuration
+        """
+        path = "/v1/actions/get_kafka_info"
+        response = self.rpc.get(path=path)
+        if not response or not response.get("success"):
+            raise ValueError(f"Failed to fetch Kafka config: {response.get('message', 'No response')}")
+        encoded_ip = response["data"]["ip"]
+        encoded_port = response["data"]["port"]
+        ip = base64.b64decode(encoded_ip).decode("utf-8")
+        port = base64.b64decode(encoded_port).decode("utf-8")
+        bootstrap_servers = f"{ip}:{port}"
+        # logging.info(f"Retrieved Kafka bootstrap servers: {bootstrap_servers}")
+        return bootstrap_servers
 
     @log_errors(default_return=(None, "Error processing response", "Response processing failed"), log_error=True)
     def handle_response(self, resp, success_message, error_message):

{matrice_compute-0.1.25.dist-info → matrice_compute-0.1.26.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: matrice_compute
-Version: 0.1.25
+Version: 0.1.26
 Summary: Common server utilities for Matrice.ai services
 Author-email: "Matrice.ai" <dipendra@matrice.ai>
 License-Expression: MIT

{matrice_compute-0.1.25.dist-info → matrice_compute-0.1.26.dist-info}/RECORD

@@ -1,17 +1,18 @@
 matrice_compute/__init__.py,sha256=ZzQcFsT005VCgq9VZUh565f4upOooEb_FwZ6RgweNZs,597
 matrice_compute/action_instance.py,sha256=SYUZrfj6dtcgEjeEgCyKlrc2p2o08jlW84Y__V4Aqew,69552
-matrice_compute/actions_manager.py,sha256=5U-xM6tl_Z6x96bi-c7AJM9ru80LqTN8f5Oce8dAu_A,7780
+matrice_compute/actions_manager.py,sha256=Iex5uw0PLRR4pvIAZDxc2CypucbanKDbJ3SK8mMGXK8,18148
 matrice_compute/actions_scaledown_manager.py,sha256=pJ0nduNwHWZ10GnqJNx0Ok7cVWabQ_M8E2Vb9pH3A_k,2002
-matrice_compute/instance_manager.py,sha256=8USyX09ZxLvnVNIrjRogbyUeMCfgWnasuRqYkkVF4tQ,10146
+matrice_compute/compute_operations_handler.py,sha256=amcMhmXtv2irE6qK8Vbgec_8uFqjWmVVp0VWq-73_MU,17781
+matrice_compute/instance_manager.py,sha256=sUkDsy_XrPp7CKQxlujQRz3E_8rVbVZOy7byJOgMlEs,11376
 matrice_compute/instance_utils.py,sha256=N4yPDvNukFEEBngR0lEt4x_XT5hur1q0P-spM2xQIlU,42025
 matrice_compute/prechecks.py,sha256=W9YmNF3RcLhOf4U8WBlExvFqDw1aGWSNTlJtA73lbDQ,17196
 matrice_compute/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-matrice_compute/resources_tracker.py,sha256=pkdt0aVKx_TpY_Sq---73w9INkDffZZe3mZGlp1EftE,22573
-matrice_compute/scaling.py,sha256=CeT_lxJNkjJamRETG1lWaOtdSr5ySmcaMcqt7-lFRbo,23731
+matrice_compute/resources_tracker.py,sha256=wy1huqB3Tw_kYC2wfnLa9iSyhDmgI7WQ5I9Kyr-1RSs,22829
+matrice_compute/scaling.py,sha256=JNOgSpAPqbTlZ4qJokkdS9PehqyFwfPh4q98qrfNVCQ,24708
 matrice_compute/shutdown_manager.py,sha256=0MYV_AqygqR9NEntYf7atUC-PbWXyNkm1f-8c2aizgA,13234
 matrice_compute/task_utils.py,sha256=3qIutiQdYPyGRxH9ZwLbqdg8sZcnp6jp08pszWCRFl0,2820
-matrice_compute-0.1.25.dist-info/licenses/LICENSE.txt,sha256=_uQUZpgO0mRYL5-fPoEvLSbNnLPv6OmbeEDCHXhK6Qc,1066
-matrice_compute-0.1.25.dist-info/METADATA,sha256=YxPD7gjTuET4wsbq0ywgIw8AmR8U7-EdAuZlIVIramg,1038
-matrice_compute-0.1.25.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-matrice_compute-0.1.25.dist-info/top_level.txt,sha256=63Plr3L1GzBUWZO5JZaFkiv8IcB10xUPU-9w3i6ptvE,16
-matrice_compute-0.1.25.dist-info/RECORD,,
+matrice_compute-0.1.26.dist-info/licenses/LICENSE.txt,sha256=_uQUZpgO0mRYL5-fPoEvLSbNnLPv6OmbeEDCHXhK6Qc,1066
+matrice_compute-0.1.26.dist-info/METADATA,sha256=t7TsI5DcNElRmlKsa8CArXCcA4iBO-9QwZ6j9UQOdg0,1038
+matrice_compute-0.1.26.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+matrice_compute-0.1.26.dist-info/top_level.txt,sha256=63Plr3L1GzBUWZO5JZaFkiv8IcB10xUPU-9w3i6ptvE,16
+matrice_compute-0.1.26.dist-info/RECORD,,