cavisson-pythonagent 0.0.1__py3-none-any.whl

pythonagent/agent/probes/havoc/havoc_manager.py

@@ -0,0 +1,981 @@
+import json
+from collections import namedtuple
+import random
+import time
+import logging
+import os
+import threading
+
+from pythonagent.agent.internal import udp
+from pythonagent.agent.probes.havoc.havoc_constants import HavocType, HAVOC_TYPES, HAVOC_TYPE_NAMES, EXPIRE_DURATION_MS, SIZE_OF_OBJECTS_BYTES, HAVOC_PROFILES_JSON
+from pythonagent.agent.probes.havoc.custom_memory_stress import apply_memory_leak, stop_memory_leak
+from pythonagent.agent.probes.havoc.custom_thread_stress import apply_thread_leak, stop_thread_leak, ThreadAllocationError
+from pythonagent.utils import get_current_timestamp_in_ms
+import pythonagent.agent as agent
+
+
+
+class NDHavocException(Exception):
+    def __init__(self, message):
+        self.message = message
+        super().__init__(self.message)
+
+
+class NDNetHavocRequest(object):
+    def __init__(self, havoc_conf, header_dict):
+
+        self.logger = logging.getLogger('pythonagent.agent')
+        self.startTime = get_current_timestamp_in_ms()
+
+        self.header_dict = header_dict
+        self.nhmid = header_dict["NHMID"]
+
+        self.havocType = HavocType(int(havoc_conf.havocType))
+        self.totalDurationInSec = getattr(havoc_conf, "totalDurationInSec", 0)
+
+        if hasattr(havoc_conf, 'delayInSec'):
+            self.delayInSec = havoc_conf.delayInSec
+
+        if self.havocType in (HavocType.INBOUND_SERVICE_DELAY, HavocType.OUTBOUND_SERVICE_DELAY, HavocType.METHOD_CALL_DELAY):
+            self.havocStarttime = self.startTime
+            self.shootUpDuration = getattr(havoc_conf, 'shootUpDuration', 0)
+            self.stableDuration = getattr(havoc_conf, 'stableDuration', 0)
+            self.tumbleDuration = getattr(havoc_conf, 'tumbleDuration', 0)
+            self.stableStartTime = self.shootUpDuration
+            self.stableDurationEndTime = self.shootUpDuration + self.stableDuration
+            self.tumbleDownStartTime = self.totalDurationInSec * 1000 - self.tumbleDuration
+
+        if self.havocType in (HavocType.INBOUND_SERVICE_DELAY, HavocType.INBOUND_SERVICE_FAILURE):
+            self.bTNameMode = havoc_conf.bTNameMode
+            if hasattr(havoc_conf, 'uRL'):
+                self.uRL = havoc_conf.uRL
+            if hasattr(havoc_conf, 'bTName'):
+                self.bTName = havoc_conf.bTName
+
+        if self.havocType in (HavocType.OUTBOUND_SERVICE_DELAY, HavocType.OUTBOUND_SERVICE_FAILURE):
+            self.backendNameMod = havoc_conf.backendNameMod
+            if hasattr(havoc_conf, 'bTNameMode'):
+                self.bTNameMode = havoc_conf.bTNameMode
+            if hasattr(havoc_conf, 'uRL'):
+                self.uRL = havoc_conf.uRL
+            if hasattr(havoc_conf, 'bTName'):
+                self.bTName = havoc_conf.bTName
+            if hasattr(havoc_conf, 'hostname'):
+                self.hostname = havoc_conf.hostname
+            if hasattr(havoc_conf, 'backendName'):
+                self.backendName = havoc_conf.backendName
+
+        if self.havocType in (HavocType.METHOD_CALL_DELAY, HavocType.METHOD_CALL_FAILURE):
+            self.methodMod = havoc_conf.methodMod
+            if hasattr(havoc_conf, 'methodFQM'):
+                self.methodFQM = havoc_conf.methodFQM
+            if hasattr(havoc_conf, 'bTNameMode'):
+                self.bTNameMode = havoc_conf.bTNameMode
+            if hasattr(havoc_conf, 'uRL'):
+                self.uRL = havoc_conf.uRL
+            if hasattr(havoc_conf, 'bTName'):
+                self.bTName = havoc_conf.bTName
+
+        if self.havocType == HavocType.CUSTOM_MEMORY_LEAK:
+            self.leaksINMB = havoc_conf.leaksINMB
+            self.objectSize = havoc_conf.objectSize
+            self.shootUpDuration = getattr(havoc_conf, "shootUpDuration", 0)
+
+        if self.havocType == HavocType.APPLICATION_KILL:
+            self.methodMod = havoc_conf.methodMod
+            self.methodFQM = havoc_conf.methodFQM
+            self.customExitCode = havoc_conf.customExitCode
+            self.methodEntryOrExit = havoc_conf.methodEntryOrExit
+            self.killApplication = havoc_conf.killApplication
+            self.latencyInSec = havoc_conf.latencyInSec
+
+        if self.havocType == HavocType.CUSTOM_THREAD_LEAK:
+            self.numberOfThreads = havoc_conf.numberOfThreads
+            self.threadSleepTimeInMS = havoc_conf.threadSleepTimeInMS
+
+            self.shootUpDuration = getattr(havoc_conf, "shootUpDuration", 0)
+            self.stableDuration = getattr(havoc_conf, "stableDuration", 0)
+            self.tumbleDuration = getattr(havoc_conf, "tumbleDuration", 0)
+
+        try:
+            
+            self.protocol = getattr(havoc_conf, 'protocol', None)
+            self.bTName = havoc_conf.bTName
+            self.isHavocEnable = False
+            self.threshOld = havoc_conf.threshOld
+            self.backendName = havoc_conf.backendName
+
+        except Exception as e:
+            self.logger.warning("Optional parameters not found {}".format(e))
+
+
+class NDNetHavocMonitor(object):
+
+    __instance = None
+
+    @staticmethod
+    def get_instance():
+        if NDNetHavocMonitor.__instance == None:
+            NDNetHavocMonitor()
+        return NDNetHavocMonitor.__instance
+
+    def __init__(self):
+        """ Virtually private constructor. """
+        if NDNetHavocMonitor.__instance != None:
+            raise Exception("This class is a singleton! Use get_instance()")
+        else:
+            NDNetHavocMonitor.__instance = self
+
+            self.enableNetHavoc = False
+
+            self.netHavocConfigMap = {
+                "InBoundService_Delay": [],
+                "InBoundService_Failure": [],
+                "OutBoundService_Delay": [],
+                "OutBoundService_Failure": [],
+                "MethodCall_Delay": [],
+                "MethodCall_Failure": [],
+                "Custom_Memory_Leak": [],
+                "Custom_Thread_Leak": [],
+                "Application_Kill": []
+            }
+
+            self.logger = logging.getLogger('pythonagent.agent')
+
+    def set_enable_net_havoc(self, enable_logs_net_havoc):
+        try:
+            if enable_logs_net_havoc == 1:
+                self.enableNetHavoc = True
+            else:
+                self.enableNetHavoc = False
+        except Exception as e:
+            raise e
+
+    def validate_config(self, havoc_conf):
+        self.logger.info("Validating config")
+        is_valid = False
+
+        # `enable` field is no longer present in incoming JSON, so this validation is disabled to keep inbound Havoc working.
+        '''
+        # Validate enable
+        if not hasattr(havoc_conf, 'enable'):
+            return_message = "enable not found"
+            return is_valid, return_message
+
+        else:
+            if havoc_conf.enable not in [0, 1]:
+                return_message = "invalid enable not found"
+                return is_valid, return_message
+            else:
+                logging.debug("enable validated")
+        '''
+        # Validate havocType
+        if not hasattr(havoc_conf, 'havocType'):
+            return_message = "havocType not found"
+            return is_valid, return_message
+        else:
+            try:
+                havoc_type = HavocType(int(havoc_conf.havocType))
+            except ValueError:
+                return_message = "invalid havocType"
+                return is_valid, return_message
+            logging.debug("havocType validated")
+
+        if havoc_type == HavocType.APPLICATION_KILL:
+            is_valid, return_message = self.validate_application_kill_config(havoc_conf)
+            return is_valid, return_message
+
+        if havoc_type == HavocType.CUSTOM_THREAD_LEAK:
+            is_valid, return_message = self.validate_thread_leak_config(havoc_conf)
+            return is_valid, return_message
+
+        # Validate totalDurationInSec
+        if not hasattr(havoc_conf, 'totalDurationInSec'):
+            return_message = "totalDurationInSec not found"
+            return is_valid, return_message
+        else:
+            if not isinstance(havoc_conf.totalDurationInSec, int):
+                return_message = "invalid totalDurationInSec"
+                return is_valid, return_message
+            else:
+                logging.debug("totalDurationInSec validated")
+
+        # Validate threshOld
+        if not hasattr(havoc_conf, 'threshOld'):
+            return_message = "threshOld not found"
+            return is_valid, return_message
+        else:
+            if not hasattr(havoc_conf.threshOld, 'percentage'):
+                return_message = "invalid threshOld"
+                return is_valid, return_message
+            else:
+                self.logger.debug("threshOld validated")
+
+        # Validate uRL for Havoc Type 1 and 2
+        if havoc_type in (HavocType.INBOUND_SERVICE_DELAY, HavocType.INBOUND_SERVICE_FAILURE):
+            self.logger.debug("bTNameMode: {}".format(havoc_conf.bTNameMode))
+            if havoc_conf.bTNameMode == 0:
+                pass
+            elif havoc_conf.bTNameMode in [1, 2]:
+                if not hasattr(havoc_conf, 'bTName'):
+                    return_message = "bTName not found for inbound service"
+                    return is_valid, return_message
+                elif len(havoc_conf.bTName.strip()) == 0:
+                    return_message = "invalid bTName for inbound service"
+                    return is_valid, return_message
+                else:
+                    self.logger.debug("bTName validated")
+            elif havoc_conf.bTNameMode == 3:
+                
+                if not hasattr(havoc_conf, 'uRL'):
+                    return_message = "uRL not found for inbound service"
+                    return is_valid, return_message
+                
+                else:
+                    if len(havoc_conf.uRL) == 0:
+                        return_message = "invalid uRL  for inbound service"
+                        return is_valid, return_message
+                    else:
+                        self.logger.debug("uRL validated")
+                
+            else:
+                return_message = "invalid bTNameMode"
+                return is_valid, return_message
+        else:
+            self.logger.debug("uRL not needed")
+
+        # Validate hostname and protocol for Havoc Type 3 and 4
+        if havoc_type in (HavocType.OUTBOUND_SERVICE_DELAY, HavocType.OUTBOUND_SERVICE_FAILURE):
+            if hasattr(havoc_conf, 'bTNameMode'):
+                if havoc_conf.bTNameMode not in [0, 1, 2, 3]:
+                    return_message = "invalid bTNameMode for outbound service"
+                    return is_valid, return_message
+                if havoc_conf.bTNameMode in [1, 2]:
+                    if not hasattr(havoc_conf, 'bTName'):
+                        return_message = "bTName not found for outbound service"
+                        return is_valid, return_message
+                    if len(havoc_conf.bTName.strip()) == 0:
+                        return_message = "invalid bTName for outbound service"
+                        return is_valid, return_message
+                if havoc_conf.bTNameMode == 3:
+                    if not hasattr(havoc_conf, 'uRL'):
+                        return_message = "uRL not found for outbound service"
+                        return is_valid, return_message
+                    if len(havoc_conf.uRL.strip()) == 0:
+                        return_message = "invalid uRL for outbound service"
+                        return is_valid, return_message
+
+            if havoc_conf.backendNameMod == 0:
+                if hasattr(havoc_conf, 'backendName') and len(havoc_conf.backendName.strip()) == 0:
+                    return_message = "invalid backendName for outbound service"
+                    return is_valid, return_message
+            elif havoc_conf.backendNameMod in [1, 2]:
+                if not hasattr(havoc_conf, 'backendName'):
+                    return_message = "backendName not found for outbound service"
+                    return is_valid, return_message
+                elif len(havoc_conf.backendName.strip()) == 0:
+                    return_message = "invalid backendName for outbound service"
+                    return is_valid, return_message
+                else:
+                    self.logger.debug("backendName validated")
+            elif havoc_conf.backendNameMod == 3:
+                if not hasattr(havoc_conf, 'hostname'):
+                    return_message = "hostname not found for outbound service"
+                    return is_valid, return_message
+                else:
+                    if len(havoc_conf.hostname) == 0:
+                        return_message = "invalid hostname for outbound service"
+                        return is_valid, return_message
+                    else:
+                        self.logger.debug("hostname validated")
+
+                    if not hasattr(havoc_conf, 'protocol'):
+                        if havoc_conf.protocol not in ["Http", "Non Http"]:
+                            return_message = "invalid protocol for outbound service"
+                            return is_valid, return_message
+                        else:
+                            self.logger.debug("protocol validated")
+                    else:
+                        self.logger.debug("protocol not found")
+            elif havoc_conf.backendNameMod == 4:
+                if not hasattr(havoc_conf, 'hostname'):
+                    return_message = "hostname not found for outbound service"
+                    return is_valid, return_message
+                elif len(havoc_conf.hostname) == 0:
+                    return_message = "invalid hostname for outbound service"
+                    return is_valid, return_message
+                else:
+                    self.logger.debug("hostname validated")
+            else:
+                return_message = "invalid backendNameMod"
+                return is_valid, return_message
+
+        else:
+            self.logger.debug("hostname not needed")
+
+        # Validate methodFQM
+        if havoc_type in (HavocType.METHOD_CALL_DELAY, HavocType.METHOD_CALL_FAILURE):
+            if not hasattr(havoc_conf, 'methodFQM'):
+                return_message = "methodFQM not found for method"
+                return is_valid, return_message
+            else:
+                if len(havoc_conf.methodFQM) == 0:
+                    return_message = "invalid methodFQM for method"
+
+                    return is_valid, return_message
+                else:
+                    self.logger.debug("methodFQM validated")
+
+            if hasattr(havoc_conf, 'bTNameMode'):
+                if havoc_conf.bTNameMode not in [0, 1, 2, 3]:
+                    return_message = "invalid bTNameMode for method"
+                    return is_valid, return_message
+                if havoc_conf.bTNameMode in [1, 2]:
+                    if not hasattr(havoc_conf, 'bTName'):
+                        return_message = "bTName not found for method"
+                        return is_valid, return_message
+                    if len(havoc_conf.bTName.strip()) == 0:
+                        return_message = "invalid bTName for method"
+                        return is_valid, return_message
+                if havoc_conf.bTNameMode == 3:
+                    if not hasattr(havoc_conf, 'uRL'):
+                        return_message = "uRL not found for method"
+                        return is_valid, return_message
+                    if len(havoc_conf.uRL.strip()) == 0:
+                        return_message = "invalid uRL for method"
+                        return is_valid, return_message
+        else:
+            self.logger.debug("methodFQM not needed")
+
+        if havoc_type in (HavocType.INBOUND_SERVICE_DELAY, HavocType.OUTBOUND_SERVICE_DELAY, HavocType.METHOD_CALL_DELAY):
+            if not hasattr(havoc_conf, 'delayInSec'):
+                return_message = "delayInSec not found for delay"
+                return is_valid, return_message
+            else:
+                if not isinstance(havoc_conf.delayInSec, int):
+                    return_message = "invalid delayInSec for delay"
+                    return is_valid, return_message
+                else:
+                    self.logger.debug("delayInSec validated")
+        else:
+            self.logger.debug("delayInSec not needed")
+
+        if havoc_type == HavocType.CUSTOM_MEMORY_LEAK:
+            if not hasattr(havoc_conf, "leaksINMB"):
+                return_message = "leaksINMB not found for custom_memory_leak"
+                return is_valid, return_message
+            else:
+                if not hasattr(havoc_conf, "objectSize"):
+                    return_message = "objectSize not found for custom_memory_leak"
+                    return is_valid, return_message
+                else:
+                    self.logger.debug("objectSize validated")
+        else:
+            self.logger.debug("leaksINMB, objectsize not needed")
+
+        if havoc_type == HavocType.CUSTOM_THREAD_LEAK:
+            if not hasattr(havoc_conf, "numberOfThreads"):
+                return_message = "numberOfThreads not found for thread_leak"
+                return is_valid, return_message
+
+            if not hasattr(havoc_conf, "threadSleepTimeInMS"):
+                return_message = "threadSleepTimeInMS not found for thread_leak"
+                return is_valid, return_message
+
+        is_valid = True
+        return_message = "Config Validated"
+        return is_valid, return_message
+
+    def validate_application_kill_config(self, havoc_conf):
+        int_fields = ["methodMod", "customExitCode", "methodEntryOrExit", "killApplication", "latencyInSec"]
+        for field in int_fields:
+            if not hasattr(havoc_conf, field):
+                return False, "{} not found for application kill".format(field)
+            if not isinstance(getattr(havoc_conf, field), int):
+                return False, "invalid {} for application kill".format(field)
+
+        if not hasattr(havoc_conf, "methodFQM"):
+            return False, "methodFQM not found for application kill"
+
+        if havoc_conf.methodMod != 1 or havoc_conf.methodFQM != "":
+            return False, "only direct application kill is supported"
+
+        if havoc_conf.killApplication not in [1, 2, 3, 4]:
+            return False, "invalid killApplication for application kill"
+
+        if havoc_conf.latencyInSec < 0:
+            return False, "invalid latencyInSec for application kill"
+
+        return True, "Config Validated"
+
+    def validate_thread_leak_config(self, havoc_conf):
+        if not hasattr(havoc_conf, "numberOfThreads"):
+            return False, "numberOfThreads not found for thread_leak"
+        if not isinstance(havoc_conf.numberOfThreads, int):
+            return False, "invalid numberOfThreads for thread_leak"
+        if havoc_conf.numberOfThreads < 0:
+            return False, "invalid numberOfThreads for thread_leak"
+        if not hasattr(havoc_conf, "threadSleepTimeInMS"):
+            return False, "threadSleepTimeInMS not found for thread_leak"
+        if not isinstance(havoc_conf.threadSleepTimeInMS, int):
+            return False, "invalid threadSleepTimeInMS for thread_leak"
+        if havoc_conf.threadSleepTimeInMS < 0:
+            return False, "invalid threadSleepTimeInMS for thread_leak"
+        if not hasattr(havoc_conf, "totalDurationInSec"):
+            return False, "totalDurationInSec not found for thread_leak"
+        if not isinstance(havoc_conf.totalDurationInSec, int):
+            return False, "invalid totalDurationInSec for thread_leak"
+        return True, "Config Validated"
+
+    def parse_nethavoc_config(self, json_conf, header_dict):
+        self.logger.info("Parsing config")
+
+        try:
+            havoc_conf = json.loads(json_conf, object_hook=lambda d: namedtuple('X', d.keys())(*d.values()))
+            havoc_type = HavocType(int(havoc_conf.havocType))
+            self.logger.info("Havoc type: {}".format(havoc_type))
+            self.logger.debug("header_dict: {}".format(header_dict))
+            nhmid = header_dict["NHMID"]
+            self.logger.debug("nhmid: {}".format(nhmid))
+            self.logger.info("checking if havoc already exists")
+
+            havoc_type_full_str = HAVOC_TYPE_NAMES[havoc_type]
+            config_list = self.netHavocConfigMap[havoc_type_full_str]
+            self.logger.info("config_list: {}".format(config_list))
+
+            if config_list:
+                for config in config_list:
+                    if nhmid == config.nhmid:
+                        # enableHavoc is sent in the message HEADER, not the JSON
+                        # body - confirmed from live traffic. enableHavoc == 0 is
+                        # a stop request for this already-running havoc.
+                        if int(header_dict.get("enableHavoc", 1)) == 0:
+                            self.logger.info("Stop requested for existing havoc {}".format(nhmid))
+                            config_list.remove(config)
+                            if havoc_type == HavocType.CUSTOM_THREAD_LEAK:
+                                stop_thread_leak()
+                            if havoc_type == HavocType.CUSTOM_MEMORY_LEAK:
+                                stop_memory_leak()
+                            return True
+                        self.logger.info("havoc already exists skipping")
+                        return
+
+            is_valid, ret_message = self.validate_config(havoc_conf)
+
+            if is_valid:
+                self.logger.info(ret_message)
+                req_obj = NDNetHavocRequest(havoc_conf, header_dict)
+                self.logger.debug("Request Object: ".format(req_obj))
+
+                havoc_type_full_str = HAVOC_TYPE_NAMES[havoc_type]
+                self.netHavocConfigMap[havoc_type_full_str].append(req_obj)
+
+                if havoc_type == HavocType.APPLICATION_KILL:
+                    self.apply_application_kill(req_obj)
+                    return True
+
+                if havoc_type == HavocType.CUSTOM_THREAD_LEAK:
+                    # Launch the custom thread-leak worker in a background thread.
+                    self.logger.info("ThreadLeakHavoc: Starting custom thread leak worker thread for NHMID %s", nhmid)
+                    thread_leak_havoc = threading.Thread(target=self.increase_thread_count,
+                                                    args=(),
+                                                    daemon=True)
+                    thread_leak_havoc.start()
+
+                if havoc_type == HavocType.CUSTOM_MEMORY_LEAK:
+                    # Launch the custom memory-leak worker in a background thread.
+                    self.logger.info("MemoryLeakHavoc: Starting custom memory leak worker thread for NHMID %s", nhmid)
+                    havoc_thread = threading.Thread(target=self.increase_memory_usage,
+                                                    args=(),
+                                                    daemon=True)
+
+                    havoc_thread.start()
+
+            else:
+                self.logger.info("Invalid config: {}".format(ret_message))
+
+            
+
+        except Exception as e:
+            self.logger.error("havoc parsing exception: {}".format(e))
+            raise e
+
+    @staticmethod
+    def get_havoc_completion_header(header_dict, status="Success", message="HavocCompleted"):
+        response_header = dict(header_dict)
+        response_header["Status"] = status
+        response_header["Message"] = message
+        response_header["MsgType"] = "1"
+        response_header.pop("SubOperation", None)
+        response_header["Ack"] = "0"
+
+        header_str = "NetDiagnosticMessage 2.0;"
+        for key, value in response_header.items():
+            header_str += "{}:{};".format(key, value)
+        return header_str + "\n\n"
+
+    def send_havoc_status_response(self, config, status, message):
+        try:
+            header_str = self.get_havoc_completion_header(config.header_dict, status=status, message=message)
+            agent_obj = agent.get_agent_instance()
+            agent_obj.send_havoc_response(header_str)
+        except Exception:
+            self.logger.exception("ThreadLeakHavoc: failed to send havoc status response")
+
+
+    def apply_application_kill(self, config):
+        self.logger.info("Applying direct application kill havoc")
+        self.send_havoc_status_response(config, "Success", "HavocCompleted")
+        self._application_kill_after_latency(config.killApplication, config.customExitCode, config.latencyInSec)
+
+    def _application_kill_after_latency(self, kill_application, custom_exit_code, latency_in_sec):
+        if latency_in_sec > 0:
+            self.logger.info("Application kill scheduled after {} seconds".format(latency_in_sec))
+            time.sleep(latency_in_sec)
+        self.kill_application(kill_application, custom_exit_code)
+
+    def kill_application(self, kill_application, custom_exit_code=0):
+        self.logger.warning("Killing application using mode: {}".format(kill_application))
+        logging.shutdown()
+        if kill_application == 1:
+            os._exit(0)
+        elif kill_application == 2:
+            os._exit(1)
+        elif kill_application == 3:
+            os.abort()
+        elif kill_application == 4:
+            os._exit(custom_exit_code)
+        else:
+            self.logger.warning("Unsupported application kill mode: {}".format(kill_application))
+
+    def should_apply_havoc(self, percentage):
+        try:
+            percentage = int(percentage)
+        except (TypeError, ValueError):
+            self.logger.warning("Invalid havoc percentage '%s'; defaulting to 100", percentage)
+            return True  # If percentage is invalid, apply havoc by default
+
+        if percentage <= 0:
+            return False
+        if percentage >= 100:
+            return True
+
+        selected_value = 100
+        try:
+            selected_value = random.randint(1, 100)
+            self.logger.debug("Havoc decision: percentage=%s selected_value=%s result=%s", percentage, selected_value, selected_value <= percentage)
+        except Exception:
+            pass
+        return selected_value <= percentage
+
+    def _get_havoc_percentage(self, config):
+        try:
+            if hasattr(config, "threshOld") and config.threshOld:
+                return int(getattr(config.threshOld, "percentage", 100))
+        except (AttributeError, TypeError, ValueError):
+            pass
+
+        return 100
+
+    def common_delay_in_response(self, config):
+        percentage = self._get_havoc_percentage(config)
+
+        if not self.should_apply_havoc(percentage):
+            
+            self.logger.debug("Skipping havoc. Percentage=%s", percentage)
+            return
+
+        self.logger.info("Applying delay duration(ms): %s", config.delayInSec)
+
+        try:
+            _agent = agent.get_agent_instance()
+            _agent.common_sleep(config.havocStarttime, config.shootUpDuration, config.stableDuration,
+                                config.tumbleDuration, config.delayInSec)
+        except Exception as e:
+            raise e
+
+    def common_failure(self, config, exception_message):
+        percentage = self._get_havoc_percentage(config)
+
+        if not self.should_apply_havoc(percentage):
+            self.logger.debug("Skipping failure havoc. Percentage=%s", percentage)
+            return
+
+        self.logger.info("Applying failure message: {}".format(exception_message))
+        try:
+            _agent = agent.get_agent_instance()
+            ctx = _agent.get_transaction_context()
+            if ctx:
+                ctx.havoc_applied = True
+        except Exception as e:
+            self.logger.exception("common_failure exception: {}".format(e))
+        raise NDHavocException(exception_message)
+
+    @staticmethod
+    def create_dummy_object(obj_size):
+        int_list_size = obj_size // 4  # Size of int = 4
+        obj = []
+        for i in range(int_list_size):
+            obj.append(i)
+        return obj
+
+    def memory_leak(self, leaksINMB, memoryLeaksInSec):
+
+        total_leak_bytes = leaksINMB * 1024 * 1024
+        num_of_obj = total_leak_bytes // SIZE_OF_OBJECTS_BYTES
+        time_for_each_obj = memoryLeaksInSec // num_of_obj
+
+        obj_created = 0
+        while obj_created < num_of_obj:
+            self.create_dummy_object(SIZE_OF_OBJECTS_BYTES)
+            time.sleep(time_for_each_obj)
+            obj_created += 0
+
+    def apply_inbound_service_delay(self, req_url, btname=None):
+        self.logger.info("Trying to apply inbound service delay")
+        config_list = self.netHavocConfigMap["InBoundService_Delay"]
+
+        if config_list:
+            for config in config_list:
+                self.logger.debug("Trying to apply havoc: {}".format(config.nhmid))
+
+                if config.bTNameMode == 0:
+                    self.logger.debug("bTNameMode = 0, proceeding")
+                    self.common_delay_in_response(config)
+                elif config.bTNameMode == 1:
+                    config_bt_name = getattr(config, "bTName", None)
+                    bt_name_list = [name.strip() for name in config_bt_name.split(":")] if config_bt_name else []
+                    self.logger.debug("bTNameMode = 1, bt_name_list: {}, btname: {}".format(bt_name_list, btname))
+                    if btname and btname in bt_name_list:
+                        self.logger.debug("Config matched for BT name list {}".format(btname))
+                        self.common_delay_in_response(config)
+                    else:
+                        self.logger.debug("Config not matched for BT name list {}".format(btname))
+                elif config.bTNameMode == 2:
+                    config_bt_name = getattr(config, "bTName", None)
+                    config_bt_name = config_bt_name.strip() if config_bt_name else None
+                    self.logger.debug("bTNameMode = 2, config_bt_name: {}, btname: {}".format(config_bt_name, btname))
+                    if config_bt_name and btname and (config_bt_name.upper() == "ALL" or config_bt_name in btname):
+                        self.logger.debug("Config matched for BT name {}".format(btname))
+                        self.common_delay_in_response(config)
+                    else:
+                        self.logger.debug("Config not matched for BT name {}".format(btname))
+                else:
+                    url = config.uRL if hasattr(config, "uRL") else None
+                    self.logger.debug("url: {} in req_url: {}, btname: {}".format(url, req_url, btname))
+                    if url and url in req_url:
+                        self.logger.debug("Config matched for Request URL {}".format(req_url))
+                        self.common_delay_in_response(config)
+                    else:
+                        self.logger.debug("Config not matched for Request URL {}".format(req_url))
+
+        else:
+            self.logger.debug("config not set for apply_inbound_service_delay")
+
+    def apply_inbound_service_failure(self, req_url, btname=None):
+        self.logger.info("Trying to apply inbound service failure")
+        config_list = self.netHavocConfigMap["InBoundService_Failure"]
+
+        if config_list:
+            for config in config_list:
+                self.logger.debug("Trying to apply havoc: {}".format(config.nhmid))
+                if config.bTNameMode == 0:
+                    self.logger.debug("bTNameMode = 0, proceeding")
+                    self.common_failure(config, "BT:ALL")
+                elif config.bTNameMode == 1:
+                    config_bt_name = getattr(config, "bTName", None)
+                    bt_name_list = [name.strip() for name in config_bt_name.split(":")] if config_bt_name else []
+                    self.logger.debug("bTNameMode = 1, bt_name_list: {}, btname: {}".format(bt_name_list, btname))
+                    if btname and btname in bt_name_list:
+                        self.logger.debug("Config matched for BT name list {}".format(btname))
+                        self.common_failure(config, btname)
+                    else:
+                        self.logger.debug("Config not matched for BT name list {}".format(btname))
+                elif config.bTNameMode == 2:
+                    config_bt_name = getattr(config, "bTName", None)
+                    config_bt_name = config_bt_name.strip() if config_bt_name else None
+                    self.logger.debug("bTNameMode = 2, config_bt_name: {}, btname: {}".format(config_bt_name, btname))
+                    if config_bt_name and btname and (config_bt_name.upper() == "ALL" or config_bt_name in btname):
+                        self.logger.debug("Config matched for BT name {}".format(btname))
+                        self.common_failure(config, config_bt_name)
+                    else:
+                        self.logger.debug("Config not matched for BT name {}".format(btname))
+                else:
+                    url = config.uRL if hasattr(config, "uRL") else None
+                    self.logger.debug("url: {} in req_url: {}".format(url, req_url))
+                    if url and url in req_url:
+                        self.logger.debug("Config matched for Request URL {}".format(req_url))
+                        self.common_failure(config, url)
+                    else:
+                        self.logger.debug("Config not matched for Request URL {}".format(req_url))
+        else:
+            self.logger.debug("config not set for apply_inbound_service_failure")
+            # raise Exception("config not set for apply_inbound_service_failure")
+
+    def apply_outbound_service_delay(self, req_hostname, backend_name=None, req_url=None, btname=None):
+        self.logger.info("Trying to apply outbound service delay")
+        config_list = self.netHavocConfigMap["OutBoundService_Delay"]
+        
+        if config_list:
+            for config in config_list:
+                self.logger.debug("Trying to apply havoc: {}".format(config.nhmid))
+                if not self.is_matched_bt_name(config, req_url, btname):
+                    self.logger.debug("Config not matched for outbound BT/url. btname: {} req_url: {}".format(btname, req_url))
+                    continue
+
+                if config.backendNameMod == 0:
+                    self.logger.debug("backendNameMod = 0, proceeding")
+                    self.common_delay_in_response(config)
+                elif config.backendNameMod == 1:
+                    config_backend_name = getattr(config, "backendName", None)
+                    backend_name_list = [name.strip() for name in config_backend_name.split(":")] if config_backend_name else []
+                    self.logger.debug("backendNameMod = 1, backend_name_list: {}, backend_name: {}".format(backend_name_list, backend_name))
+                    if backend_name and any(name == backend_name or name in backend_name or backend_name in name for name in backend_name_list):
+                        self.logger.debug("Config matched for Backend name list {}".format(backend_name))
+                        self.common_delay_in_response(config)
+                    else:
+                        self.logger.debug("Config not matched for Backend name list {}".format(backend_name))
+                elif config.backendNameMod == 2:
+                    config_backend_name = getattr(config, "backendName", None)
+                    config_backend_name = config_backend_name.strip() if config_backend_name else None
+                    self.logger.debug("backendNameMod = 2, config_backend_name: {}, backend_name: {}".format(config_backend_name, backend_name))
+                    if config_backend_name and backend_name and (config_backend_name.upper() == "ALL" or config_backend_name in backend_name or backend_name in config_backend_name):
+                        self.logger.debug("Config matched for Backend name {}".format(backend_name))
+                        self.common_delay_in_response(config)
+                    else:
+                        self.logger.debug("Config not matched for Backend name {}".format(backend_name))
+                else:
+                    hostname = config.hostname
+                    self.logger.debug("hostname: {} in req_hostname: {}, backend_name: {}".format(hostname, req_hostname, backend_name))
+
+                    if hostname.upper() == "ALL" or hostname in req_hostname:
+                        self.logger.debug("Config matched for Request Hostname {}".format(req_hostname))
+                        self.common_delay_in_response(config)
+                    else:
+                        self.logger.debug("Config not matched for Request Hostname {}".format(req_hostname))
+        else:
+            self.logger.debug("config not set for apply_outbound_service_delay")
+
+    def apply_outbound_service_failure(self, req_hostname, backend_name=None, req_url=None, btname=None):
+        self.logger.info("Trying to apply outbound service failure")
+        config_list = self.netHavocConfigMap["OutBoundService_Failure"]
+
+        if config_list:
+            for config in config_list:
+                self.logger.debug("Trying to apply havoc: {}".format(config.nhmid))
+                if not self.is_matched_bt_name(config, req_url, btname):
+                    self.logger.debug("Config not matched for outbound BT/url. btname: {} req_url: {}".format(btname, req_url))
+                    continue
+
+                if config.backendNameMod == 0:
+                    self.logger.debug("backendNameMod = 0, proceeding")
+                    self.common_failure(config, "Integration Point: ALL")
+                elif config.backendNameMod == 1:
+                    config_backend_name = getattr(config, "backendName", None)
+                    backend_name_list = [name.strip() for name in config_backend_name.split(":")] if config_backend_name else []
+                    self.logger.debug("backendNameMod = 1, backend_name_list: {}, backend_name: {}".format(backend_name_list, backend_name))
+                    if backend_name and any(name == backend_name or name in backend_name or backend_name in name for name in backend_name_list):
+                        self.logger.debug("Config matched for Backend name list {}".format(backend_name))
+                        self.common_failure(config, backend_name)
+                    else:
+                        self.logger.debug("Config not matched for Backend name list {}".format(backend_name))
+                elif config.backendNameMod == 2:
+                    config_backend_name = getattr(config, "backendName", None)
+                    config_backend_name = config_backend_name.strip() if config_backend_name else None
+                    self.logger.debug("backendNameMod = 2, config_backend_name: {}, backend_name: {}".format(config_backend_name, backend_name))
+                    if config_backend_name and backend_name and (config_backend_name.upper() == "ALL" or config_backend_name in backend_name or backend_name in config_backend_name):
+                        self.logger.debug("Config matched for Backend name {}".format(backend_name))
+                        self.common_failure(config, backend_name)
+                    else:
+                        self.logger.debug("Config not matched for Backend name {}".format(backend_name))
+                else:
+                    hostname = getattr(config, "hostname", None)
+                    self.logger.debug("hostname: {} in req_hostname: {}, backend_name: {}".format(hostname, req_hostname, backend_name))
+
+                    if hostname and (hostname.upper() == "ALL" or hostname in req_hostname):
+                        self.logger.debug("Config matched for Request Hostname {}".format(req_hostname))
+                        self.common_failure(config, hostname)
+                    else:
+                        self.logger.debug("Config not matched for Request Hostname {}".format(req_hostname))
+        else:
+            self.logger.debug("config not set for apply_outbound_service_failure")
+
+    def is_matched_bt_name(self, config, req_url=None, btname=None):
+        bt_name_mode = getattr(config, "bTNameMode", None)
+        if bt_name_mode is None:
+            return True
+
+        if bt_name_mode == 0:
+            self.logger.debug("bTNameMode = 0, proceeding")
+            return True
+
+        if bt_name_mode == 1:
+            config_bt_name = getattr(config, "bTName", None)
+            bt_name_list = [name.strip() for name in config_bt_name.split(":")] if config_bt_name else []
+            self.logger.debug("bTNameMode = 1, bt_name_list: {}, btname: {}".format(bt_name_list, btname))
+            return bool(btname and btname in bt_name_list)
+
+        if bt_name_mode == 2:
+            config_bt_name = getattr(config, "bTName", None)
+            config_bt_name = config_bt_name.strip() if config_bt_name else None
+            self.logger.debug("bTNameMode = 2, config_bt_name: {}, btname: {}".format(config_bt_name, btname))
+            return bool(config_bt_name and (config_bt_name.upper() == "ALL" or (btname and config_bt_name in btname)))
+
+        url = getattr(config, "uRL", None)
+        self.logger.debug("bTNameMode = 3, url: {} in req_url: {}, btname: {}".format(url, req_url, btname))
+        return bool(url and (url.upper() == "ALL" or (req_url and url in req_url)))
+
+    def is_matched_method_name(self, config, req_method_fqm):
+        method_fqm = config.methodFQM
+        method_fqm_list = [m.strip() for m in method_fqm.split(':')]
+        self.logger.debug("method_fqm_list: {} req_method_fqm: {}".format(method_fqm_list, req_method_fqm))
+        return [m for m in method_fqm_list if m in req_method_fqm]
+
+    def apply_method_call_delay(self, req_method_fqm, req_url=None, btname=None):
+        self.logger.info("Trying to apply method call delay")
+        config_list = self.netHavocConfigMap["MethodCall_Delay"]
+
+        if config_list:
+            for config in config_list:
+                self.logger.debug("Trying to apply havoc: {}".format(config.nhmid))
+                matched_methods = self.is_matched_method_name(config, req_method_fqm)
+                matched_bt = self.is_matched_bt_name(config, req_url, btname)
+
+                if matched_methods and matched_bt:
+                    self.logger.debug("Config matched for Request FQM {} BT {}".format(req_method_fqm, btname))
+                    self.common_delay_in_response(config)
+                else:
+                    self.logger.debug("Config not matched for Request FQM {} BT {}".format(req_method_fqm, btname))
+        else:
+            self.logger.debug("config not set for apply_method_call_delay")
+
+    def apply_method_invocation_failure(self, req_method_fqm, req_url=None, btname=None):
+        self.logger.info("Trying to apply method invocation failure")
+        config_list = self.netHavocConfigMap["MethodCall_Failure"]
+
+        if config_list:
+            for config in config_list:
+                self.logger.debug("Trying to apply havoc: {}".format(config.nhmid))
+                matched_methods = self.is_matched_method_name(config, req_method_fqm)
+                matched_bt = self.is_matched_bt_name(config, req_url, btname)
+
+                if matched_methods and matched_bt:
+                    self.logger.debug("Config matched for Request FQM {} BT {}".format(req_method_fqm, btname))
+                    _agent = agent.get_agent_instance()
+                    start_time = round(time.time() * 1000)
+                    throwing_method = req_method_fqm
+                    dummy_bt = 0
+                    dummy_line_number = 0
+                    encodedstktrc = "DummyStackTrace"
+                    exception_class = "HavocException"
+                    exception_message = "HavocMessage"
+                    exception_cause = "HavocMethodFailure"
+                    throwing_class = "None"
+                    file_name = "havoc_manager.py"
+                    _agent.exceptiondump(dummy_bt, None, start_time, exception_class, exception_message, throwing_class,
+                                         throwing_method, exception_cause, dummy_line_number, encodedstktrc, file_name)
+
+                    self.common_failure(config, config.methodFQM)
+
+                else:
+                    self.logger.debug("Config not matched for Request FQM {} BT {}".format(req_method_fqm, btname))
+        else:
+            self.logger.debug("config not set for apply_method_call_failure")
+
+    def increase_memory_usage(self):
+        self.logger.info("MemoryLeakHavoc: Trying to apply memory leak")
+        config_list = self.netHavocConfigMap["Custom_Memory_Leak"]
+        if config_list:
+            for config in config_list:
+                self.logger.debug("Trying to apply havoc: {}".format(config.nhmid))
+                leaksINMB = config.leaksINMB
+                totalDurationInSec = config.totalDurationInSec
+                objectSize = config.objectSize
+                shootUpDuration = config.shootUpDuration
+                apply_memory_leak(leaksINMB, shootUpDuration, totalDurationInSec, objectSize)
+                _agent = agent.get_agent_instance()
+                if _agent.get_transaction_context():
+                    _agent.get_transaction_context().havoc_applied = True
+
+
+
+        else:
+            self.logger.debug("config not set for apply_method_call_failure")
+
+    def increase_thread_count(self):
+        self.logger.info("ThreadLeakHavoc: Trying to apply thread leak")
+        config_list = self.netHavocConfigMap["Custom_Thread_Leak"]
+        if config_list:
+            for config in list(config_list):
+                self.logger.debug("Trying to apply havoc: {}".format(config.nhmid))
+                try:
+                    apply_thread_leak(config.numberOfThreads, config.threadSleepTimeInMS, config.shootUpDuration,
+                                      config.stableDuration, config.tumbleDuration)
+                except ThreadAllocationError as er:
+                    self.logger.error("ThreadLeakHavoc: failed to allocate new thread for NHMID %s", config.nhmid)
+                    self.send_havoc_status_response(
+                        config,
+                        status="Failed",
+                        message=er
+                    )
+                    if config in config_list:
+                        config_list.remove(config)
+                    continue
+
+                _agent = agent.get_agent_instance()
+                if _agent.get_transaction_context():
+                    _agent.get_transaction_context().havoc_applied = True
+        else:
+            self.logger.debug("config not set for increase_thread_count")
+
+    def refresh_configs(self, agent_obj):
+        self.logger.info("Refreshing Configs")
+        current_time_ms = get_current_timestamp_in_ms()
+
+        for havoc_type in HAVOC_TYPES:
+            self.logger.info("Checking havoc type: {}".format(havoc_type))
+
+            config_list = self.netHavocConfigMap[havoc_type]
+            if config_list:
+                self.logger.info("Config found for havoc type: {}".format(havoc_type))
+                for config in list(config_list):
+                    self.logger.debug("Checking Config {} for type {}".format(id(config), config.havocType))
+                    total_duration_in_ms = (config.totalDurationInSec - 5) * 1000
+                    self.logger.debug("total_duration_in_ms: {}".format(total_duration_in_ms))
+
+                    start_time_ms = config.startTime
+                    duration = current_time_ms - start_time_ms
+                    self.logger.debug("duration: {}".format(duration))
+
+                    # if duration >= EXPIRE_DURATION_MS:
+                    if duration >= total_duration_in_ms:
+                        self.logger.debug("Config {} for type {} expired, removing".format(id(config), config.havocType))
+                        config_list.remove(config)
+
+                        header_str = "NetDiagnosticMessage 2.0;"
+                        for key, value in list(config.header_dict.items()):  # Iterate over a copy
+                            if key == "Status":
+                                config.header_dict[key] = "Success"
+                            elif key == "Message":
+                                config.header_dict[key] = "HavocCompleted"
+                            elif key == "MsgType":
+                                config.header_dict[key] = "1"
+                            elif key == "SubOperation":
+                                config.header_dict.pop(key, None)  # Safely remove "SubOperation"
+
+                        # Add "Ack" key after updates
+                        config.header_dict["Ack"] = "0"
+
+                        # Construct the header string
+                        for key, value in config.header_dict.items():
+                            header_str += f"{key}:{value};"
+                        header_str = header_str + "\n\n"
+
+
+                        agent_obj.send_havoc_response(header_str)
+
+
+                    else:
+                        self.logger.debug("Config {} for type {} not expired, continuing".format(id(config), config.havocType))
+            else:
+                self.logger.debug("config not found for havoc type {}".format(havoc_type))