cavisson-pythonagent 0.0.1__py3-none-any.whl

pythonagent/agent/probes/grpc/server_interceptor.py

@@ -0,0 +1,129 @@
+"""Server-side gRPC interceptor.
+
+intercept_service() synchronously receives the RpcMethodHandler for the
+matched RPC and wraps its actual behavior callable directly - unlike Java's
+ServerCall.Listener (registration now, completion callback later), so the BT
+handle is captured once as a closure variable in intercept_service and
+threaded explicitly into the wrapped callable, rather than relying on
+ambient per-thread state. gRPC's own docs note interceptor and handler are
+not guaranteed to run on the same thread; passing bt explicitly through the
+closure makes this correct regardless (see DESIGN notes in the plan).
+"""
+
+import sys
+
+import grpc
+
+from pythonagent.agent.probes.base import EntryPointInterceptor
+from pythonagent.utils import get_current_thread_time_in_us
+
+
+class NDGrpcServerInterceptor(EntryPointInterceptor, grpc.ServerInterceptor):
+
+    def __init__(self, agent):
+        super(NDGrpcServerInterceptor, self).__init__(agent, None)
+
+    @staticmethod
+    def _extract_fp_instance(invocation_metadata):
+        if not invocation_metadata:
+            return None
+        for key, value in invocation_metadata:
+            if key and key.lower() == 'cavndfpinstance':
+                return value
+        return None
+
+    def _start(self, handler_call_details):
+        method = handler_call_details.method or ""
+        fp_instance = self._extract_fp_instance(getattr(handler_call_details, 'invocation_metadata', None))
+        cpu_time_before = get_current_thread_time_in_us()
+        bt = self.start_business_transaction(method, "", None, None, None, fp_instance)
+        return bt, cpu_time_before
+
+    def _finish(self, bt, cpu_time_before):
+        with self.log_exceptions():
+            cpu_time_after = get_current_thread_time_in_us()
+            self.end_business_transaction(bt, cpu_time_after - cpu_time_before)
+
+    @staticmethod
+    def _wrap_context_status(context):
+        """Capture the status the moment the handler sets it, rather than
+        reading it back afterward (ServicerContext.code() is EXPERIMENTAL).
+        Returns a callable that yields the captured status (defaulting to OK).
+        """
+        status_holder = {'code': grpc.StatusCode.OK}
+        real_set_code = context.set_code
+        real_abort = context.abort
+
+        def set_code(code, *args, **kwargs):
+            status_holder['code'] = code
+            return real_set_code(code, *args, **kwargs)
+
+        def abort(code, details=None, *args, **kwargs):
+            status_holder['code'] = code
+            return real_abort(code, details, *args, **kwargs)
+
+        context.set_code = set_code
+        context.abort = abort
+
+        def get_status_code():
+            code = status_holder['code']
+            return 200 if (code is None or code == grpc.StatusCode.OK) else 500
+
+        return get_status_code
+
+    def _handle_exception(self, bt):
+        with self.log_exceptions():
+            if bt:
+                bt.add_exception(*sys.exc_info())
+
+    def intercept_service(self, continuation, handler_call_details):
+        handler = continuation(handler_call_details)
+        if handler is None:
+            return handler
+
+        if handler.request_streaming and handler.response_streaming:
+            behavior = self._wrap_streaming(handler.stream_stream, handler_call_details)
+            return grpc.stream_stream_rpc_method_handler(
+                behavior, handler.request_deserializer, handler.response_serializer)
+        if handler.request_streaming and not handler.response_streaming:
+            behavior = self._wrap_unary(handler.stream_unary, handler_call_details)
+            return grpc.stream_unary_rpc_method_handler(
+                behavior, handler.request_deserializer, handler.response_serializer)
+        if not handler.request_streaming and handler.response_streaming:
+            behavior = self._wrap_streaming(handler.unary_stream, handler_call_details)
+            return grpc.unary_stream_rpc_method_handler(
+                behavior, handler.request_deserializer, handler.response_serializer)
+
+        behavior = self._wrap_unary(handler.unary_unary, handler_call_details)
+        return grpc.unary_unary_rpc_method_handler(
+            behavior, handler.request_deserializer, handler.response_serializer)
+
+    def _wrap_unary(self, real_behavior, handler_call_details):
+        def wrapped(request_or_iterator, context):
+            bt, cpu_time_before = self._start(handler_call_details)
+            get_status_code = self._wrap_context_status(context)
+            try:
+                response = real_behavior(request_or_iterator, context)
+            except Exception:
+                self._handle_exception(bt)
+                self._finish(bt, cpu_time_before)
+                raise
+            self.handle_http_status_code(bt, get_status_code(), "")
+            self._finish(bt, cpu_time_before)
+            return response
+        return wrapped
+
+    def _wrap_streaming(self, real_behavior, handler_call_details):
+        def wrapped(request_or_iterator, context):
+            bt, cpu_time_before = self._start(handler_call_details)
+            get_status_code = self._wrap_context_status(context)
+            try:
+                for response in real_behavior(request_or_iterator, context):
+                    yield response
+            except Exception:
+                self._handle_exception(bt)
+                self._finish(bt, cpu_time_before)
+                raise
+            self.handle_http_status_code(bt, get_status_code(), "")
+            self._finish(bt, cpu_time_before)
+        return wrapped

pythonagent/agent/probes/havoc/custom_memory_stress.py

@@ -0,0 +1,186 @@
+"""
+Allocates heap memory by creating lists of Python float objects,
+stressing the CPython heap allocator and cyclic garbage collector.
+
+Why floats and not bytearray?
+  • bytearray(n)          → single C malloc, NOT tracked by the GC, zero heap object pressure
+  • [i+0.1 for i in ...]  → n individual PyFloatObject allocations, each ~24 bytes,
+                            all tracked by the cyclic GC — true heap pressure
+  • array.array('d', ...) → like bytearray: one C buffer, no GC objects
+
+Sizing math:
+  Each PyFloatObject = 24 bytes on 64-bit CPython (ob_refcnt + ob_type + ob_fval)
+  Plus list slot pointer = 8 bytes  →  ~32 bytes effective cost per element
+  count = ceil(objectSizeInBytes / 32)
+"""
+
+import time
+import math
+import logging
+import gc
+import threading
+from pythonagent.agent.internal.proc_compat import get_virtual_memory, get_free_memory_mb
+
+logger = logging.getLogger('pythonagent.agent')
+
+MEGA = 2 ** 20
+GIGA = 2 ** 30
+
+# CPython PyFloatObject size on 64-bit: ob_refcnt(8) + ob_type*(8) + ob_fval(8) = 24 bytes
+# list slot (pointer to object) adds 8 bytes → ~32 bytes effective per element
+_FLOAT_OBJECT_BYTES = 32
+
+
+# Module-level so a stop request (from havoc_manager.parse_nethavoc_config) can
+# reach a memory-leak havoc that's currently running in its own background thread.
+_stop_event = threading.Event()
+
+
+def stop_memory_leak():
+    """Signal any running memory-leak havoc to stop immediately."""
+    logger.info("MemoryLeakHavoc: stop_memory_leak called")
+    _stop_event.set()
+
+
+class MemoryLeakObject:
+    """
+    Allocates a list of Python floats to consume approximately
+    `object_size_in_bytes` of heap memory via PyFloatObject instances.
+
+    i + 0.1 is used to:
+      1. Prevent CPython's small-int cache from de-duplicating values
+      2. Ensure every element is a distinct PyFloatObject on the heap
+         (integers < 256 are cached singletons; floats never are)
+    """
+
+    __slots__ = ('numbers',)
+
+    def __init__(self, object_size_in_bytes: int):
+        count = max(1, math.ceil(object_size_in_bytes / _FLOAT_OBJECT_BYTES))
+        # Each `i + 0.1` produces a unique PyFloatObject allocated on the heap.
+        # Python floats have NO intern/cache pool — every one is a fresh
+        # allocation, forcing pymalloc to work.
+        self.numbers: list = [i + 0.1 for i in range(count)]
+
+    def clear(self) -> None:
+        """
+        Release the reference to the float list.
+        Once all MemoryLeakObject references are dropped and GC runs,
+        the memory becomes eligible for reclamation.
+        """
+        self.numbers = None
+
+
+def apply_memory_leak(leaksINMB: float, shootUpDurationInSec: float,
+                      totalDurationInSec: float, objectSizeINKB: float) -> None:
+    """
+    Entry point — signature matches custom_memory_stress.apply_memory_leak
+    so havoc_manager.py needs zero changes.
+    """
+    try:
+        # Start the memory-leak havoc request with the requested pressure target.
+        logger.info("MemoryLeakHavoc: preparing leak request target=%.1fMB", leaksINMB)
+        free_mb = get_free_memory_mb()
+        if free_mb < leaksINMB:
+            logger.error(
+                "MemoryLeakHavoc: Memory havoc aborted: requested %.1fMB but only %.1fMB free",
+                leaksINMB, free_mb
+            )
+            return
+
+        # Reset any stop signal left over from a previous run so this run starts clean.
+        _stop_event.clear()
+
+        logger.warning(
+            "MemoryLeakHavoc: Heap Pressure Havoc starting: target=%sMB chunkSize=%sKB "
+            "shootUp=%ss total=%ss",
+            leaksINMB, objectSizeINKB, shootUpDurationInSec, totalDurationInSec
+        )
+        _run_heap_pressure(leaksINMB, shootUpDurationInSec, totalDurationInSec, objectSizeINKB)
+    except Exception:
+        logger.exception("MemoryLeakHavoc: Failed while applying memory leak havoc")
+
+
+def _run_heap_pressure(leaksINMB: float, shootUpDurationInSec: float,
+                       totalDurationInSec: float, objectSizeINKB: float) -> None:
+    """
+    Core ramp-up loop:
+
+      objectCount  = ceil(totalBytes / objectSizeBytes)
+      delayPerObj  = floor(rampUpMs / objectCount)   [seconds here]
+
+    All MemoryLeakObject instances are retained in `memory_leaks` so the GC
+    cannot reclaim them during the hold phase.
+    """
+    memory_leaks: list = []   # keeps all objects reachable during the hold phase
+
+    try:
+        _log_vm_stats("before allocation")
+
+        total_bytes      = max(1, int(leaksINMB * MEGA))
+        # Default 30 KB if objectSizeINKB <= 0
+        object_size_kb   = objectSizeINKB if objectSizeINKB > 0 else 30.0
+        object_size_bytes = max(1024, int(object_size_kb * 1024))
+
+        object_count     = max(1, math.ceil(total_bytes / object_size_bytes))
+        
+        delay_per_object = (shootUpDurationInSec / object_count) if shootUpDurationInSec > 0 else 0.0
+
+        logger.info(
+            "MemoryLeakHavoc: Heap pressure plan | targetBytes=%d | objectSizeBytes=%d | "
+            "objectCount=%d | delayPerObject=%.4fs",
+            total_bytes, object_size_bytes, object_count, delay_per_object
+        )
+
+        allocated_bytes = 0
+        for i in range(object_count):
+            if _stop_event.is_set():
+                logger.info("MemoryLeakHavoc: stop signal received during allocation, stopping after %d/%d objects", i, object_count)
+                break
+            try:
+                obj = MemoryLeakObject(object_size_bytes)
+                memory_leaks.append(obj)           # retain reference — keeps object reachable
+                allocated_bytes += object_size_bytes
+            except MemoryError:
+                logger.error(
+                    "MemoryError at object %d/%d after %d bytes — stopping ramp",
+                    i + 1, object_count, allocated_bytes
+                )
+                break
+
+            if delay_per_object > 0:
+                if _stop_event.wait(timeout=delay_per_object):
+                    logger.info("MemoryLeakHavoc: stop signal received during ramp-up delay")
+                    break
+
+        _log_vm_stats("after allocation (%.1fMB held)" % (allocated_bytes / MEGA))
+        logger.info("MemoryLeakHavoc: Heap pressure allocation complete — %d objects holding ~%d bytes",
+                    len(memory_leaks), allocated_bytes)
+
+        hold_secs = max(0.0, totalDurationInSec - shootUpDurationInSec)
+        logger.info("MemoryLeakHavoc: Holding heap pressure for %.1fs", hold_secs)
+        if hold_secs > 0:
+            if _stop_event.wait(timeout=hold_secs):
+                logger.info("MemoryLeakHavoc: stop signal received during hold phase")
+
+    except Exception:
+        logger.exception("MemoryLeakHavoc: Unexpected error in heap pressure havoc")
+    finally:
+        # Drop all references → objects become GC-eligible
+        for obj in memory_leaks:
+            obj.clear()
+        memory_leaks.clear()
+        gc.collect()   # nudge GC to reclaim PyFloatObjects immediately after hold
+        _log_vm_stats("after release")
+        logger.warning("MemoryLeakHavoc: Heap Pressure Havoc complete")
+
+
+def _log_vm_stats(label: str = "") -> None:
+    try:
+        tot, avail, percent, used, free = get_virtual_memory()
+        logger.warning(
+            "MemoryLeakHavoc: VM [%s]: total=%.2fGB used=%.2fGB avail=%.2fGB free=%.2fGB percent=%.1f%%",
+            label, tot / GIGA, used / GIGA, avail / GIGA, free / GIGA, percent
+        )
+    except Exception:
+        logger.exception("MemoryLeakHavoc: Could not read VM stats")

pythonagent/agent/probes/havoc/custom_thread_stress.py

@@ -0,0 +1,187 @@
+import logging
+import threading
+import time
+
+logger = logging.getLogger('pythonagent.agent')
+
+
+class ThreadAllocationError(RuntimeError):
+    """Raised when a havoc worker thread cannot be allocated."""
+
+
+# Module-level so a stop request (from havoc_manager.parse_nethavoc_config) can
+# reach a thread-leak havoc that's currently running in its own daemon thread.
+_stop_event = threading.Event()
+
+
+def stop_thread_leak():
+    """Signal any running thread-leak havoc to stop immediately."""
+    logger.info("ThreadLeakHavoc: stop_thread_leak called")
+    _stop_event.set()
+
+
+def monitored_sleep(duration_sec):
+    """Sleep for duration_sec while monitoring actual elapsed time.
+
+    This helper detects early wake-up from the sleep call and logs the
+    requested duration vs actual elapsed time.
+    """
+    start = time.monotonic()
+    try:
+        time.sleep(duration_sec)
+    finally:
+        end = time.monotonic()
+        elapsed = end - start
+        logger.debug("ThreadLeakHavoc: Requested sleep: %.3fs, elapsed: %.3fs", duration_sec, elapsed)
+        if elapsed < duration_sec:
+            logger.warning("ThreadLeakHavoc: Sleep returned early: requested=%.3fs elapsed=%.3fs", duration_sec, elapsed)
+        else:
+            logger.debug("ThreadLeakHavoc: Sleep completed or overslept: requested=%.3fs elapsed=%.3fs", duration_sec, elapsed)
+        return elapsed
+
+
+def apply_thread_leak(numberOfThreads, threadSleepTimeInMS, shootUpDuration, stableDuration, tumbleDuration):
+    """Apply thread-leak havoc with ramp-up, stable, and ramp-down phases."""
+
+    def thread_sleep_function(stop_event):
+        # Worker thread waits until the stop event is set.
+        while not stop_event.is_set() and not _stop_event.is_set():
+            stop_event.wait(thread_sleep_time_sec)
+
+    def start_havoc_thread():
+        # Create and start a new daemon thread that will sleep repeatedly.
+        stop_event = threading.Event()
+        thread_name = "TLHavoc{}".format(len(active_threads))
+        thread = threading.Thread(target=thread_sleep_function, args=(stop_event,),
+                                  name=thread_name, daemon=True)
+        try:
+            thread.start()
+        except RuntimeError as exc:
+            stop_event.set()
+            logger.exception("ThreadLeakHavoc: failed to start a worker thread: %s", exc)
+            raise ThreadAllocationError("new thread could not be allocated") from exc
+        stop_events.append(stop_event)
+        active_threads.append(thread)
+
+    def stop_havoc_threads(count):
+        # Signal `count` threads to stop and remove them from tracking lists.
+        for _ in range(min(count, len(stop_events))):
+            stop_event = stop_events.pop(0)
+            thread = active_threads.pop(0)
+            stop_event.set()
+            if thread.is_alive():
+                thread.join(timeout=1.0)
+
+    try:
+        # Track stop events and running threads for this thread-leak instance.
+        stop_events = []
+        active_threads = []
+
+        # Reset any stop signal left over from a previous run so this run starts clean.
+        _stop_event.clear()
+
+        # Begin the thread-leak havoc setup with the requested parameters.
+        logger.info("ThreadLeakHavoc: starting thread leak setup requested=%d sleep_ms=%s", numberOfThreads, threadSleepTimeInMS)
+
+        # Normalize inputs and convert sleep duration to seconds.
+        thread_sleep_time_sec = max(threadSleepTimeInMS, 1) / 1000.0
+        shootUpDuration = max(shootUpDuration, 0)
+        tumbleDuration = max(tumbleDuration, 0)
+        stableDuration = max(stableDuration, 0)
+
+        logger.info("ThreadLeakHavoc: apply_thread_leak called: requested=%d, sleep_ms=%s, shootUp=%d, stable=%d, tumble=%d", numberOfThreads, threadSleepTimeInMS, shootUpDuration, stableDuration, tumbleDuration)
+
+        if numberOfThreads == 0:
+            logger.info("ThreadLeakHavoc: Thread leak requested with zero threads")
+            return
+
+        # Ramp-up phase: gradually create the requested number of worker threads.
+        startTime = time.monotonic()
+        for i in range(shootUpDuration):
+            if _stop_event.is_set():
+                logger.info("ThreadLeakHavoc: stop signal received during ramp-up")
+                stop_havoc_threads(len(active_threads))
+                return
+
+            threads_per_second = max(1, int(round((numberOfThreads - len(active_threads)) / max(1, shootUpDuration - i))))
+            logger.debug("ThreadLeakHavoc: ramp-up target per second=%d", threads_per_second)
+            stime = time.monotonic()
+            for _ in range(threads_per_second):
+                try:
+                    start_havoc_thread()
+                except ThreadAllocationError:
+                    logger.error("ThreadLeakHavoc: thread creation failed before full allocation; successfully created=%d", len(active_threads))
+                    er= "Havoc failed because new thread could not be allocated after " + str(len(active_threads))
+                    stop_havoc_threads(len(active_threads))
+                    raise ThreadAllocationError(er)
+
+                if len(active_threads) >= numberOfThreads:
+                    break
+
+            logger.debug("ThreadLeakHavoc: Thread in this Second=%d, in time=%.3f", threads_per_second, time.monotonic() - stime)
+            elapsed = time.monotonic() - stime
+            if elapsed < 1.0:
+                time.sleep(1.0 - elapsed)
+            else:
+                logger.warning("ThreadLeakHavoc: ramp-up loop exceeded its 1s budget; continuing immediately")
+
+        while len(active_threads) < numberOfThreads:
+            if _stop_event.is_set():
+                logger.info("ThreadLeakHavoc: stop signal received before full allocation")
+                stop_havoc_threads(len(active_threads))
+                return
+            try:
+                start_havoc_thread()
+            except ThreadAllocationError:
+                logger.error("ThreadLeakHavoc: thread creation failed before full allocation; successfully created=%d", len(active_threads))
+                er= "Havoc failed because new thread could not be allocated after " + str(len(active_threads))
+                stop_havoc_threads(len(active_threads))
+                raise ThreadAllocationError(er)
+
+        logger.info("ThreadLeakHavoc: Thread leak allocation completed active_threads: %d, in time: %.3f", len(active_threads), time.monotonic() - startTime)
+
+        # Stable phase: hold thread count steady for the configured duration,
+        # checking every second so a stop request cuts it short instead of
+        # blocking for the full stableDuration.
+        elapsed = 0
+        while elapsed < stableDuration:
+            if _stop_event.is_set():
+                logger.info("ThreadLeakHavoc: stop signal received during stable phase")
+                stop_havoc_threads(len(active_threads))
+                return
+            monitored_sleep(1)
+            elapsed += 1
+
+        # Ramp-down phase: gradually stop threads over the tumble duration.
+        logger.info("ThreadLeakHavoc: entering ramp-down phase")
+        startTime = time.monotonic()
+        for i in range(tumbleDuration):
+            if _stop_event.is_set():
+                logger.info("ThreadLeakHavoc: stop signal received during ramp-down")
+                break
+            logger.info("ThreadLeakHavoc: Starting ramp-down: tumbleDuration=%d, active_threads=%d", tumbleDuration, len(active_threads))
+            threads_per_second = max(1, int(round(len(active_threads) / max(1, tumbleDuration - i))))
+            stime = time.monotonic()
+            stop_havoc_threads(threads_per_second)
+            elapsed = time.monotonic() - stime
+            if elapsed < 1.0:
+                time.sleep(1.0 - elapsed)
+            else:
+                logger.warning("ThreadLeakHavoc: ramp-down loop exceeded its 1s budget; continuing immediately")
+
+        if len(active_threads):
+            logger.info("ThreadLeakHavoc: Stopping active threads without rampdown")
+            stop_havoc_threads(len(active_threads))
+
+        logger.info("ThreadLeakHavoc: Thread leak completed: requested=%d, remaining_active=%d, time_taken=%f", numberOfThreads, len(active_threads), time.monotonic() - startTime)
+    except ThreadAllocationError:
+        er= "Havoc failed because new thread could not be allocated after " + str(len(active_threads))
+        logger.exception("ThreadLeakHavoc: Thread allocation failed after creating=%d threads, propagating to manager", len(active_threads))
+        stop_havoc_threads(len(active_threads))
+        raise ThreadAllocationError(er)
+    except Exception:
+        logger.exception("ThreadLeakHavoc: Thread allocation failed after creating=%d threads, propagating to manager", len(active_threads))
+        stop_havoc_threads(len(active_threads))
+    finally:
+        if len(active_threads):
+            stop_havoc_threads(len(active_threads))