hyperprobe-agent 1.2.24__py3-none-any.whl

hyperprobe/core/probe_output.py

@@ -0,0 +1,177 @@
+import queue
+import sys
+import threading
+from datetime import datetime, timezone
+
+
+_COLOR_RESET = "\x1b[0m"
+_COLOR_BOLD = "\x1b[1m"
+_COLOR_GRAY = "\x1b[90m"
+_COLOR_MAGENTA = "\x1b[35m"
+_LEVEL_COLORS = {
+    "ERROR": "\x1b[31m",
+    "INFO": "\x1b[32m",
+}
+_STOP = object()
+
+
+def normalize_probe_log_level(value):
+    candidate = value.strip().upper() if isinstance(value, str) else ""
+    return candidate if candidate in _LEVEL_COLORS else "INFO"
+
+
+def sanitize_probe_log_message(value):
+    message = value if isinstance(value, str) else str(value)
+    sanitized = []
+    escapes = {
+        "\n": r"\n",
+        "\r": r"\r",
+        "\t": r"\t",
+    }
+
+    for character in message:
+        if character in escapes:
+            sanitized.append(escapes[character])
+        elif character.isprintable():
+            sanitized.append(character)
+        else:
+            codepoint = ord(character)
+            if codepoint <= 0xFF:
+                sanitized.append(f"\\x{codepoint:02x}")
+            elif codepoint <= 0xFFFF:
+                sanitized.append(f"\\u{codepoint:04x}")
+            else:
+                sanitized.append(f"\\U{codepoint:08x}")
+
+    return "".join(sanitized)
+
+
+def format_probe_log(timestamp_ms, level, message, use_color=False):
+    normalized_level = normalize_probe_log_level(level)
+    timestamp = datetime.fromtimestamp(
+        timestamp_ms / 1000,
+        tz=timezone.utc,
+    ).isoformat(timespec="milliseconds").replace("+00:00", "Z")
+    safe_message = sanitize_probe_log_message(message)
+
+    if not use_color:
+        return f"[{timestamp}] [{normalized_level}] [HyperProbe LOG] {safe_message}\n"
+
+    level_color = _LEVEL_COLORS[normalized_level]
+    return (
+        f"{_COLOR_GRAY}[{timestamp}]{_COLOR_RESET} "
+        f"{_COLOR_BOLD}{level_color}[{normalized_level}]{_COLOR_RESET} "
+        f"{_COLOR_BOLD}{_COLOR_MAGENTA}[HyperProbe LOG]{_COLOR_RESET} "
+        f"{safe_message}\n"
+    )
+
+
+class ProbeLogWriter:
+    """Best-effort probe output that never blocks capture threads."""
+
+    def __init__(self, max_queue_size, stdout=None, stderr=None):
+        self._queue = queue.Queue(maxsize=max_queue_size)
+        self._stdout = stdout
+        self._stderr = stderr
+        self._lifecycle_lock = threading.RLock()
+        self._thread = None
+        self._closed = False
+        self.dropped_count = 0
+        self.error_count = 0
+
+    def start(self):
+        with self._lifecycle_lock:
+            return self._start_locked()
+
+    def _start_locked(self):
+        if self._closed:
+            return False
+        if self._thread and self._thread.is_alive():
+            return True
+        self._thread = threading.Thread(
+            target=self._run,
+            name="hyperprobe-probe-output",
+            daemon=True,
+        )
+        self._thread.start()
+        return True
+
+    def is_running(self):
+        with self._lifecycle_lock:
+            return bool(self._thread and self._thread.is_alive())
+
+    def submit(self, timestamp_ms, level, message):
+        record = (timestamp_ms, normalize_probe_log_level(level), message)
+        with self._lifecycle_lock:
+            if self._closed:
+                return False
+            try:
+                self._queue.put_nowait(record)
+                self._start_locked()
+                return True
+            except queue.Full:
+                self.dropped_count += 1
+                return False
+            except Exception:
+                self.error_count += 1
+                return False
+
+    def stop(self, timeout=2.0):
+        with self._lifecycle_lock:
+            self._closed = True
+            thread = self._thread
+            if thread:
+                try:
+                    self._queue.put_nowait(_STOP)
+                except queue.Full:
+                    pass
+
+        if thread and thread is not threading.current_thread():
+            thread.join(timeout=timeout)
+
+    def _run(self):
+        while True:
+            try:
+                record = self._queue.get(timeout=0.1)
+            except queue.Empty:
+                if self._retire_if_empty():
+                    return
+                continue
+
+            try:
+                if record is not _STOP:
+                    self._write(record)
+            finally:
+                self._queue.task_done()
+
+            if self._retire_if_empty():
+                return
+
+    def _retire_if_empty(self):
+        with self._lifecycle_lock:
+            if not self._queue.empty():
+                return False
+            if self._thread is threading.current_thread():
+                self._thread = None
+            return True
+
+    def _write(self, record):
+        timestamp_ms, level, message = record
+        stream = (
+            self._stderr if self._stderr is not None else sys.stderr
+        ) if level == "ERROR" else (
+            self._stdout if self._stdout is not None else sys.stdout
+        )
+
+        try:
+            use_color = bool(getattr(stream, "isatty", lambda: False)())
+            output = format_probe_log(timestamp_ms, level, message, use_color)
+            written = stream.write(output)
+            if written is not None and written != len(output):
+                with self._lifecycle_lock:
+                    self.error_count += 1
+                return
+            stream.flush()
+        except Exception:
+            with self._lifecycle_lock:
+                self.error_count += 1

hyperprobe/core/quota.py

@@ -0,0 +1,73 @@
+import time
+import threading
+
+
+class QuotaReservation:
+    def __init__(self, bucket, count: float):
+        self._bucket = bucket
+        self._count = count
+        self._closed = False
+        self._lock = threading.Lock()
+
+    def commit(self):
+        with self._lock:
+            self._closed = True
+
+    def release(self):
+        with self._lock:
+            if self._closed:
+                return
+            self._closed = True
+        self._bucket.refund(self._count)
+
+
+class TokenBucket:
+    def __init__(self, capacity: float, refill_rate_per_sec: float):
+        self.capacity = capacity
+        self.refill_rate = refill_rate_per_sec  # tokens per second
+        self.tokens = capacity
+        self.last_refill = time.monotonic()
+        self.lock = threading.Lock()
+
+    def try_consume(self, count: float = 1.0) -> bool:
+        reservation = self.reserve(count)
+        if reservation is None:
+            return False
+        reservation.commit()
+        return True
+
+    def reserve(self, count: float = 1.0):
+        with self.lock:
+            self._refill()
+            if self.tokens >= count:
+                self.tokens -= count
+                return QuotaReservation(self, count)
+            return None
+
+    def refund(self, count: float):
+        with self.lock:
+            self._refill()
+            self.tokens = min(self.capacity, self.tokens + count)
+
+    def _refill(self):
+        now = time.monotonic()
+        delta = now - self.last_refill
+        amount = delta * self.refill_rate
+        self.tokens = min(self.capacity, self.tokens + amount)
+        self.last_refill = now
+
+
+class QuotaManager:
+    def __init__(self, hits_per_sec: float = 10.0, bytes_per_sec: float = 200.0 * 1024.0):
+        # Burst capacity of hits_per_sec and bytes_per_sec (1 second worth)
+        self.eval_bucket = TokenBucket(hits_per_sec, hits_per_sec)
+        self.bandwidth_bucket = TokenBucket(bytes_per_sec, bytes_per_sec)
+
+    def can_evaluate(self) -> bool:
+        return self.eval_bucket.try_consume(1.0)
+
+    def can_send(self, bytes_count: float) -> bool:
+        return self.bandwidth_bucket.try_consume(bytes_count)
+
+    def reserve_bandwidth(self, bytes_count: float):
+        return self.bandwidth_bucket.reserve(bytes_count)

hyperprobe/core/safety.py

@@ -0,0 +1,136 @@
+import time
+import threading
+
+class AgentHealth:
+    GREEN = 'GREEN'   # All good
+    YELLOW = 'YELLOW' # Rate limited (Reactive Shedding)
+    RED = 'RED'       # Critical Lag (Proactive Shedding - Detach)
+
+
+class SafetyMonitor:
+    def __init__(self, on_state_change, max_lag_ms=50.0, pause_budget_ms=15.0):
+        self.on_state_change = on_state_change
+        self.max_lag_ms = max_lag_ms
+        self.pause_budget_ms = pause_budget_ms
+        
+        self.health = AgentHealth.GREEN
+        self.cumulative_pause_time = 0.0
+        self.last_window_reset = time.monotonic()
+        self.last_thread_lag_ms = 0.0
+        
+        self.lock = threading.RLock()
+        self._notification_lock = threading.RLock()
+        
+        self.is_running = False
+        self.thread = None
+        self.stop_event = threading.Event()
+
+    def start(self):
+        self.stop_event.clear()
+        self.is_running = True
+        self.thread = threading.Thread(target=self._run_loop, name="hyperprobe-safety")
+        self.thread.daemon = True
+        self.thread.start()
+
+    def stop(self):
+        self.is_running = False
+        self.stop_event.set()
+        if self.thread and self.thread is not threading.current_thread():
+            self.thread.join(timeout=2.0)
+        self.thread = None
+
+    def report_pause_duration(self, ms: float):
+        with self.lock:
+            self.cumulative_pause_time += ms
+            transition = self._evaluate_health_locked(self.last_thread_lag_ms)
+
+        self._notify_state_change(transition)
+
+    def _run_loop(self):
+        last_heartbeat = time.monotonic()
+
+        while self.is_running and not self.stop_event.wait(0.1):
+            now = time.monotonic()
+            thread_lag_ms = (now - last_heartbeat - 0.1) * 1000.0
+            last_heartbeat = now
+
+            with self.lock:
+                self.last_thread_lag_ms = thread_lag_ms
+                if now - self.last_window_reset > 1.0:
+                    self.cumulative_pause_time = 0.0
+                    self.last_window_reset = now
+
+                transition = self._evaluate_health_locked(thread_lag_ms)
+
+            self._notify_state_change(transition)
+
+    def _check_health(self, thread_lag_ms: float = 0.0):
+        """
+        Preserves the existing method signature while ensuring that
+        callbacks execute outside the state lock.
+        """
+        with self.lock:
+            transition = self._evaluate_health_locked(thread_lag_ms)
+
+        self._notify_state_change(transition)
+
+    def _evaluate_health_locked(self, thread_lag_ms: float = 0.0):
+        previous_health = self.health
+        reason = None
+
+        if thread_lag_ms > self.max_lag_ms:
+            self.health = AgentHealth.RED
+            reason = (
+                f"Execution Thread Lag ({thread_lag_ms:.1f}ms) exceeded "
+                f"limit ({self.max_lag_ms}ms)"
+            )
+        elif self.cumulative_pause_time > self.pause_budget_ms:
+            self.health = AgentHealth.RED
+            reason = (
+                f"Cumulative Pause Budget ({self.cumulative_pause_time:.1f}ms) "
+                f"exceeded limit ({self.pause_budget_ms}ms)"
+            )
+        elif thread_lag_ms > self.max_lag_ms / 2.0:
+            self.health = AgentHealth.YELLOW
+            reason = (
+                f"Moderate impact: Execution Thread Lag "
+                f"({thread_lag_ms:.1f}ms) reached 50% of limit "
+                f"({self.max_lag_ms}ms)"
+            )
+        elif self.cumulative_pause_time > self.pause_budget_ms / 2.0:
+            self.health = AgentHealth.YELLOW
+            reason = (
+                f"Moderate impact: Cumulative Pause Budget "
+                f"({self.cumulative_pause_time:.1f}ms) reached 50% of limit "
+                f"({self.pause_budget_ms}ms)"
+            )
+        else:
+            self.health = AgentHealth.GREEN
+            if previous_health != AgentHealth.GREEN:
+                reason = "System stabilized."
+
+        if self.health != previous_health:
+            return self.health, reason
+
+        return None
+
+    def _notify_state_change(self, transition):
+        if transition is None:
+            return
+
+        health, reason = transition
+
+        with self._notification_lock:
+            with self.lock:
+                if self.health != health:
+                    return
+
+            try:
+                self.on_state_change(health, reason)
+            except Exception:
+                # Safety callbacks must never break application execution.
+                pass
+
+    def get_health(self) -> str:
+        with self.lock:
+            return self.health

hyperprobe/core/serializer.py

@@ -0,0 +1,214 @@
+import math
+
+
+def _type_summary(obj):
+    type_name = type(obj).__name__
+    if isinstance(obj, (bytes, bytearray, memoryview)):
+        try:
+            return f"[{type_name}: {len(obj)} bytes]"
+        except Exception:
+            return f"[{type_name}]"
+    if isinstance(obj, dict):
+        try:
+            return f"[Dict: {len(obj)} keys]"
+        except Exception:
+            return "[Dict]"
+    if isinstance(obj, (list, tuple, set)):
+        try:
+            return f"[List: {len(obj)} items]"
+        except Exception:
+            return "[List]"
+    return f"[Obj: {type_name}]"
+
+
+def _safe_key(key):
+    if isinstance(key, (str, int, float, bool)) or key is None:
+        return str(key)
+    return f"[{type(key).__name__} key]"
+
+
+def _safe_length(value):
+    try:
+        return len(value)
+    except Exception:
+        return None
+
+
+def serialize(
+    obj,
+    max_depth=3,
+    max_array_length=3,
+    max_object_properties=50,
+    max_string_length=1024,
+    redact_keys_re=None,
+    redact_values_re=None,
+    visited=None,
+    depth=0,
+    path="$",
+):
+    """Serialize a bounded object graph without invoking arbitrary repr()."""
+    if visited is None:
+        visited = {}
+
+    if obj is None:
+        return None
+    if isinstance(obj, bool):
+        return obj
+    if isinstance(obj, int):
+        return obj
+    if isinstance(obj, float):
+        if not math.isfinite(obj):
+            return str(obj)
+        return obj
+    if isinstance(obj, str):
+        value = obj
+        if len(value) > max_string_length:
+            value = (
+                value[:max_string_length]
+                + f"... [Truncated: +{len(value) - max_string_length} more chars]"
+            )
+        if redact_values_re and redact_values_re.search(value):
+            value = redact_values_re.sub("[REDACTED Value]", value)
+        return value
+    if isinstance(obj, (bytes, bytearray, memoryview)):
+        return _type_summary(obj)
+
+    obj_id = id(obj)
+    previous = visited.get(obj_id)
+    if previous is not None:
+        previous_obj, previous_path = previous
+        if previous_obj is obj:
+            return f"[REF - {previous_path}]"
+
+    if depth > max_depth:
+        return _type_summary(obj)
+
+    # Retain the object for the whole serialization operation. This preserves
+    # aliases and prevents stale id() reuse while traversing generators.
+    visited[obj_id] = (obj, path)
+
+    if isinstance(obj, dict):
+        result = {}
+        processed = 0
+        try:
+            iterator = iter(obj.items())
+            while processed < max_object_properties:
+                try:
+                    key, value = next(iterator)
+                except StopIteration:
+                    break
+
+                key_string = _safe_key(key)
+                if redact_keys_re and redact_keys_re.search(key_string):
+                    result[key_string] = "[REDACTED Key]"
+                else:
+                    child_path = f"{path}.{key_string}" if path else key_string
+                    result[key_string] = serialize(
+                        value,
+                        max_depth=max_depth,
+                        max_array_length=max_array_length,
+                        max_object_properties=max_object_properties,
+                        max_string_length=max_string_length,
+                        redact_keys_re=redact_keys_re,
+                        redact_values_re=redact_values_re,
+                        visited=visited,
+                        depth=depth + 1,
+                        path=child_path,
+                    )
+                processed += 1
+
+            total = _safe_length(obj)
+            if total is not None and total > processed:
+                result["__probe_meta"] = (
+                    f"+ {total - processed} more properties truncated"
+                )
+            elif total is None:
+                try:
+                    next(iterator)
+                except StopIteration:
+                    pass
+                else:
+                    result["__probe_meta"] = "+ more properties truncated"
+        except Exception:
+            result["__probe_meta"] = "[Dictionary traversal failed]"
+        return result
+
+    if isinstance(obj, (list, tuple, set)):
+        result = []
+        processed = 0
+        try:
+            iterator = iter(obj)
+            while processed < max_array_length:
+                try:
+                    value = next(iterator)
+                except StopIteration:
+                    break
+                result.append(
+                    serialize(
+                        value,
+                        max_depth=max_depth,
+                        max_array_length=max_array_length,
+                        max_object_properties=max_object_properties,
+                        max_string_length=max_string_length,
+                        redact_keys_re=redact_keys_re,
+                        redact_values_re=redact_values_re,
+                        visited=visited,
+                        depth=depth + 1,
+                        path=f"{path}[{processed}]",
+                    )
+                )
+                processed += 1
+
+            total = _safe_length(obj)
+            if total is not None and total > processed:
+                result.append(f"[+ {total - processed} more items truncated]")
+            elif total is None:
+                try:
+                    next(iterator)
+                except StopIteration:
+                    pass
+                else:
+                    result.append("[+ more items truncated]")
+        except Exception:
+            result.append("[Iterable traversal failed]")
+        return result
+
+    try:
+        properties = vars(obj)
+    except Exception:
+        return _type_summary(obj)
+
+    result = {}
+    try:
+        keys = [key for key in properties if not key.startswith("_")]
+    except Exception:
+        return _type_summary(obj)
+
+    for index, key in enumerate(keys[:max_object_properties]):
+        if redact_keys_re and redact_keys_re.search(key):
+            result[key] = "[REDACTED Key]"
+            continue
+
+        child_path = f"{path}.{key}" if path else key
+        try:
+            value = properties[key]
+            result[key] = serialize(
+                value,
+                max_depth=max_depth,
+                max_array_length=max_array_length,
+                max_object_properties=max_object_properties,
+                max_string_length=max_string_length,
+                redact_keys_re=redact_keys_re,
+                redact_values_re=redact_values_re,
+                visited=visited,
+                depth=depth + 1,
+                path=child_path,
+            )
+        except Exception:
+            result[key] = "[Inaccessible]"
+
+    if len(keys) > max_object_properties:
+        result["__probe_meta"] = (
+            f"+ {len(keys) - max_object_properties} more properties truncated"
+        )
+    return result

hyperprobe/core/trace_extractor.py

@@ -0,0 +1,93 @@
+import sys
+import os
+import importlib
+from hyperprobe.core.logger import get_logger
+
+logger = get_logger("hyperprobe:trace")
+
+def extract_trace_context(custom_get_trace_id=None):
+    """
+    Safely extract the active trace ID from standard APMs via sys.modules reflection
+    or utilize a custom user-defined hook.
+    """
+    # 1. User-defined custom hook (Highest Priority - passed via code)
+    if custom_get_trace_id and callable(custom_get_trace_id):
+        try:
+            trace_id = custom_get_trace_id()
+            if trace_id:
+                return str(trace_id)
+        except Exception as e:
+            logger.info(f"[HyperProbe] Custom trace extractor failed: {e}")
+
+    # 1.5. Environment variable hook 
+    hook_path = os.getenv("HYPERPROBE_TRACE_HOOK")
+    if hook_path and ":" in hook_path:
+        try:
+            module_name, func_name = hook_path.split(":", 1)
+            # Safely ensure CWD is in the import path
+            cwd = os.getcwd()
+            added_to_path = False
+            
+            if cwd not in sys.path:
+                sys.path.insert(0, cwd)
+                added_to_path = True
+                
+            try:
+                user_module = importlib.import_module(module_name)
+                user_func = getattr(user_module, func_name)
+                trace_id = user_func()
+                if trace_id:
+                    return str(trace_id)
+            finally:
+                # Clean up if we modified sys.path
+                if added_to_path:
+                    sys.path.remove(cwd)
+                    
+        except Exception as e:
+            logger.info(f"[HyperProbe] Failed to execute HYPERPROBE_TRACE_HOOK: {e}")
+
+    # 2. OpenTelemetry
+    try:
+        if 'opentelemetry.trace' in sys.modules:
+            otel_trace = sys.modules['opentelemetry.trace']
+            span = otel_trace.get_current_span()
+            if span:
+                ctx = span.get_span_context()
+                if ctx and ctx.is_valid:
+                    return format(ctx.trace_id, '032x')
+    except Exception:
+        pass
+
+    # 3. Datadog (ddtrace)
+    try:
+        if 'ddtrace' in sys.modules:
+            ddtrace = sys.modules['ddtrace']
+            span = ddtrace.tracer.current_span()
+            if span and span.trace_id:
+                return str(span.trace_id)
+    except Exception:
+        pass
+
+    # 4. New Relic
+    try:
+        if 'newrelic.agent' in sys.modules:
+            nr = sys.modules['newrelic.agent']
+            if hasattr(nr, 'current_trace_id'):
+                trace_id = nr.current_trace_id()
+                if trace_id:
+                    return str(trace_id)
+    except Exception:
+        pass
+
+    # 5. Elastic APM
+    try:
+        if 'elasticapm' in sys.modules:
+            elasticapm = sys.modules['elasticapm']
+            trace_id = elasticapm.get_trace_id()
+            if trace_id:
+                return str(trace_id)
+    except Exception:
+        pass
+    
+    logger.info("[HyperProbe] trace id could not be extracted.")
+    return None

hyperprobe/protos/__init__.py

@@ -0,0 +1,7 @@
+import sys
+import os
+
+# Inject current directory into sys.path temporarily so generated protobuf imports resolve correctly
+current_dir = os.path.dirname(os.path.abspath(__file__))
+if current_dir not in sys.path:
+    sys.path.insert(0, current_dir)