hyperprobe-agent 1.2.24__py3-none-any.whl

hyperprobe/core/monitoring_engine.py

@@ -0,0 +1,788 @@
+import json
+import math
+import os
+import re
+import sys
+import threading
+import time
+import weakref
+
+from hyperprobe.core.evaluator import ProbeEvaluator
+from hyperprobe.core.logger import get_logger
+from hyperprobe.core.serializer import serialize
+from hyperprobe.core.trace_extractor import extract_trace_context
+from hyperprobe.protos import agent_pb2
+
+logger = get_logger("hyperprobe:monitor")
+
+class MonitoringEngine:
+    TOOL_ID_CANDIDATES = ( 3, 4)
+    PENDING_POLL_INTERVAL_SEC = 0.5
+    DURATION_TTL_SECONDS = 60.0
+    _JS_FLOAT_PREFIX_RE = re.compile(
+        r"^\s*([+-]?(?:(?:[0-9]+\.?[0-9]*)|(?:\.[0-9]+))(?:[eE][+-]?[0-9]+)?)"
+    )
+
+    def __init__(
+        self,
+        quota_manager,
+        safety_monitor,
+        on_capture,
+        custom_set_trace_id=None
+    ):
+        self.quota_manager = quota_manager
+        self.safety_monitor = safety_monitor
+        self.on_capture = on_capture
+        self.custom_set_trace_id = custom_set_trace_id
+        self.tool_id = None
+        self._closed = False
+
+        self.lock = threading.RLock()  # Reentrant Lock for safety during callbacks
+
+        # Memory-safe weak references to pivoted code objects
+        self.pivoted_codes = weakref.WeakSet()
+        self.pivoted_locations = set()  # (filepath, line_number)
+        self.pending_pivots = set()     # (filepath, line_number)
+        
+        self.active_probes = {}
+        self.secondary_duration_probes = {}
+        self.instrumented_files = set()
+        self.is_active = False
+        self.is_suspended = False
+        self.global_config = {}
+        self._redact_keys_re = None
+        self._redact_values_re = None
+        self.total_hits = 0
+        self.total_skips = 0
+
+        self.duration_starts = {}
+        self._next_duration_cleanup = time.monotonic() + self.DURATION_TTL_SECONDS
+
+        # Cached resolved file paths
+        self._file_cache = {}
+        self._line_start_cache = weakref.WeakKeyDictionary()
+
+        self._claim_tool()
+
+    def _claim_tool(self):
+        for tool_id in self.TOOL_ID_CANDIDATES:
+            claimed = False
+            try:
+                if sys.monitoring.get_tool(tool_id) is not None:
+                    continue
+                sys.monitoring.use_tool_id(tool_id, "hyperprobe")
+                claimed = True
+                sys.monitoring.register_callback(
+                    tool_id,
+                    sys.monitoring.events.LINE,
+                    self._line_callback,
+                )
+                self.tool_id = tool_id
+                return
+            except ValueError:
+                continue
+            except Exception as e:
+                logger.forceError(f"\033[1m\033[33m⚠️ [HyperProbe] CRITICAL FAIL ON TOOL ID {tool_id}: {type(e).__name__} - {str(e)}")
+                if claimed:
+                    try:
+                        sys.monitoring.free_tool_id(tool_id)
+                    except Exception:
+                        pass
+                    self.tool_id = None
+                continue
+
+        logger.forceError("\033[1m\033[33m⚠️  [HyperProbe] No available sys.monitoring tool ID for agent. Running application uninstrumented.")
+
+    def set_global_config(self, config):
+        with self.lock:
+            candidate = dict(config or {})
+            try:
+                redact_keys, redact_keys_re = self._compile_redaction_patterns(
+                    candidate.get("redact_keys", [])
+                )
+                redact_values, redact_values_re = self._compile_redaction_patterns(
+                    candidate.get("redact_values", [])
+                )
+            except (re.error, TypeError) as e:
+                logger.error(f'[HyperProbe] error while compiling regex : {type(e).__name__}: {e}')
+                return False
+
+            candidate["redact_keys"] = redact_keys
+            candidate["redact_values"] = redact_values
+            self.global_config = candidate
+            self._redact_keys_re = redact_keys_re
+            self._redact_values_re = redact_values_re
+            return True
+
+    @staticmethod
+    def _compile_redaction_patterns(patterns):
+        if patterns is None:
+            patterns = ()
+        elif isinstance(patterns, str):
+            raise TypeError("Redaction patterns must be a collection of strings")
+
+        normalized = []
+        for pattern in patterns:
+            if not isinstance(pattern, str):
+                raise TypeError("Redaction patterns must be strings")
+            pattern = pattern.strip()
+            if not pattern:
+                continue
+            re.compile(pattern, re.IGNORECASE)
+            normalized.append(pattern)
+
+        combined = (
+            re.compile(
+                "|".join(f"(?:{pattern})" for pattern in normalized),
+                re.IGNORECASE,
+            )
+            if normalized
+            else None
+        )
+        return normalized, combined
+
+    def suspend(self):
+        """Disable instrumentation without discarding the configured probes."""
+        with self.lock:
+            if self._closed or self.is_suspended:
+                return
+            self.is_suspended = True
+            if self.tool_id is not None:
+                for code in list(self.pivoted_codes):
+                    try:
+                        sys.monitoring.set_local_events(self.tool_id, code, 0)
+                    except (RuntimeError, ValueError):
+                        pass
+            self._stop_monitoring()
+
+    def resume(self):
+        """Re-enable instrumentation after safety recovery."""
+        with self.lock:
+            if self._closed:
+                return
+            self.pending_pivots.update(self.pivoted_locations)
+            self.is_suspended = False
+            self._update_monitoring_state()
+
+    def get_stats(self):
+        with self.lock:
+            self._cleanup_durations_locked(time.monotonic())
+            stats = {"hits": self.total_hits, "skips": self.total_skips}
+            self.total_hits = 0
+            self.total_skips = 0
+            return stats
+
+    def _resolve_filename_cached(self, filename):
+        resolved = self._file_cache.get(filename)
+        if resolved is None:
+            resolved = os.path.abspath(os.path.realpath(filename))
+            self._file_cache[filename] = resolved
+        return resolved
+
+    def _is_first_event_for_line(self, code, line_number, instruction_offset):
+        line_starts = self._line_start_cache.get(code)
+        if line_starts is None:
+            line_starts = {}
+            for start_offset, _, source_line in code.co_lines():
+                if source_line is not None and source_line not in line_starts:
+                    line_starts[source_line] = start_offset
+            self._line_start_cache[code] = line_starts
+        return line_starts.get(line_number) == instruction_offset
+
+    def set_probes(self, probes):
+        with self.lock:
+            if self._closed:
+                return
+
+            new_active_probes = {}
+            new_secondary_duration_probes = {}
+            new_instrumented_files = set()
+            new_pending_pivots = set()
+            now = time.monotonic()
+
+            for probe in probes:
+                filepath = self._resolve_filepath(probe.runtime_location)
+                loc_key = (filepath, probe.runtime_line)
+                if loc_key not in new_active_probes:
+                    new_active_probes[loc_key] = []
+                new_active_probes[loc_key].append(probe)
+                new_instrumented_files.add(filepath)
+
+                # Only search for files we haven't pivoted yet
+                if loc_key not in self.pivoted_locations:
+                    new_pending_pivots.add(loc_key)
+
+                if (
+                    getattr(probe, "type", agent_pb2.PROBE_TYPE_UNSPECIFIED)
+                    == agent_pb2.PROBE_TYPE_DURATION
+                ):
+                    secondary_line = getattr(probe, "secondary_runtime_line", 0)
+                    if secondary_line:
+                        secondary_location = (
+                            getattr(probe, "secondary_runtime_location", "")
+                            or probe.runtime_location
+                        )
+                        secondary_filepath = self._resolve_filepath(secondary_location)
+                        secondary_key = (secondary_filepath, secondary_line)
+                        if secondary_key not in new_secondary_duration_probes:
+                            new_secondary_duration_probes[secondary_key] = []
+                        new_secondary_duration_probes[secondary_key].append(probe)
+                        new_instrumented_files.add(secondary_filepath)
+                        if secondary_key not in self.pivoted_locations:
+                            new_pending_pivots.add(secondary_key)
+
+            target_locations = set(new_active_probes) | set(new_secondary_duration_probes)
+
+            # Clean stale pivoted locations
+            self.pivoted_locations = {
+                loc for loc in self.pivoted_locations if loc in target_locations
+            }
+
+            # Atomic swap to guarantee thread safety
+            self.active_probes = new_active_probes
+            self.secondary_duration_probes = new_secondary_duration_probes
+            self.instrumented_files = new_instrumented_files
+            self.pending_pivots = new_pending_pivots
+            logger.info(
+                f"[HyperProbe] set_probes count={len(probes)} "
+                f"pending={len(self.pending_pivots)} files={len(self.instrumented_files)}"
+            )
+
+            self._update_monitoring_state()
+
+    def _update_monitoring_state(self):
+        # Must be called under a lock
+        if self._closed or self.tool_id is None:
+            return
+
+        # Clean local events for code objects from untargeted files
+        for code in list(self.pivoted_codes):
+            resolved_filename = self._resolve_filename_cached(code.co_filename)
+            if resolved_filename not in self.instrumented_files:
+                try:
+                    sys.monitoring.set_local_events(self.tool_id, code, 0)
+                except (RuntimeError, ValueError):
+                    pass
+                self.pivoted_codes.discard(code)
+
+        if self.is_suspended:
+            return
+
+        # Update global searchlight line callbacks (Keep the callback registered)
+        if self.pending_pivots:
+            logger.info(f"[HyperProbe] pending pivots remaining : {len(self.pending_pivots)}. starting global search.")
+            sys.monitoring.set_events(self.tool_id, sys.monitoring.events.LINE)
+            self.is_active = True
+        else:
+            logger.info(f"[HyperProbe] pending pivots remaining : {len(self.pending_pivots)}. stopped global search.")
+            sys.monitoring.set_events(self.tool_id, 0)
+            self.is_active = False
+
+    def _stop_monitoring(self):
+        # Disable global search events.
+        if self.tool_id is None:
+            return
+        try:
+            sys.monitoring.set_events(self.tool_id, 0)
+        except (RuntimeError, ValueError) as e:
+            logger.error(
+                f"[HyperProbe] Failed to disable monitoring: {type(e).__name__}: {e}"
+            )
+        else:
+            logger.info("[HyperProbe] Monitoring disabled.")
+        self.is_active = False
+
+    def close(self):
+        with self.lock:
+            if self._closed:
+                return
+            self._closed = True
+            self._stop_monitoring()
+
+            for code in list(self.pivoted_codes):
+                try:
+                    sys.monitoring.set_local_events(self.tool_id, code, 0)
+                except (RuntimeError, ValueError):
+                    pass
+
+            self.pivoted_codes.clear()
+            self.pivoted_locations.clear()
+            self.pending_pivots.clear()
+
+            if self.tool_id is not None:
+                try:
+                    sys.monitoring.register_callback(
+                        self.tool_id,
+                        sys.monitoring.events.LINE,
+                        None,
+                    )
+                except Exception:
+                    pass
+                try:
+                    sys.monitoring.free_tool_id(self.tool_id)
+                except Exception:
+                    pass
+                self.tool_id = None
+
+    def _line_callback(self, code, line_number):
+        with self.lock:
+            if self._closed or self.is_suspended:
+                return None
+
+            filepath = self._resolve_filename_cached(code.co_filename)
+            if filepath not in self.instrumented_files:
+                return None
+
+            loc_key = (filepath, line_number)
+            
+            # Catch and Pivot Searchlight logic
+            if loc_key in self.pending_pivots:
+                try:
+                    sys.monitoring.set_local_events(self.tool_id, code, sys.monitoring.events.LINE)
+                    self.pivoted_codes.add(code)
+                    self.pivoted_locations.add(loc_key)
+                except ValueError:
+                    pass
+                self.pending_pivots.discard(loc_key)
+                self._update_monitoring_state()
+
+            probes_list = self.active_probes.get(loc_key)
+            secondary_probes_list = self.secondary_duration_probes.get(loc_key)
+            if not probes_list and not secondary_probes_list:
+                return None
+            try:
+                frame = sys._getframe(1)
+            except ValueError:
+                return None
+            # A multiline statement can leave and re-enter its opening source line.
+            if not self._is_first_event_for_line(code, line_number, frame.f_lasti):
+                return None
+
+        # Execute hit capture outside the self.lock to avoid serialization contention
+        try:
+            if frame:
+                for probe in probes_list or ():
+                    self._handle_probe_hit(probe, frame, is_secondary=False)
+                for probe in secondary_probes_list or ():
+                    self._handle_probe_hit(probe, frame, is_secondary=True)
+        except Exception:
+            pass
+
+        return None
+
+    def _handle_probe_hit(self, probe, frame, is_secondary=False):
+        probe_type = getattr(probe, "type", agent_pb2.PROBE_TYPE_UNSPECIFIED)
+        if probe_type in (
+            agent_pb2.PROBE_TYPE_COUNTER,
+            agent_pb2.PROBE_TYPE_METRIC,
+            agent_pb2.PROBE_TYPE_DURATION,
+        ):
+            self._handle_metric_probe_hit(probe, frame, is_secondary)
+            return
+
+        start_time = time.perf_counter()
+        # Check admission before touching the frame or evaluating user code.
+        if not self.quota_manager.can_evaluate():
+            with self.lock:
+                self.total_skips += 1
+            logger.error(f"[HyperProbe] Probe ID={probe.id} hit rate-limited by QuotaManager.")
+            return
+
+        globals_dict = frame.f_globals
+        locals_dict = frame.f_locals
+
+        # Get config snapshot under lock
+        with self.lock:
+            config_snapshot = self.global_config.copy() if self.global_config else {}
+            redact_keys_re = self._redact_keys_re
+            redact_values_re = self._redact_values_re
+
+        if probe.condition:
+            try:
+                cond_val = ProbeEvaluator.safe_eval(probe.condition, globals_dict, locals_dict)
+                if not cond_val:
+                    return
+            except Exception as e:
+                self._report_error(probe.id, f"Condition evaluation failed: {type(e).__name__}: {str(e)}")
+                return
+
+        logger.info(f"[HyperProbe] Hit probe ID={probe.id}! Capturing telemetry...")
+
+        # Prepare parameters from snapshot
+        max_depth = getattr(probe, 'max_object_depth', None) or config_snapshot.get('max_object_depth', 3)
+        max_array_length = getattr(probe, 'max_array_length', None) or config_snapshot.get('max_array_length', 3)
+        max_object_properties = getattr(probe, 'max_object_properties', None) or config_snapshot.get('max_object_properties', 50)
+        max_string_length = getattr(probe, 'max_string_length', None) or config_snapshot.get('max_string_length', 1024)
+        stack_frame_depth = getattr(probe, 'stack_frame_depth', None) or config_snapshot.get('stack_frame_depth', 3)
+
+        # Build telemetry structure
+        event = {
+            "probe_id": probe.id,
+            "timestamp_ms": int(time.time() * 1000),
+            "stack_frames": [],
+            "captured_vars_json": "",
+            "watch_results_json": "",
+            "evaluated_log": "",
+            "metric_value": 0.0,
+            "capture_error": "",
+            "trace_id": None
+        }
+
+        if getattr(probe, 'should_capture_trace_id', False):
+            trace_id = extract_trace_context(self.custom_set_trace_id)
+            if trace_id:
+                event["trace_id"] = trace_id
+            logger.info(f"[HyperProbe] trace id found : {trace_id}")
+
+        try:
+            # SNAPSHOT CAPTURE
+            if probe.type == 1:
+                serialization_context = {}
+
+                if getattr(probe, 'watch_expressions', None):
+                    watches_result = ProbeEvaluator.evaluate_watches(probe.watch_expressions, globals_dict, locals_dict)
+                    serialized_watches = {}
+                    for watch_name, watch_value in watches_result.items():
+                        watch_path = f"watch[{json.dumps(str(watch_name))}]"
+                        serialized_watches[watch_name] = serialize(
+                            watch_value,
+                            max_depth=max_depth,
+                            max_array_length=max_array_length,
+                            max_object_properties=max_object_properties,
+                            max_string_length=max_string_length,
+                            redact_keys_re=redact_keys_re,
+                            redact_values_re=redact_values_re,
+                            visited=serialization_context,
+                            path=watch_path,
+                        )
+                    event["watch_results_json"] = json.dumps(serialized_watches)
+                
+                curr_frame = frame
+                depth = 0
+                wrapped_vars = []
+                
+                while curr_frame and depth < stack_frame_depth:
+                    event["stack_frames"].append({
+                        "function_name": curr_frame.f_code.co_name,
+                        "file_name": curr_frame.f_code.co_filename,
+                        "line_number": curr_frame.f_lineno,
+                        "column_number": 0
+                    })
+                    
+                    clean_locals = {k: v for k, v in curr_frame.f_locals.items() if not k.startswith('_') and k != 'hyperprobe'}
+                    locals_path = f"frame[{depth}].scopes[0]"
+                    serialized_locals = serialize(
+                        clean_locals,
+                        max_depth=max_depth,
+                        max_array_length=max_array_length,
+                        max_object_properties=max_object_properties,
+                        max_string_length=max_string_length,
+                        redact_keys_re=redact_keys_re,
+                        redact_values_re=redact_values_re,
+                        visited=serialization_context,
+                        path=locals_path,
+                    )
+                    
+                    wrapped_vars.append([
+                        {
+                            "type": "local",
+                            "name": "Local",
+                            "vars": serialized_locals
+                        }
+                    ])
+                    
+                    curr_frame = curr_frame.f_back
+                    depth += 1
+                
+                event["captured_vars_json"] = json.dumps(wrapped_vars)
+
+            # LOG TEMPLATE CAPTURE
+            elif probe.type == 2:
+                evaluated = ProbeEvaluator.evaluate_log_template(probe.template, globals_dict, locals_dict)
+                if redact_values_re and redact_values_re.search(evaluated):
+                    evaluated = redact_values_re.sub("[REDACTED Value]", evaluated)
+                if len(evaluated) > max_string_length:
+                    evaluated = evaluated[:max_string_length] + f"... [Truncated: +{len(evaluated) - max_string_length} more chars]"
+                event["evaluated_log"] = evaluated
+
+            # Safety budget calculation
+            # Fire callback
+            with self.lock:
+                self.total_hits += 1
+            self.on_capture(event)
+            logger.info(f"[HyperProbe] Successfully captured and queued telemetry for probe ID={probe.id}.")
+
+        except Exception as err:
+            err_msg = f"Capture failed: {type(err).__name__}: {str(err)}"
+            logger.error(f"[HyperProbe] Error during capture: {err_msg}")
+            event["capture_error"] = err_msg
+            with self.lock:
+                self.total_hits += 1
+            self.on_capture(event)
+        finally:
+            duration_ms = (time.perf_counter() - start_time) * 1000.0
+            self.safety_monitor.report_pause_duration(duration_ms)
+
+    def _handle_metric_probe_hit(self, probe, frame, is_secondary):
+        """Handle counter, metric, and duration probes without snapshot capture."""
+        handler_started_at = time.perf_counter()
+        globals_dict = frame.f_globals
+        locals_dict = frame.f_locals
+
+        try:
+            if probe.condition:
+                try:
+                    if not ProbeEvaluator.safe_eval(
+                        probe.condition, globals_dict, locals_dict
+                    ):
+                        return
+                except Exception as error:
+                    self._report_metric_error(probe, str(error))
+                    return
+
+            if probe.type == agent_pb2.PROBE_TYPE_COUNTER:
+                if not self._consume_evaluation_quota(probe.id):
+                    return
+                self._emit_metric_event(probe, metric_value=1.0)
+                return
+
+            if probe.type == agent_pb2.PROBE_TYPE_METRIC:
+                metric_expression = getattr(probe, "metric_expression", "")
+                if not metric_expression:
+                    # The backend validates this field. Node also emits nothing when
+                    # an empty expression reaches the SDK.
+                    return
+                try:
+                    raw_value = ProbeEvaluator.safe_eval(
+                        metric_expression, globals_dict, locals_dict
+                    )
+                except Exception as error:
+                    self._report_metric_error(probe, str(error))
+                    return
+
+                if not self._consume_evaluation_quota(probe.id):
+                    return
+
+                metric_value = self._coerce_metric_value(raw_value)
+                if metric_value is None:
+                    self._emit_metric_event(
+                        probe,
+                        capture_error=f"Metric evaluation failed: {raw_value}",
+                    )
+                else:
+                    self._emit_metric_event(probe, metric_value=metric_value)
+                return
+
+            correlation_expression = getattr(
+                probe, "correlation_expression", ""
+            )
+            if correlation_expression:
+                try:
+                    correlation_value = ProbeEvaluator.safe_eval(
+                        correlation_expression, globals_dict, locals_dict
+                    )
+                except Exception as error:
+                    self._report_metric_error(probe, f"Error: {error}")
+                    return
+            else:
+                correlation_value = None
+
+            try:
+                correlation_key = self._normalize_correlation_key(correlation_value)
+            except (TypeError, ValueError) as error:
+                self._report_metric_error(probe, str(error))
+                return
+
+            if not is_secondary:
+                self._start_duration(probe.id, correlation_key)
+                return
+
+            duration_ms = self._finish_duration(probe.id, correlation_key)
+            if duration_ms is None:
+                return
+            if not self._consume_evaluation_quota(probe.id):
+                return
+            self._emit_metric_event(probe, metric_value=duration_ms)
+        finally:
+            try:
+                self.safety_monitor.report_pause_duration(
+                    (time.perf_counter() - handler_started_at) * 1000.0
+                )
+            except Exception:
+                # Safety accounting must never affect application execution.
+                pass
+
+    def _consume_evaluation_quota(self, probe_id):
+        if self.quota_manager.can_evaluate():
+            return True
+        with self.lock:
+            self.total_skips += 1
+        logger.error(
+            f"[HyperProbe] Probe ID={probe_id} hit rate-limited by QuotaManager."
+        )
+        return False
+
+    def _build_metric_event(self, probe, metric_value=0.0, capture_error=""):
+        event = {
+            "probe_id": probe.id,
+            "timestamp_ms": int(time.time() * 1000),
+            "stack_frames": [],
+            "captured_vars_json": "",
+            "watch_results_json": "",
+            "evaluated_log": "",
+            "metric_value": float(metric_value),
+            "capture_error": capture_error,
+            "trace_id": None,
+        }
+        if getattr(probe, "should_capture_trace_id", False):
+            event["trace_id"] = extract_trace_context(self.custom_set_trace_id)
+        return event
+
+    def _emit_metric_event(self, probe, metric_value=0.0, capture_error=""):
+        event = self._build_metric_event(probe, metric_value, capture_error)
+
+        # no need to check for bandwidth here.
+        # bandwidth check is owned by on_capture()
+        
+        # payload_size = len(
+        #     json.dumps(event, separators=(",", ":"), ensure_ascii=False).encode(
+        #         "utf-8"
+        #     )
+        # )
+       
+        # if not self.quota_manager.can_send(payload_size):
+        #     with self.lock:
+        #         self.total_skips += 1
+        #     return False
+
+        with self.lock:
+            self.total_hits += 1
+        self.on_capture(event)
+
+    def _report_metric_error(self, probe, error_message):
+        self._emit_metric_event(probe, capture_error=error_message)
+
+    @classmethod
+    def _coerce_metric_value(cls, value):
+        """Mirror Node's finite-number/parseFloat behavior for common values."""
+        if isinstance(value, bool):
+            return None
+
+        if isinstance(value, (int, float)):
+            try:
+                parsed = float(value)
+            except (OverflowError, TypeError, ValueError):
+                return None
+            return parsed if math.isfinite(parsed) else None
+
+        if isinstance(value, str):
+            match = cls._JS_FLOAT_PREFIX_RE.match(value)
+            if not match:
+                return None
+            try:
+                parsed = float(match.group(1))
+            except (OverflowError, ValueError):
+                return None
+            return parsed if math.isfinite(parsed) else None
+
+        return None
+
+    @staticmethod
+    def _node_type_name(value):
+        if isinstance(value, bool):
+            return "boolean"
+        if isinstance(value, str):
+            return "string"
+        if isinstance(value, (int, float)):
+            return "number"
+        if value is None:
+            return "object"
+        return "object"
+
+    @classmethod
+    def _normalize_correlation_key(cls, value):
+        if value is None:
+            normalized = "static-singleton"
+        elif isinstance(value, bool):
+            raise TypeError(
+                "Correlation expression must evaluate to a string or number, "
+                f"got {cls._node_type_name(value)}"
+            )
+        elif isinstance(value, str):
+            if value.startswith("Error: "):
+                raise ValueError(value)
+            normalized = value
+        elif isinstance(value, int):
+            normalized = str(value)
+        elif isinstance(value, float):
+            if math.isnan(value):
+                normalized = "NaN"
+            elif math.isinf(value):
+                normalized = "Infinity" if value > 0 else "-Infinity"
+            elif value.is_integer():
+                normalized = str(int(value))
+            else:
+                normalized = str(value)
+        else:
+            raise TypeError(
+                "Correlation expression must evaluate to a string or number, "
+                f"got {cls._node_type_name(value)}"
+            )
+
+        return normalized
+
+    def _start_duration(self, probe_id, correlation_key):
+        start_time = time.perf_counter()
+        created_at = time.monotonic()
+        duration_key = (probe_id, correlation_key)
+        with self.lock:
+            self._cleanup_durations_locked(created_at)
+            self.duration_starts[duration_key] = (start_time, created_at)
+
+    def _finish_duration(self, probe_id, correlation_key):
+        end_time = time.perf_counter()
+        duration_key = (probe_id, correlation_key)
+        with self.lock:
+            self._cleanup_durations_locked(time.monotonic())
+            entry = self.duration_starts.pop(duration_key, None)
+        if entry is None:
+            return None
+        return (end_time - entry[0]) * 1000.0
+
+    def _cleanup_durations_locked(self, now):
+        if now < self._next_duration_cleanup:
+            return
+
+        for duration_key, (_, created_at) in list(self.duration_starts.items()):
+            if now - created_at > self.DURATION_TTL_SECONDS:
+                del self.duration_starts[duration_key]
+
+        self._next_duration_cleanup = now + self.DURATION_TTL_SECONDS
+
+    def _report_error(self, probe_id, err_msg):
+        event = {
+            "probe_id": probe_id,
+            "timestamp_ms": int(time.time() * 1000),
+            "stack_frames": [],
+            "captured_vars_json": "",
+            "watch_results_json": "",
+            "evaluated_log": "",
+            "metric_value": 0.0,
+            "capture_error": err_msg,
+            "trace_id": None
+        }
+        with self.lock:
+            self.total_hits += 1
+        self.on_capture(event)
+
+    def _resolve_filepath(self, runtime_location):
+        if os.path.isabs(runtime_location):
+            return os.path.abspath(os.path.realpath(runtime_location))
+        cwd = os.getcwd()
+        parts = runtime_location.split('/')
+        for i in range(len(parts)):
+            suffix = os.path.join(*parts[i:])
+            attempt = os.path.abspath(os.path.realpath(os.path.join(cwd, suffix)))
+            if os.path.exists(attempt):
+                return attempt
+        return os.path.abspath(os.path.realpath(os.path.join(cwd, runtime_location)))