hyperprobe-agent 1.2.25b9__tar.gz → 1.2.26b2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {hyperprobe_agent-1.2.25b9 → hyperprobe_agent-1.2.26b2}/PKG-INFO +32 -25
- {hyperprobe_agent-1.2.25b9 → hyperprobe_agent-1.2.26b2}/README.md +31 -24
- {hyperprobe_agent-1.2.25b9 → hyperprobe_agent-1.2.26b2}/hyperprobe/agent.py +283 -24
- {hyperprobe_agent-1.2.25b9 → hyperprobe_agent-1.2.26b2}/hyperprobe_agent.egg-info/PKG-INFO +32 -25
- {hyperprobe_agent-1.2.25b9 → hyperprobe_agent-1.2.26b2}/hyperprobe_agent.egg-info/SOURCES.txt +3 -5
- {hyperprobe_agent-1.2.25b9 → hyperprobe_agent-1.2.26b2}/setup.py +1 -6
- hyperprobe_agent-1.2.26b2/tests/test_fork.py +271 -0
- hyperprobe_agent-1.2.26b2/tests/test_startup.py +99 -0
- hyperprobe_agent-1.2.25b9/hyperprobe/bootstrap.py +0 -40
- hyperprobe_agent-1.2.25b9/hyperprobe/core/injection/__init__.py +0 -0
- hyperprobe_agent-1.2.25b9/hyperprobe/core/injection/sitecustomize.py +0 -18
- hyperprobe_agent-1.2.25b9/hyperprobe_agent.egg-info/entry_points.txt +0 -2
- {hyperprobe_agent-1.2.25b9 → hyperprobe_agent-1.2.26b2}/LICENSE +0 -0
- {hyperprobe_agent-1.2.25b9 → hyperprobe_agent-1.2.26b2}/hyperprobe/__init__.py +0 -0
- {hyperprobe_agent-1.2.25b9 → hyperprobe_agent-1.2.26b2}/hyperprobe/core/__init__.py +0 -0
- {hyperprobe_agent-1.2.25b9 → hyperprobe_agent-1.2.26b2}/hyperprobe/core/broker.py +0 -0
- {hyperprobe_agent-1.2.25b9 → hyperprobe_agent-1.2.26b2}/hyperprobe/core/evaluator.py +0 -0
- {hyperprobe_agent-1.2.25b9 → hyperprobe_agent-1.2.26b2}/hyperprobe/core/logger.py +0 -0
- {hyperprobe_agent-1.2.25b9 → hyperprobe_agent-1.2.26b2}/hyperprobe/core/monitoring_engine.py +0 -0
- {hyperprobe_agent-1.2.25b9 → hyperprobe_agent-1.2.26b2}/hyperprobe/core/probe_output.py +0 -0
- {hyperprobe_agent-1.2.25b9 → hyperprobe_agent-1.2.26b2}/hyperprobe/core/quota.py +0 -0
- {hyperprobe_agent-1.2.25b9 → hyperprobe_agent-1.2.26b2}/hyperprobe/core/safety.py +0 -0
- {hyperprobe_agent-1.2.25b9 → hyperprobe_agent-1.2.26b2}/hyperprobe/core/serializer.py +0 -0
- {hyperprobe_agent-1.2.25b9 → hyperprobe_agent-1.2.26b2}/hyperprobe/core/trace_extractor.py +0 -0
- {hyperprobe_agent-1.2.25b9 → hyperprobe_agent-1.2.26b2}/hyperprobe/protos/__init__.py +0 -0
- {hyperprobe_agent-1.2.25b9 → hyperprobe_agent-1.2.26b2}/hyperprobe/protos/agent_pb2.py +0 -0
- {hyperprobe_agent-1.2.25b9 → hyperprobe_agent-1.2.26b2}/hyperprobe/protos/agent_pb2_grpc.py +0 -0
- {hyperprobe_agent-1.2.25b9 → hyperprobe_agent-1.2.26b2}/hyperprobe_agent.egg-info/dependency_links.txt +0 -0
- {hyperprobe_agent-1.2.25b9 → hyperprobe_agent-1.2.26b2}/hyperprobe_agent.egg-info/requires.txt +0 -0
- {hyperprobe_agent-1.2.25b9 → hyperprobe_agent-1.2.26b2}/hyperprobe_agent.egg-info/top_level.txt +0 -0
- {hyperprobe_agent-1.2.25b9 → hyperprobe_agent-1.2.26b2}/setup.cfg +0 -0
- {hyperprobe_agent-1.2.25b9 → hyperprobe_agent-1.2.26b2}/tests/test_agent.py +0 -0
- {hyperprobe_agent-1.2.25b9 → hyperprobe_agent-1.2.26b2}/tests/test_evaluator.py +0 -0
- {hyperprobe_agent-1.2.25b9 → hyperprobe_agent-1.2.26b2}/tests/test_logger.py +0 -0
- {hyperprobe_agent-1.2.25b9 → hyperprobe_agent-1.2.26b2}/tests/test_metric_probes.py +0 -0
- {hyperprobe_agent-1.2.25b9 → hyperprobe_agent-1.2.26b2}/tests/test_multithreading_integration.py +0 -0
- {hyperprobe_agent-1.2.25b9 → hyperprobe_agent-1.2.26b2}/tests/test_p1_regressions.py +0 -0
- {hyperprobe_agent-1.2.25b9 → hyperprobe_agent-1.2.26b2}/tests/test_probe_output.py +0 -0
- {hyperprobe_agent-1.2.25b9 → hyperprobe_agent-1.2.26b2}/tests/test_serializer.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: hyperprobe-agent
|
|
3
|
-
Version: 1.2.
|
|
3
|
+
Version: 1.2.26b2
|
|
4
4
|
Summary: Production-grade, non-breaking live debugger and telemetry agent for Python.
|
|
5
5
|
Home-page: https://www.hyperprobe.co
|
|
6
6
|
Author: Arif, Saksham, Karan
|
|
@@ -46,7 +46,6 @@ HyperProbe allows you to debug running Python applications in real-time without
|
|
|
46
46
|
- **Counter Probes:** Emit a value of `1` whenever a configured source line is reached.
|
|
47
47
|
- **Metric Probes:** Safely evaluate a numerical Python expression and emit its value.
|
|
48
48
|
- **Duration Probes:** Measure elapsed monotonic time in milliseconds between two source lines, with optional request correlation.
|
|
49
|
-
- **Zero-Code-Change Auto-Instrumentation:** Prepend our bootstrap runtime to any Python process without editing your application's code.
|
|
50
49
|
- **Circular Reference & Deep Object Protection:** Memory-safe serialization of deeply nested or cyclic variables.
|
|
51
50
|
- **High-Performance Safety Guard:** Automatically pauses monitoring and goes idle if CPU/memory boundaries or pause budgets are exceeded.
|
|
52
51
|
|
|
@@ -74,16 +73,24 @@ pip install hyperprobe-agent
|
|
|
74
73
|
|
|
75
74
|
## Usage
|
|
76
75
|
|
|
77
|
-
|
|
76
|
+
Start the agent programmatically at your application's entrypoint:
|
|
78
77
|
|
|
79
|
-
```
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
78
|
+
```python
|
|
79
|
+
import os
|
|
80
|
+
from hyperprobe import HyperProbe
|
|
81
|
+
|
|
82
|
+
agent = HyperProbe.start({
|
|
83
|
+
"service_id": "<service-uuid-from-dashboard>",
|
|
84
|
+
"environment": os.getenv("PYTHON_ENV"),
|
|
85
|
+
"broker_url": "https://logger.app.hyperprobe.co",
|
|
86
|
+
"commit_sha": os.getenv("GIT_COMMIT"),
|
|
87
|
+
})
|
|
85
88
|
```
|
|
86
89
|
|
|
90
|
+
Required identity values may also be supplied through the environment variables
|
|
91
|
+
below. Explicit options take precedence. Initialization failures are reported
|
|
92
|
+
and the application continues without instrumentation.
|
|
93
|
+
|
|
87
94
|
### Configuration Environment Variables
|
|
88
95
|
|
|
89
96
|
Configure the agent using the following environment variables:
|
|
@@ -111,6 +118,7 @@ Configure the agent using the following environment variables:
|
|
|
111
118
|
| `HYPERPROBE_STACK_FRAME_DEPTH` | Maximum captured stack-frame depth. | `3` |
|
|
112
119
|
| `HYPERPROBE_MAX_OBJECT_PROPERTIES` | Maximum serialized properties per object. | `50` |
|
|
113
120
|
| `HYPERPROBE_MAX_STRING_LENGTH` | Maximum serialized string length. | `1024` |
|
|
121
|
+
| `HYPERPROBE_FORK_MODE` | Set to `worker` for preloaded, prefork servers. The parent defers agent initialization and every worker starts fresh process-local state. | `none` |
|
|
114
122
|
|
|
115
123
|
### Debug logging
|
|
116
124
|
|
|
@@ -118,9 +126,9 @@ Set `DEBUG` to a comma- or space-separated list of namespaces. Patterns accept
|
|
|
118
126
|
`*`, and exclusions begin with `-`:
|
|
119
127
|
|
|
120
128
|
```bash
|
|
121
|
-
DEBUG=hyperprobe
|
|
129
|
+
DEBUG=hyperprobe:* # all HyperProbe SDK logs
|
|
122
130
|
DEBUG=hyperprobe:broker,hyperprobe:monitor # selected components
|
|
123
|
-
DEBUG=hyperprobe
|
|
131
|
+
DEBUG=hyperprobe:*,-hyperprobe:evaluator # exclude a component
|
|
124
132
|
DEBUG_COLORS=1 # force ANSI colors
|
|
125
133
|
```
|
|
126
134
|
|
|
@@ -129,26 +137,25 @@ Without `*`, a selector is exact: `DEBUG=hyperprobe` does not match
|
|
|
129
137
|
Set `DEBUG_COLORS=0` to disable colors. Logging configuration is read during
|
|
130
138
|
SDK initialization, so set these variables before starting the process.
|
|
131
139
|
|
|
132
|
-
##
|
|
140
|
+
## Shutdown
|
|
133
141
|
|
|
134
|
-
|
|
142
|
+
Call `HyperProbe.shutdown()` when the application process stops to close
|
|
143
|
+
monitoring, workers, and the broker connection cleanly.
|
|
135
144
|
|
|
136
|
-
|
|
145
|
+
For servers that import the application before forking workers, such as
|
|
146
|
+
Gunicorn with `--preload`, enable worker fork mode:
|
|
137
147
|
|
|
138
148
|
```python
|
|
139
|
-
import os
|
|
140
|
-
from hyperprobe import HyperProbe
|
|
141
|
-
|
|
142
|
-
# Start the agent programmatically at your application's entrypoint
|
|
143
149
|
agent = HyperProbe.start({
|
|
144
150
|
"service_id": "<service-uuid-from-dashboard>",
|
|
145
|
-
"environment": os.getenv("PYTHON_ENV"),
|
|
151
|
+
"environment": os.getenv("PYTHON_ENV"),
|
|
146
152
|
"broker_url": "https://logger.app.hyperprobe.co",
|
|
147
|
-
"commit_sha": os.getenv("GIT_COMMIT"),
|
|
153
|
+
"commit_sha": os.getenv("GIT_COMMIT"),
|
|
154
|
+
"fork_mode": "worker",
|
|
148
155
|
})
|
|
149
|
-
|
|
150
|
-
# Your application code goes here...
|
|
151
|
-
|
|
152
|
-
# To cleanly shut down the agent when your app stops:
|
|
153
|
-
HyperProbe.shutdown()
|
|
154
156
|
```
|
|
157
|
+
|
|
158
|
+
The prefork parent remains agent-free. Every worker receives a unique agent ID,
|
|
159
|
+
telemetry queue, gRPC channel, monitoring engine, and set of background threads.
|
|
160
|
+
Application code running inside those workers must use the `spawn` process start
|
|
161
|
+
method rather than creating additional children with `fork`.
|
|
@@ -11,7 +11,6 @@ HyperProbe allows you to debug running Python applications in real-time without
|
|
|
11
11
|
- **Counter Probes:** Emit a value of `1` whenever a configured source line is reached.
|
|
12
12
|
- **Metric Probes:** Safely evaluate a numerical Python expression and emit its value.
|
|
13
13
|
- **Duration Probes:** Measure elapsed monotonic time in milliseconds between two source lines, with optional request correlation.
|
|
14
|
-
- **Zero-Code-Change Auto-Instrumentation:** Prepend our bootstrap runtime to any Python process without editing your application's code.
|
|
15
14
|
- **Circular Reference & Deep Object Protection:** Memory-safe serialization of deeply nested or cyclic variables.
|
|
16
15
|
- **High-Performance Safety Guard:** Automatically pauses monitoring and goes idle if CPU/memory boundaries or pause budgets are exceeded.
|
|
17
16
|
|
|
@@ -39,16 +38,24 @@ pip install hyperprobe-agent
|
|
|
39
38
|
|
|
40
39
|
## Usage
|
|
41
40
|
|
|
42
|
-
|
|
41
|
+
Start the agent programmatically at your application's entrypoint:
|
|
43
42
|
|
|
44
|
-
```
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
43
|
+
```python
|
|
44
|
+
import os
|
|
45
|
+
from hyperprobe import HyperProbe
|
|
46
|
+
|
|
47
|
+
agent = HyperProbe.start({
|
|
48
|
+
"service_id": "<service-uuid-from-dashboard>",
|
|
49
|
+
"environment": os.getenv("PYTHON_ENV"),
|
|
50
|
+
"broker_url": "https://logger.app.hyperprobe.co",
|
|
51
|
+
"commit_sha": os.getenv("GIT_COMMIT"),
|
|
52
|
+
})
|
|
50
53
|
```
|
|
51
54
|
|
|
55
|
+
Required identity values may also be supplied through the environment variables
|
|
56
|
+
below. Explicit options take precedence. Initialization failures are reported
|
|
57
|
+
and the application continues without instrumentation.
|
|
58
|
+
|
|
52
59
|
### Configuration Environment Variables
|
|
53
60
|
|
|
54
61
|
Configure the agent using the following environment variables:
|
|
@@ -76,6 +83,7 @@ Configure the agent using the following environment variables:
|
|
|
76
83
|
| `HYPERPROBE_STACK_FRAME_DEPTH` | Maximum captured stack-frame depth. | `3` |
|
|
77
84
|
| `HYPERPROBE_MAX_OBJECT_PROPERTIES` | Maximum serialized properties per object. | `50` |
|
|
78
85
|
| `HYPERPROBE_MAX_STRING_LENGTH` | Maximum serialized string length. | `1024` |
|
|
86
|
+
| `HYPERPROBE_FORK_MODE` | Set to `worker` for preloaded, prefork servers. The parent defers agent initialization and every worker starts fresh process-local state. | `none` |
|
|
79
87
|
|
|
80
88
|
### Debug logging
|
|
81
89
|
|
|
@@ -83,9 +91,9 @@ Set `DEBUG` to a comma- or space-separated list of namespaces. Patterns accept
|
|
|
83
91
|
`*`, and exclusions begin with `-`:
|
|
84
92
|
|
|
85
93
|
```bash
|
|
86
|
-
DEBUG=hyperprobe
|
|
94
|
+
DEBUG=hyperprobe:* # all HyperProbe SDK logs
|
|
87
95
|
DEBUG=hyperprobe:broker,hyperprobe:monitor # selected components
|
|
88
|
-
DEBUG=hyperprobe
|
|
96
|
+
DEBUG=hyperprobe:*,-hyperprobe:evaluator # exclude a component
|
|
89
97
|
DEBUG_COLORS=1 # force ANSI colors
|
|
90
98
|
```
|
|
91
99
|
|
|
@@ -94,26 +102,25 @@ Without `*`, a selector is exact: `DEBUG=hyperprobe` does not match
|
|
|
94
102
|
Set `DEBUG_COLORS=0` to disable colors. Logging configuration is read during
|
|
95
103
|
SDK initialization, so set these variables before starting the process.
|
|
96
104
|
|
|
97
|
-
##
|
|
105
|
+
## Shutdown
|
|
98
106
|
|
|
99
|
-
|
|
107
|
+
Call `HyperProbe.shutdown()` when the application process stops to close
|
|
108
|
+
monitoring, workers, and the broker connection cleanly.
|
|
100
109
|
|
|
101
|
-
|
|
110
|
+
For servers that import the application before forking workers, such as
|
|
111
|
+
Gunicorn with `--preload`, enable worker fork mode:
|
|
102
112
|
|
|
103
113
|
```python
|
|
104
|
-
import os
|
|
105
|
-
from hyperprobe import HyperProbe
|
|
106
|
-
|
|
107
|
-
# Start the agent programmatically at your application's entrypoint
|
|
108
114
|
agent = HyperProbe.start({
|
|
109
115
|
"service_id": "<service-uuid-from-dashboard>",
|
|
110
|
-
"environment": os.getenv("PYTHON_ENV"),
|
|
116
|
+
"environment": os.getenv("PYTHON_ENV"),
|
|
111
117
|
"broker_url": "https://logger.app.hyperprobe.co",
|
|
112
|
-
"commit_sha": os.getenv("GIT_COMMIT"),
|
|
118
|
+
"commit_sha": os.getenv("GIT_COMMIT"),
|
|
119
|
+
"fork_mode": "worker",
|
|
113
120
|
})
|
|
114
|
-
|
|
115
|
-
# Your application code goes here...
|
|
116
|
-
|
|
117
|
-
# To cleanly shut down the agent when your app stops:
|
|
118
|
-
HyperProbe.shutdown()
|
|
119
121
|
```
|
|
122
|
+
|
|
123
|
+
The prefork parent remains agent-free. Every worker receives a unique agent ID,
|
|
124
|
+
telemetry queue, gRPC channel, monitoring engine, and set of background threads.
|
|
125
|
+
Application code running inside those workers must use the `spawn` process start
|
|
126
|
+
method rather than creating additional children with `fork`.
|
|
@@ -14,6 +14,7 @@ from hyperprobe.core.logger import get_logger
|
|
|
14
14
|
from hyperprobe.core.probe_output import ProbeLogWriter
|
|
15
15
|
|
|
16
16
|
logger = get_logger("hyperprobe:agent")
|
|
17
|
+
broker_logger = get_logger("hyperprobe:broker")
|
|
17
18
|
from hyperprobe.protos.agent_pb2 import PROBE_TYPE_UNSPECIFIED,PROBE_TYPE_SNAPSHOT,PROBE_TYPE_LOG,PROBE_TYPE_COUNTER,PROBE_TYPE_METRIC,PROBE_TYPE_DURATION
|
|
18
19
|
|
|
19
20
|
UUID_REGEX = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$", re.IGNORECASE)
|
|
@@ -45,13 +46,26 @@ def is_valid_broker_url(url):
|
|
|
45
46
|
class HyperProbeAgent:
|
|
46
47
|
_instance = None
|
|
47
48
|
_lock = threading.Lock()
|
|
49
|
+
_fork_handlers_registered = False
|
|
50
|
+
_fork_owner_pid = None
|
|
51
|
+
_fork_options = None
|
|
52
|
+
_fork_agent = None
|
|
53
|
+
_fork_pending = False
|
|
48
54
|
|
|
49
55
|
def __init__(self, options):
|
|
50
|
-
self.options = options
|
|
56
|
+
self.options = dict(options)
|
|
57
|
+
self.owner_pid = os.getpid()
|
|
51
58
|
self.agent_id = str(uuid.uuid4())
|
|
52
59
|
self.is_shutdown = False
|
|
53
60
|
self.stop_event = threading.Event()
|
|
54
61
|
|
|
62
|
+
self.fork_mode = str(
|
|
63
|
+
self.options.get("fork_mode")
|
|
64
|
+
or os.getenv("HYPERPROBE_FORK_MODE", "none")
|
|
65
|
+
).strip().lower()
|
|
66
|
+
if self.fork_mode not in {"none", "worker"}:
|
|
67
|
+
raise ValueError('fork_mode must be either "none" or "worker"')
|
|
68
|
+
|
|
55
69
|
# Options & Envs Parsing
|
|
56
70
|
self.sync_interval_sec = (options.get("sync_interval_ms") or int(os.getenv("HYPERPROBE_SYNC_INTERVAL_MS", 60000))) / 1000.0
|
|
57
71
|
self.flush_interval_sec = (options.get("flush_interval_ms") or int(os.getenv("HYPERPROBE_FLUSH_INTERVAL_MS", 1000))) / 1000.0
|
|
@@ -118,8 +132,21 @@ class HyperProbeAgent:
|
|
|
118
132
|
@classmethod
|
|
119
133
|
def start(cls, options):
|
|
120
134
|
"""Starts the global HyperProbe Agent singleton."""
|
|
135
|
+
options = dict(options or {})
|
|
121
136
|
with cls._lock:
|
|
137
|
+
if (
|
|
138
|
+
cls._fork_owner_pid == os.getpid()
|
|
139
|
+
and cls._fork_agent is not None
|
|
140
|
+
):
|
|
141
|
+
logger.forceInfo("[HyperProbe] Worker fork mode is already configured.")
|
|
142
|
+
return cls._fork_agent
|
|
122
143
|
if cls._instance is not None:
|
|
144
|
+
if cls._instance.owner_pid != os.getpid():
|
|
145
|
+
logger.forceError(
|
|
146
|
+
"[HyperProbe] Inherited agent state detected in a forked "
|
|
147
|
+
"process. Configure fork_mode='worker' before forking."
|
|
148
|
+
)
|
|
149
|
+
return None
|
|
123
150
|
logger.forceInfo("[HyperProbe] Agent is already running.")
|
|
124
151
|
return cls._instance
|
|
125
152
|
|
|
@@ -134,12 +161,15 @@ class HyperProbeAgent:
|
|
|
134
161
|
return None
|
|
135
162
|
options["commit_sha"] = commit_sha
|
|
136
163
|
|
|
137
|
-
service_id = options.get("service_id")
|
|
164
|
+
service_id = options.get("service_id") or os.getenv(
|
|
165
|
+
"HYPERPROBE_SERVICE_ID"
|
|
166
|
+
)
|
|
138
167
|
if not service_id or not str(service_id).strip():
|
|
139
168
|
logger.forceError('\033[1m\033[33m⚠️ [HyperProbe] CRITICAL: Failed to start agent. HYPERPROBE_SERVICE_ID is required.\033[0m')
|
|
140
169
|
return None
|
|
141
170
|
if not UUID_REGEX.match(str(service_id).strip()):
|
|
142
171
|
logger.forceInfo('\033[1m\033[33m⚠️ [HyperProbe] WARN: service_id is not a valid UUID, please check again.\033[0m')
|
|
172
|
+
options["service_id"] = service_id
|
|
143
173
|
|
|
144
174
|
environment = options.get("environment") or os.getenv("HYPERPROBE_ENVIRONMENT")
|
|
145
175
|
if not environment or not str(environment).strip():
|
|
@@ -147,26 +177,164 @@ class HyperProbeAgent:
|
|
|
147
177
|
return None
|
|
148
178
|
options["environment"] = environment
|
|
149
179
|
|
|
150
|
-
broker_url = options.get("broker_url")
|
|
180
|
+
broker_url = options.get("broker_url") or os.getenv(
|
|
181
|
+
"HYPERPROBE_BROKER_URL"
|
|
182
|
+
)
|
|
151
183
|
if not is_valid_broker_url(broker_url):
|
|
152
184
|
logger.forceError(f'\033[1m\033[33m⚠️ [HyperProbe] CRITICAL: Failed to start agent. Invalid broker_url "{broker_url}". It must be a valid URL (http:// or https://) or a raw gRPC target (e.g. localhost:60051), with no query parameters and no trailing slash.\033[0m')
|
|
153
185
|
return None
|
|
186
|
+
options["broker_url"] = broker_url
|
|
187
|
+
|
|
188
|
+
fork_mode = str(
|
|
189
|
+
options.get("fork_mode")
|
|
190
|
+
or os.getenv("HYPERPROBE_FORK_MODE", "none")
|
|
191
|
+
).strip().lower()
|
|
192
|
+
if fork_mode not in {"none", "worker"}:
|
|
193
|
+
logger.forceError(
|
|
194
|
+
'[HyperProbe] Invalid fork_mode. Expected "none" or "worker".'
|
|
195
|
+
)
|
|
196
|
+
return None
|
|
197
|
+
options["fork_mode"] = fork_mode
|
|
198
|
+
|
|
199
|
+
if fork_mode == "worker":
|
|
200
|
+
return cls._prepare_worker_fork_mode_locked(options)
|
|
154
201
|
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
202
|
+
instance = cls.__new__(cls)
|
|
203
|
+
try:
|
|
204
|
+
instance.__init__(options)
|
|
205
|
+
cls._instance = instance
|
|
206
|
+
instance._start_background_loops()
|
|
207
|
+
except Exception as exc:
|
|
208
|
+
instance._cleanup_failed_start()
|
|
209
|
+
cls._instance = None
|
|
210
|
+
logger.forceError(
|
|
211
|
+
f"[HyperProbe] Agent failed to initialize "
|
|
212
|
+
f"({type(exc).__name__}: {exc}). "
|
|
213
|
+
"Running application uninstrumented."
|
|
214
|
+
)
|
|
215
|
+
return None
|
|
216
|
+
logger.forceInfo(
|
|
217
|
+
f"[HyperProbe] Python agent successfully started "
|
|
218
|
+
f"(ID: {cls._instance.agent_id}, PID: {cls._instance.owner_pid})."
|
|
219
|
+
)
|
|
158
220
|
return cls._instance
|
|
159
221
|
|
|
222
|
+
@classmethod
|
|
223
|
+
def _prepare_worker_fork_mode_locked(cls, options):
|
|
224
|
+
if not hasattr(os, "register_at_fork"):
|
|
225
|
+
logger.forceError(
|
|
226
|
+
"[HyperProbe] fork_mode='worker' is not supported on this platform."
|
|
227
|
+
)
|
|
228
|
+
return None
|
|
229
|
+
|
|
230
|
+
# Return a stable placeholder to application code imported by the
|
|
231
|
+
# prefork parent. It is initialized in place after each child fork.
|
|
232
|
+
instance = cls.__new__(cls)
|
|
233
|
+
instance.options = dict(options)
|
|
234
|
+
instance.owner_pid = os.getpid()
|
|
235
|
+
instance.agent_id = None
|
|
236
|
+
instance.is_shutdown = True
|
|
237
|
+
instance.fork_mode = "worker"
|
|
238
|
+
|
|
239
|
+
cls._fork_owner_pid = os.getpid()
|
|
240
|
+
cls._fork_options = dict(options)
|
|
241
|
+
cls._fork_agent = instance
|
|
242
|
+
|
|
243
|
+
if cls._fork_handlers_registered:
|
|
244
|
+
logger.forceInfo(
|
|
245
|
+
f"[HyperProbe] Worker fork mode configured (PID: {os.getpid()})."
|
|
246
|
+
)
|
|
247
|
+
return instance
|
|
248
|
+
|
|
249
|
+
os.register_at_fork(
|
|
250
|
+
before=cls._before_fork,
|
|
251
|
+
after_in_parent=cls._after_fork_parent,
|
|
252
|
+
after_in_child=cls._after_fork_child,
|
|
253
|
+
)
|
|
254
|
+
cls._fork_handlers_registered = True
|
|
255
|
+
logger.forceInfo(
|
|
256
|
+
f"[HyperProbe] Worker fork mode configured (PID: {os.getpid()})."
|
|
257
|
+
)
|
|
258
|
+
return instance
|
|
259
|
+
|
|
260
|
+
@classmethod
|
|
261
|
+
def _before_fork(cls):
|
|
262
|
+
if cls._fork_owner_pid != os.getpid() or cls._fork_options is None:
|
|
263
|
+
return
|
|
264
|
+
|
|
265
|
+
cls._lock.acquire()
|
|
266
|
+
cls._fork_pending = True
|
|
267
|
+
|
|
268
|
+
@classmethod
|
|
269
|
+
def _after_fork_parent(cls):
|
|
270
|
+
if not cls._fork_pending:
|
|
271
|
+
return
|
|
272
|
+
cls._fork_pending = False
|
|
273
|
+
cls._lock.release()
|
|
274
|
+
|
|
275
|
+
@classmethod
|
|
276
|
+
def _after_fork_child(cls):
|
|
277
|
+
if not cls._fork_pending:
|
|
278
|
+
return
|
|
279
|
+
|
|
280
|
+
options = dict(cls._fork_options or {})
|
|
281
|
+
instance = cls._fork_agent
|
|
282
|
+
|
|
283
|
+
# Locks and process resources inherited from a multithreaded parent must
|
|
284
|
+
# never be reused in the child.
|
|
285
|
+
cls._lock = threading.Lock()
|
|
286
|
+
cls._instance = None
|
|
287
|
+
cls._fork_pending = False
|
|
288
|
+
cls._fork_owner_pid = None
|
|
289
|
+
cls._fork_options = None
|
|
290
|
+
cls._fork_agent = None
|
|
291
|
+
|
|
292
|
+
if instance is None or not options:
|
|
293
|
+
return
|
|
294
|
+
|
|
295
|
+
try:
|
|
296
|
+
# Reinitialize in place so the object returned by HyperProbe.start()
|
|
297
|
+
# remains valid in application modules imported before the fork.
|
|
298
|
+
instance.__init__(options)
|
|
299
|
+
cls._instance = instance
|
|
300
|
+
instance._start_background_loops()
|
|
301
|
+
except Exception as exc:
|
|
302
|
+
instance._cleanup_failed_start()
|
|
303
|
+
cls._instance = None
|
|
304
|
+
logger.forceError(
|
|
305
|
+
f"[HyperProbe] Failed to restart the agent after fork: {exc}"
|
|
306
|
+
)
|
|
307
|
+
return
|
|
308
|
+
|
|
309
|
+
logger.forceInfo(
|
|
310
|
+
f"[HyperProbe] Python agent started after fork "
|
|
311
|
+
f"(ID: {instance.agent_id}, PID: {instance.owner_pid})."
|
|
312
|
+
)
|
|
313
|
+
|
|
160
314
|
@classmethod
|
|
161
315
|
def shutdown(cls):
|
|
162
316
|
"""Cleanly and safely shuts down the running agent singleton."""
|
|
163
317
|
with cls._lock:
|
|
164
318
|
if cls._instance is None:
|
|
319
|
+
if cls._fork_owner_pid == os.getpid():
|
|
320
|
+
cls._clear_fork_state_locked()
|
|
321
|
+
return
|
|
322
|
+
if cls._instance.owner_pid != os.getpid():
|
|
323
|
+
cls._instance = None
|
|
165
324
|
return
|
|
166
325
|
cls._instance._stop()
|
|
167
326
|
cls._instance = None
|
|
327
|
+
if cls._fork_owner_pid == os.getpid():
|
|
328
|
+
cls._clear_fork_state_locked()
|
|
168
329
|
print("[HyperProbe] Python agent successfully shut down.")
|
|
169
330
|
|
|
331
|
+
@classmethod
|
|
332
|
+
def _clear_fork_state_locked(cls):
|
|
333
|
+
cls._fork_owner_pid = None
|
|
334
|
+
cls._fork_options = None
|
|
335
|
+
cls._fork_agent = None
|
|
336
|
+
cls._fork_pending = False
|
|
337
|
+
|
|
170
338
|
def _init_instrumentation_engine(self):
|
|
171
339
|
from hyperprobe.core.monitoring_engine import MonitoringEngine
|
|
172
340
|
return MonitoringEngine(
|
|
@@ -177,6 +345,8 @@ class HyperProbeAgent:
|
|
|
177
345
|
)
|
|
178
346
|
|
|
179
347
|
def _start_background_loops(self):
|
|
348
|
+
if self.owner_pid != os.getpid():
|
|
349
|
+
return
|
|
180
350
|
self.stop_event.clear()
|
|
181
351
|
self.safety_monitor.start()
|
|
182
352
|
|
|
@@ -195,29 +365,107 @@ class HyperProbeAgent:
|
|
|
195
365
|
def _stop(self):
|
|
196
366
|
self.is_shutdown = True
|
|
197
367
|
self.stop_event.set()
|
|
198
|
-
self.safety_monitor.stop()
|
|
199
368
|
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
369
|
+
try:
|
|
370
|
+
self.safety_monitor.stop()
|
|
371
|
+
except Exception as exc:
|
|
372
|
+
logger.error(f"[HyperProbe] Failed to stop safety monitor: {exc}")
|
|
204
373
|
|
|
205
|
-
|
|
206
|
-
self.
|
|
207
|
-
|
|
208
|
-
|
|
374
|
+
try:
|
|
375
|
+
self.engine.set_probes([])
|
|
376
|
+
except Exception as exc:
|
|
377
|
+
logger.error(f"[HyperProbe] Failed to clear monitoring probes: {exc}")
|
|
378
|
+
try:
|
|
379
|
+
self.engine.close()
|
|
380
|
+
except Exception as exc:
|
|
381
|
+
logger.error(f"[HyperProbe] Failed to stop monitoring engine: {exc}")
|
|
382
|
+
|
|
383
|
+
try:
|
|
384
|
+
self.probe_log_writer.stop(timeout=2.0)
|
|
385
|
+
except Exception as exc:
|
|
386
|
+
logger.error(f"[HyperProbe] Failed to stop probe output: {exc}")
|
|
387
|
+
|
|
388
|
+
cooldown_timer = None
|
|
389
|
+
try:
|
|
390
|
+
with self._agent_lock:
|
|
391
|
+
self._cooldown_generation += 1
|
|
392
|
+
cooldown_timer = self.cooldown_timer
|
|
393
|
+
self.cooldown_timer = None
|
|
394
|
+
except Exception as exc:
|
|
395
|
+
logger.error(f"[HyperProbe] Failed to clear cooldown state: {exc}")
|
|
209
396
|
if cooldown_timer:
|
|
210
|
-
|
|
397
|
+
try:
|
|
398
|
+
cooldown_timer.cancel()
|
|
399
|
+
except Exception:
|
|
400
|
+
pass
|
|
211
401
|
|
|
212
|
-
|
|
402
|
+
try:
|
|
403
|
+
self.broker_client.shutdown()
|
|
404
|
+
except Exception as exc:
|
|
405
|
+
logger.error(f"[HyperProbe] Failed to close broker client: {exc}")
|
|
213
406
|
|
|
214
407
|
current_thread = threading.current_thread()
|
|
215
408
|
for thread in (self.sync_thread, self.flush_thread, self.stats_thread):
|
|
216
|
-
|
|
217
|
-
thread.
|
|
409
|
+
try:
|
|
410
|
+
if thread and thread is not current_thread and thread.is_alive():
|
|
411
|
+
thread.join(timeout=2.0)
|
|
412
|
+
except Exception as exc:
|
|
413
|
+
logger.error(f"[HyperProbe] Failed to join worker thread: {exc}")
|
|
414
|
+
|
|
415
|
+
def _cleanup_failed_start(self):
|
|
416
|
+
"""Best-effort cleanup for a partially initialized child agent."""
|
|
417
|
+
self.is_shutdown = True
|
|
418
|
+
|
|
419
|
+
stop_event = getattr(self, "stop_event", None)
|
|
420
|
+
if stop_event is not None:
|
|
421
|
+
stop_event.set()
|
|
422
|
+
|
|
423
|
+
for component_name in ("safety_monitor", "probe_log_writer"):
|
|
424
|
+
component = getattr(self, component_name, None)
|
|
425
|
+
if component is None:
|
|
426
|
+
continue
|
|
427
|
+
try:
|
|
428
|
+
component.stop()
|
|
429
|
+
except Exception:
|
|
430
|
+
pass
|
|
431
|
+
|
|
432
|
+
engine = getattr(self, "engine", None)
|
|
433
|
+
if engine is not None:
|
|
434
|
+
try:
|
|
435
|
+
engine.close()
|
|
436
|
+
except Exception:
|
|
437
|
+
pass
|
|
438
|
+
|
|
439
|
+
broker_client = getattr(self, "broker_client", None)
|
|
440
|
+
if broker_client is not None:
|
|
441
|
+
try:
|
|
442
|
+
broker_client.shutdown()
|
|
443
|
+
except Exception:
|
|
444
|
+
pass
|
|
445
|
+
|
|
446
|
+
cooldown_timer = getattr(self, "cooldown_timer", None)
|
|
447
|
+
if cooldown_timer is not None:
|
|
448
|
+
try:
|
|
449
|
+
cooldown_timer.cancel()
|
|
450
|
+
except Exception:
|
|
451
|
+
pass
|
|
452
|
+
self.cooldown_timer = None
|
|
453
|
+
if hasattr(self, "_cooldown_generation"):
|
|
454
|
+
self._cooldown_generation += 1
|
|
455
|
+
|
|
456
|
+
current_thread = threading.current_thread()
|
|
457
|
+
for thread_name in ("sync_thread", "flush_thread", "stats_thread"):
|
|
458
|
+
thread = getattr(self, thread_name, None)
|
|
459
|
+
if thread is None or thread is current_thread:
|
|
460
|
+
continue
|
|
461
|
+
try:
|
|
462
|
+
if thread.is_alive():
|
|
463
|
+
thread.join(timeout=2.0)
|
|
464
|
+
except Exception:
|
|
465
|
+
pass
|
|
218
466
|
|
|
219
467
|
def _sync_loop(self):
|
|
220
|
-
while not self.stop_event.is_set():
|
|
468
|
+
while self.owner_pid == os.getpid() and not self.stop_event.is_set():
|
|
221
469
|
try:
|
|
222
470
|
self._sync_with_broker()
|
|
223
471
|
except Exception as e:
|
|
@@ -277,7 +525,7 @@ class HyperProbeAgent:
|
|
|
277
525
|
self.engine.set_probes(to_apply)
|
|
278
526
|
|
|
279
527
|
def _flush_loop(self):
|
|
280
|
-
while not self.stop_event.is_set():
|
|
528
|
+
while self.owner_pid == os.getpid() and not self.stop_event.is_set():
|
|
281
529
|
try:
|
|
282
530
|
self._flush_telemetry()
|
|
283
531
|
except Exception as e:
|
|
@@ -285,7 +533,7 @@ class HyperProbeAgent:
|
|
|
285
533
|
self.stop_event.wait(self.flush_interval_sec)
|
|
286
534
|
|
|
287
535
|
def _stats_loop(self):
|
|
288
|
-
while not self.stop_event.is_set():
|
|
536
|
+
while self.owner_pid == os.getpid() and not self.stop_event.is_set():
|
|
289
537
|
try:
|
|
290
538
|
stats = self.engine.get_stats()
|
|
291
539
|
if stats["hits"] > 0 or stats["skips"] > 0:
|
|
@@ -313,12 +561,18 @@ class HyperProbeAgent:
|
|
|
313
561
|
return
|
|
314
562
|
|
|
315
563
|
self._inflight_batch = batch
|
|
564
|
+
broker_logger.info(
|
|
565
|
+
f"[HyperProbe] Flushing {len(batch)} telemetry events to broker..."
|
|
566
|
+
)
|
|
316
567
|
try:
|
|
317
568
|
finished_probe_ids = self.broker_client.report_telemetry(
|
|
318
569
|
[item.event for item in batch]
|
|
319
570
|
)
|
|
571
|
+
broker_logger.info(
|
|
572
|
+
f"[HyperProbe] Successfully flushed {len(batch)} telemetry events to broker."
|
|
573
|
+
)
|
|
320
574
|
except Exception as e:
|
|
321
|
-
|
|
575
|
+
broker_logger.error(
|
|
322
576
|
f"[HyperProbe] Failed to flush telemetry: {str(e)}. "
|
|
323
577
|
f"Retaining {len(batch)} events for retry."
|
|
324
578
|
)
|
|
@@ -378,6 +632,11 @@ class HyperProbeAgent:
|
|
|
378
632
|
return list(self.active_probes.values())
|
|
379
633
|
|
|
380
634
|
def _handle_capture(self, event):
|
|
635
|
+
if (
|
|
636
|
+
getattr(self, "owner_pid", os.getpid()) != os.getpid()
|
|
637
|
+
or getattr(self, "is_shutdown", False)
|
|
638
|
+
):
|
|
639
|
+
return
|
|
381
640
|
probe_id = event["probe_id"]
|
|
382
641
|
with self._agent_lock:
|
|
383
642
|
probe = self.active_probes.get(probe_id)
|
|
@@ -449,7 +708,7 @@ class HyperProbeAgent:
|
|
|
449
708
|
self.engine.set_probes(probes_list)
|
|
450
709
|
|
|
451
710
|
def _handle_health_change(self, health, reason):
|
|
452
|
-
if self.is_shutdown:
|
|
711
|
+
if self.owner_pid != os.getpid() or self.is_shutdown:
|
|
453
712
|
return
|
|
454
713
|
|
|
455
714
|
if health == AgentHealth.RED:
|