pi-deadman 1.0.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -73
- package/extensions/canary.ts +3 -2
- package/extensions/monitor.ts +2 -0
- package/extensions/processes.ts +1 -0
- package/extensions/signals.ts +4 -0
- package/package.json +18 -10
- package/__tests__/calibration.test.ts +0 -73
- package/__tests__/canary.test.ts +0 -68
- package/__tests__/fast-watchdog.test.ts +0 -188
- package/__tests__/index.test.ts +0 -103
- package/__tests__/keywords.test.ts +0 -130
- package/__tests__/logging.test.ts +0 -128
- package/__tests__/monitor.test.ts +0 -115
- package/__tests__/processes.test.ts +0 -74
- package/__tests__/signals.test.ts +0 -59
- package/__tests__/tree.test.ts +0 -327
- package/__tests__/watchdog.test.ts +0 -421
- package/__tests__/worker.test.ts +0 -85
- package/__tests__/zones.test.ts +0 -182
- package/tsconfig.json +0 -17
- package/vitest.config.ts +0 -10
package/README.md
CHANGED
|
@@ -4,105 +4,65 @@ Dead man's switch for AI coding agents.
|
|
|
4
4
|
|
|
5
5
|
Monitors macOS memory pressure, gates heavy operations, and auto-kills runaway processes before your system locks up.
|
|
6
6
|
|
|
7
|
-
## The Problem
|
|
8
|
-
|
|
9
|
-
AI coding agents run builds, tests, installs, and browser automation with no awareness of system memory state. On an 8 GB Mac, this regularly pushes the system into swap thrashing — minutes of lag, system freezes, sometimes requiring a hard reboot.
|
|
10
|
-
|
|
11
7
|
## Install
|
|
12
8
|
|
|
13
9
|
```bash
|
|
14
|
-
pi install
|
|
10
|
+
pi install npm:pi-deadman
|
|
15
11
|
```
|
|
16
12
|
|
|
17
|
-
On first run,
|
|
13
|
+
On first run, it calibrates a baseline for your machine (~10 seconds). After that, it runs in the background — no configuration needed.
|
|
18
14
|
|
|
19
15
|
**macOS only.** Silent no-op on other platforms.
|
|
20
16
|
|
|
21
17
|
## How It Works
|
|
22
18
|
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
Every `bash` tool call goes through a tier × zone matrix:
|
|
26
|
-
|
|
27
|
-
| | GREEN | YELLOW | ORANGE | RED |
|
|
28
|
-
|---|---|---|---|---|
|
|
29
|
-
| **Tier 0–1** (read-only: ls, cat, grep) | ✅ | ✅ | ✅ | ❌ |
|
|
30
|
-
| **Tier 2** (medium: test, server, spawn) | ✅ | ✅ | ✅ | ❌ |
|
|
31
|
-
| **Tier 3** (heavy: npm install, docker, build) | ✅ | ✅ | ❌ | ❌ |
|
|
32
|
-
| **Tier 4** (destructive: kill, pkill, rm -rf) | ✅ | ✅ | ❌ | ❌ |
|
|
33
|
-
|
|
34
|
-
When blocked in **ORANGE**, you choose: run anyway or free memory first.
|
|
35
|
-
When blocked in **RED**, you must kill a process to proceed — or force-run after the first attempt.
|
|
19
|
+
**Zones** — Polls system health and classifies into GREEN, YELLOW, ORANGE, or RED based on memory pressure signals.
|
|
36
20
|
|
|
37
|
-
|
|
21
|
+
**Gate** — Every `bash` tool call is checked against the current zone. Light commands always pass; heavy operations (builds, installs, docker) are blocked in ORANGE and RED.
|
|
38
22
|
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
- **Canary test** — micro-ops (array sort, Map ops, regex, JSON parse, Buffer alloc) timed via `performance.now()`. Slowdown ratio vs baseline determines zone.
|
|
42
|
-
- **System signals** — swap usage, swap in/out rates, compression ratio, memorystatus level via `sysctl` and `vm_stat`.
|
|
43
|
-
- **Process snapshots** — footprint (true memory) via `proc_pid_rusage` Python helper, stored in a ring buffer of 10 snapshots.
|
|
44
|
-
|
|
45
|
-
### Watchdog (Confirmed RED)
|
|
46
|
-
|
|
47
|
-
When the system enters confirmed RED (3 consecutive RED polls), the watchdog auto-kills processes across **all pi sessions**. Processes with < 50 MB footprint are never kill candidates.
|
|
48
|
-
|
|
49
|
-
| Priority | Signal | Catches |
|
|
50
|
-
|---|---|---|
|
|
51
|
-
| 1. **Growing** | ≥100 MB delta across 3+ of 10 snapshots | Memory leaks, sawtooth patterns |
|
|
52
|
-
| 2. **Swarm** | ≥3 same-name processes, combined ≥500 MB | Worker pools (vitest ×7, webpack workers) |
|
|
53
|
-
| 3. **Heavy & young** | Age < 10 min AND footprint ≥ 200 MB | Burst allocators |
|
|
54
|
-
| 4. **Newest** | Appeared after last stable non-RED state | Temporal correlation (largest only) |
|
|
55
|
-
| 5. **No match** | Block commands, wait | Pressure from outside pi's tree |
|
|
56
|
-
|
|
57
|
-
Two execution paths:
|
|
58
|
-
- **Slow poll** — full canary + signals + footprint worker. Populates snapshot ring buffer.
|
|
59
|
-
- **Fast watchdog** — independent 2s loop using `ps` (~17ms). Parses fresh `etime` for age. Walks children of ALL pi instances (cross-session). Acts even when the slow poll is stuck during system thrashing.
|
|
23
|
+
**Watchdog** — In confirmed RED, automatically identifies and kills the process most likely causing the pressure. Runs independently so it works even when the system is thrashing.
|
|
60
24
|
|
|
61
25
|
## Commands
|
|
62
26
|
|
|
63
27
|
| Command | Description |
|
|
64
28
|
|---|---|
|
|
65
|
-
| `/deadman` | Show current zone
|
|
29
|
+
| `/deadman` | Show current zone and memory stats |
|
|
66
30
|
|
|
67
|
-
##
|
|
68
|
-
|
|
69
|
-
```
|
|
70
|
-
pi-deadman/
|
|
71
|
-
├── extensions/
|
|
72
|
-
│ ├── index.ts — Entry point: tool_call gate, /deadman command
|
|
73
|
-
│ ├── monitor.ts — Background polling, adaptive intervals, watchdog
|
|
74
|
-
│ ├── canary.ts — Performance micro-ops timing
|
|
75
|
-
│ ├── signals.ts — macOS kernel metrics (sysctl, vm_stat)
|
|
76
|
-
│ ├── zones.ts — Zone classification (GREEN/YELLOW/ORANGE/RED)
|
|
77
|
-
│ ├── calibration.ts — Baseline persistence
|
|
78
|
-
│ ├── keywords.ts — Command tier classification (25+ keywords, 5 tiers)
|
|
79
|
-
│ ├── processes.ts — System-wide process list (footprint.py)
|
|
80
|
-
│ ├── watchdog.ts — Kill target selection
|
|
81
|
-
│ ├── tree.ts — Snapshot diffing, growth detection, swarm detection
|
|
82
|
-
│ ├── worker.ts — Persistent Python worker for fast footprint queries
|
|
83
|
-
│ └── logging.ts — JSONL structured logs (GC after 3 days)
|
|
84
|
-
├── helpers/
|
|
85
|
-
│ ├── footprint.py — proc_pid_rusage footprint extraction
|
|
86
|
-
│ └── footprint_worker.py — Persistent worker process
|
|
87
|
-
└── __tests__/ — 202 tests across 13 files
|
|
88
|
-
```
|
|
31
|
+
## Security & System Calls
|
|
89
32
|
|
|
90
|
-
|
|
33
|
+
pi-deadman makes **no network requests** and reads **no environment variables**. All system access is local and documented here:
|
|
91
34
|
|
|
92
|
-
|
|
35
|
+
**Shell commands** (all read-only queries, no mutations):
|
|
36
|
+
| Command | File | Purpose |
|
|
37
|
+
|---|---|---|
|
|
38
|
+
| `sysctl -n kern.ostype` | canary.ts | Canary timing benchmark |
|
|
39
|
+
| `/usr/bin/true` | canary.ts | Canary timing benchmark |
|
|
40
|
+
| `sysctl -n vm.swapusage` | signals.ts | Read swap usage stats |
|
|
41
|
+
| `sysctl -n kern.memorystatus_vm_pressure_level` | signals.ts | Read memory pressure level |
|
|
42
|
+
| `sysctl -n kern.memorystatus_level` | signals.ts | Read memorystatus level |
|
|
43
|
+
| `vm_stat` | signals.ts | Read VM page statistics |
|
|
44
|
+
| `ps -eo pid,rss,comm -r` | processes.ts | List processes by memory |
|
|
45
|
+
| `ps -eo pid,ppid,etime,comm` | monitor.ts | Process tree for watchdog |
|
|
46
|
+
| `python3 helpers/footprint.py` | processes.ts | Read process footprints via `proc_pid_rusage` |
|
|
47
|
+
| `python3 helpers/footprint_worker.py` | worker.ts | Persistent worker for fast footprint queries |
|
|
48
|
+
|
|
49
|
+
**Process kills** (only pi's own child processes, only in confirmed RED):
|
|
50
|
+
| Signal | File | Trigger |
|
|
51
|
+
|---|---|---|
|
|
52
|
+
| `SIGKILL` | monitor.ts | Watchdog auto-kill in confirmed RED zone |
|
|
53
|
+
| `SIGTERM` | processes.ts | User-initiated kill from interactive menu |
|
|
93
54
|
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
55
|
+
**Filesystem writes** (scoped to `~/.pi/deadman/` only):
|
|
56
|
+
| What | File | Purpose |
|
|
57
|
+
|---|---|---|
|
|
58
|
+
| `~/.pi/deadman/baseline.json` | calibration.ts | Persisted canary baseline |
|
|
59
|
+
| `~/.pi/deadman/logs/*.jsonl` | logging.ts | Structured decision/system logs (auto-GC after 3 days) |
|
|
98
60
|
|
|
99
61
|
## Development
|
|
100
62
|
|
|
101
63
|
```bash
|
|
102
|
-
cd pi-deadman
|
|
103
64
|
npm install
|
|
104
|
-
npm test
|
|
105
|
-
npx vitest --watch # watch mode
|
|
65
|
+
npm test
|
|
106
66
|
```
|
|
107
67
|
|
|
108
68
|
## License
|
package/extensions/canary.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// canary.ts — 5 micro-operations timed in sequence for system degradation detection
|
|
2
2
|
import { execSync, spawnSync } from 'child_process';
|
|
3
3
|
import { readFileSync, readdirSync } from 'fs';
|
|
4
|
+
import { fileURLToPath } from 'url';
|
|
4
5
|
|
|
5
6
|
export interface CanaryResult {
|
|
6
7
|
sysctl_ms: number;
|
|
@@ -22,9 +23,9 @@ export async function runCanary(): Promise<CanaryResult> {
|
|
|
22
23
|
spawnSync('/usr/bin/true');
|
|
23
24
|
const spawn_ms = performance.now() - spawn_start;
|
|
24
25
|
|
|
25
|
-
// read_ms: Read
|
|
26
|
+
// read_ms: Read this file as a disk I/O benchmark
|
|
26
27
|
const read_start = performance.now();
|
|
27
|
-
readFileSync(
|
|
28
|
+
readFileSync(fileURLToPath(import.meta.url));
|
|
28
29
|
const read_ms = performance.now() - read_start;
|
|
29
30
|
|
|
30
31
|
// dir_ms: Read `/tmp` directory via `fs.readdirSync` and time it
|
package/extensions/monitor.ts
CHANGED
|
@@ -251,6 +251,7 @@ export class Monitor {
|
|
|
251
251
|
);
|
|
252
252
|
|
|
253
253
|
if (decision) {
|
|
254
|
+
// Auto-kill: SIGKILL pi's own child processes in confirmed RED zone
|
|
254
255
|
for (const target of decision.targets) {
|
|
255
256
|
try {
|
|
256
257
|
process.kill(target.pid, "SIGKILL");
|
|
@@ -411,6 +412,7 @@ export class Monitor {
|
|
|
411
412
|
);
|
|
412
413
|
|
|
413
414
|
if (decision) {
|
|
415
|
+
// Fast watchdog auto-kill: SIGKILL pi's child processes in confirmed RED
|
|
414
416
|
for (const target of decision.targets) {
|
|
415
417
|
try {
|
|
416
418
|
process.kill(target.pid, "SIGKILL");
|
package/extensions/processes.ts
CHANGED
|
@@ -84,6 +84,7 @@ export function formatProcessList(processes: ProcessInfo[], limit: number): stri
|
|
|
84
84
|
.map(proc => `${proc.name} (PID ${proc.pid}) — ${proc.footprint_mb} MB`);
|
|
85
85
|
}
|
|
86
86
|
|
|
87
|
+
// User-initiated kill from interactive menu — graceful SIGTERM
|
|
87
88
|
export function killProcess(pid: number): boolean {
|
|
88
89
|
try {
|
|
89
90
|
process.kill(pid, "SIGTERM");
|
package/extensions/signals.ts
CHANGED
|
@@ -76,6 +76,7 @@ export async function collectSignals(): Promise<SystemSignals> {
|
|
|
76
76
|
};
|
|
77
77
|
}
|
|
78
78
|
|
|
79
|
+
// Read macOS virtual memory page statistics (pages paged in/out, compressed, etc.)
|
|
79
80
|
function getVmStat(): Record<string, number> {
|
|
80
81
|
try {
|
|
81
82
|
const output = execSync("vm_stat", { encoding: "utf8", timeout: 5000 });
|
|
@@ -98,6 +99,7 @@ function getVmStat(): Record<string, number> {
|
|
|
98
99
|
}
|
|
99
100
|
}
|
|
100
101
|
|
|
102
|
+
// Read macOS memory pressure level: 1=normal, 2=warn, 4=critical
|
|
101
103
|
function getPressureLevel(): number {
|
|
102
104
|
try {
|
|
103
105
|
const output = execSync("sysctl -n kern.memorystatus_vm_pressure_level", {
|
|
@@ -114,6 +116,7 @@ function getPressureLevel(): number {
|
|
|
114
116
|
}
|
|
115
117
|
}
|
|
116
118
|
|
|
119
|
+
// Read macOS memorystatus percentage (0-100, higher = more available)
|
|
117
120
|
function getMemorystatusLevel(): number {
|
|
118
121
|
try {
|
|
119
122
|
const output = execSync("sysctl -n kern.memorystatus_level", {
|
|
@@ -127,6 +130,7 @@ function getMemorystatusLevel(): number {
|
|
|
127
130
|
}
|
|
128
131
|
}
|
|
129
132
|
|
|
133
|
+
// Read macOS swap usage: total and used bytes
|
|
130
134
|
function getSwapUsage(): [number, number] {
|
|
131
135
|
try {
|
|
132
136
|
const output = execSync("sysctl -n vm.swapusage", {
|
package/package.json
CHANGED
|
@@ -1,24 +1,32 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-deadman",
|
|
3
|
-
"version": "1.
|
|
4
|
-
"description": "Dead man's switch for AI coding agents — monitors
|
|
3
|
+
"version": "1.2.0",
|
|
4
|
+
"description": "Dead man's switch for AI coding agents — monitors memory pressure, gates heavy operations, and kills runaway processes.",
|
|
5
5
|
"type": "module",
|
|
6
|
-
"keywords": [
|
|
6
|
+
"keywords": [
|
|
7
|
+
"pi-package",
|
|
8
|
+
"memory",
|
|
9
|
+
"macos",
|
|
10
|
+
"watchdog",
|
|
11
|
+
"system-health",
|
|
12
|
+
"deadman"
|
|
13
|
+
],
|
|
7
14
|
"pi": {
|
|
8
|
-
"extensions": [
|
|
15
|
+
"extensions": [
|
|
16
|
+
"./extensions/index.ts"
|
|
17
|
+
]
|
|
9
18
|
},
|
|
10
19
|
"scripts": {
|
|
11
20
|
"test": "vitest run",
|
|
12
21
|
"test:watch": "vitest"
|
|
13
22
|
},
|
|
14
23
|
"devDependencies": {
|
|
15
|
-
"vitest": "^3.0.0",
|
|
16
24
|
"@types/node": "^22.0.0",
|
|
17
|
-
"typescript": "^5.7.0"
|
|
25
|
+
"typescript": "^5.7.0",
|
|
26
|
+
"vitest": "^3.0.0"
|
|
18
27
|
},
|
|
19
|
-
"
|
|
20
|
-
"
|
|
21
|
-
|
|
22
|
-
"os": ["darwin"],
|
|
28
|
+
"os": [
|
|
29
|
+
"darwin"
|
|
30
|
+
],
|
|
23
31
|
"license": "MIT"
|
|
24
32
|
}
|
|
@@ -1,73 +0,0 @@
|
|
|
1
|
-
// __tests__/calibration.test.ts
|
|
2
|
-
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
|
3
|
-
import { saveBaseline, loadBaseline, type Baseline, DEFAULT_BASELINE_MS } from "../extensions/calibration";
|
|
4
|
-
import * as fs from "node:fs";
|
|
5
|
-
import * as path from "node:path";
|
|
6
|
-
import * as os from "node:os";
|
|
7
|
-
|
|
8
|
-
describe("Baseline persistence", () => {
|
|
9
|
-
let tmpDir: string;
|
|
10
|
-
let baselinePath: string;
|
|
11
|
-
|
|
12
|
-
beforeEach(() => {
|
|
13
|
-
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "deadman-test-"));
|
|
14
|
-
baselinePath = path.join(tmpDir, "baseline.json");
|
|
15
|
-
});
|
|
16
|
-
|
|
17
|
-
afterEach(() => {
|
|
18
|
-
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
19
|
-
});
|
|
20
|
-
|
|
21
|
-
it("saves and loads a baseline", () => {
|
|
22
|
-
const baseline: Baseline = {
|
|
23
|
-
canary_ms: 7.5,
|
|
24
|
-
calibrated_at: "2026-02-28T10:00:00.000Z",
|
|
25
|
-
source: "calibrated",
|
|
26
|
-
};
|
|
27
|
-
saveBaseline(baseline, baselinePath);
|
|
28
|
-
const loaded = loadBaseline(baselinePath);
|
|
29
|
-
expect(loaded).toEqual(baseline);
|
|
30
|
-
});
|
|
31
|
-
|
|
32
|
-
it("returns null for missing file", () => {
|
|
33
|
-
expect(loadBaseline("/nonexistent/path/baseline.json")).toBeNull();
|
|
34
|
-
});
|
|
35
|
-
|
|
36
|
-
it("returns null for corrupted JSON", () => {
|
|
37
|
-
fs.writeFileSync(baselinePath, "not json{{{");
|
|
38
|
-
expect(loadBaseline(baselinePath)).toBeNull();
|
|
39
|
-
});
|
|
40
|
-
|
|
41
|
-
it("returns null for incomplete data", () => {
|
|
42
|
-
fs.writeFileSync(baselinePath, JSON.stringify({ canary_ms: 5 }));
|
|
43
|
-
expect(loadBaseline(baselinePath)).toBeNull();
|
|
44
|
-
});
|
|
45
|
-
|
|
46
|
-
it("creates parent directories if needed", () => {
|
|
47
|
-
const deepPath = path.join(tmpDir, "a", "b", "c", "baseline.json");
|
|
48
|
-
const baseline: Baseline = {
|
|
49
|
-
canary_ms: 8.0,
|
|
50
|
-
calibrated_at: "2026-02-28T10:00:00.000Z",
|
|
51
|
-
source: "calibrated",
|
|
52
|
-
};
|
|
53
|
-
saveBaseline(baseline, deepPath);
|
|
54
|
-
const loaded = loadBaseline(deepPath);
|
|
55
|
-
expect(loaded).toEqual(baseline);
|
|
56
|
-
});
|
|
57
|
-
|
|
58
|
-
it("default baseline backward-compatible with missing source field", () => {
|
|
59
|
-
fs.writeFileSync(baselinePath, JSON.stringify({
|
|
60
|
-
canary_ms: 9.0,
|
|
61
|
-
calibrated_at: "2026-02-28T10:00:00.000Z",
|
|
62
|
-
}));
|
|
63
|
-
const loaded = loadBaseline(baselinePath);
|
|
64
|
-
expect(loaded).not.toBeNull();
|
|
65
|
-
expect(loaded!.source).toBe("calibrated");
|
|
66
|
-
});
|
|
67
|
-
});
|
|
68
|
-
|
|
69
|
-
describe("DEFAULT_BASELINE_MS", () => {
|
|
70
|
-
it("is 10ms (conservative default)", () => {
|
|
71
|
-
expect(DEFAULT_BASELINE_MS).toBe(10.0);
|
|
72
|
-
});
|
|
73
|
-
});
|
package/__tests__/canary.test.ts
DELETED
|
@@ -1,68 +0,0 @@
|
|
|
1
|
-
// __tests__/canary.test.ts
|
|
2
|
-
import { describe, it, expect } from "vitest";
|
|
3
|
-
import { runCanary, type CanaryResult } from "../extensions/canary";
|
|
4
|
-
|
|
5
|
-
describe("runCanary", () => {
|
|
6
|
-
it("returns a CanaryResult with all 5 sub-timings", async () => {
|
|
7
|
-
const result = await runCanary();
|
|
8
|
-
expect(result).toHaveProperty("sysctl_ms");
|
|
9
|
-
expect(result).toHaveProperty("spawn_ms");
|
|
10
|
-
expect(result).toHaveProperty("read_ms");
|
|
11
|
-
expect(result).toHaveProperty("dir_ms");
|
|
12
|
-
expect(result).toHaveProperty("alloc_ms");
|
|
13
|
-
expect(result).toHaveProperty("total_ms");
|
|
14
|
-
});
|
|
15
|
-
|
|
16
|
-
it("all timings are positive numbers", async () => {
|
|
17
|
-
const result = await runCanary();
|
|
18
|
-
expect(result.sysctl_ms).toBeGreaterThan(0);
|
|
19
|
-
expect(result.spawn_ms).toBeGreaterThan(0);
|
|
20
|
-
expect(result.read_ms).toBeGreaterThan(0);
|
|
21
|
-
expect(result.dir_ms).toBeGreaterThan(0);
|
|
22
|
-
expect(result.alloc_ms).toBeGreaterThan(0);
|
|
23
|
-
expect(result.total_ms).toBeGreaterThan(0);
|
|
24
|
-
});
|
|
25
|
-
|
|
26
|
-
it("total_ms equals sum of 5 sub-timings", async () => {
|
|
27
|
-
const result = await runCanary();
|
|
28
|
-
const sum = result.sysctl_ms + result.spawn_ms + result.read_ms + result.dir_ms + result.alloc_ms;
|
|
29
|
-
expect(result.total_ms).toBeCloseTo(sum, 1);
|
|
30
|
-
});
|
|
31
|
-
|
|
32
|
-
it("completes in under 500ms on a healthy system", async () => {
|
|
33
|
-
const result = await runCanary();
|
|
34
|
-
expect(result.total_ms).toBeLessThan(500);
|
|
35
|
-
});
|
|
36
|
-
|
|
37
|
-
it("sysctl_ms reads kern.ostype successfully", async () => {
|
|
38
|
-
const result = await runCanary();
|
|
39
|
-
expect(result.sysctl_ms).toBeLessThan(50);
|
|
40
|
-
});
|
|
41
|
-
|
|
42
|
-
it("spawn_ms spawns a real process", async () => {
|
|
43
|
-
const result = await runCanary();
|
|
44
|
-
expect(result.spawn_ms).toBeLessThan(100);
|
|
45
|
-
});
|
|
46
|
-
|
|
47
|
-
it("read_ms reads a real file", async () => {
|
|
48
|
-
const result = await runCanary();
|
|
49
|
-
expect(result.read_ms).toBeLessThan(50);
|
|
50
|
-
});
|
|
51
|
-
|
|
52
|
-
it("dir_ms scans a real directory", async () => {
|
|
53
|
-
const result = await runCanary();
|
|
54
|
-
expect(result.dir_ms).toBeLessThan(50);
|
|
55
|
-
});
|
|
56
|
-
|
|
57
|
-
it("alloc_ms allocates and fills memory", async () => {
|
|
58
|
-
const result = await runCanary();
|
|
59
|
-
expect(result.alloc_ms).toBeLessThan(50);
|
|
60
|
-
});
|
|
61
|
-
|
|
62
|
-
it("is deterministic-ish — two runs produce similar results", async () => {
|
|
63
|
-
const r1 = await runCanary();
|
|
64
|
-
const r2 = await runCanary();
|
|
65
|
-
expect(r2.total_ms).toBeLessThan(r1.total_ms * 5);
|
|
66
|
-
expect(r1.total_ms).toBeLessThan(r2.total_ms * 5);
|
|
67
|
-
});
|
|
68
|
-
});
|
|
@@ -1,188 +0,0 @@
|
|
|
1
|
-
// __tests__/fast-watchdog.test.ts — integration test for the fast watchdog loop
|
|
2
|
-
// Spawns a real child process, seeds growth evidence, forces RED, verifies kill
|
|
3
|
-
|
|
4
|
-
import { describe, it, expect, beforeEach, afterEach } from "vitest";
|
|
5
|
-
import { Monitor } from "../extensions/monitor";
|
|
6
|
-
import { Zone } from "../extensions/zones";
|
|
7
|
-
import { spawn, type ChildProcess } from "node:child_process";
|
|
8
|
-
import * as fs from "node:fs";
|
|
9
|
-
import * as path from "node:path";
|
|
10
|
-
import * as os from "node:os";
|
|
11
|
-
|
|
12
|
-
describe("Fast Watchdog Integration", () => {
|
|
13
|
-
let tmpDir: string;
|
|
14
|
-
let baselinePath: string;
|
|
15
|
-
let logDir: string;
|
|
16
|
-
let monitor: Monitor;
|
|
17
|
-
let childProc: ChildProcess | null = null;
|
|
18
|
-
|
|
19
|
-
beforeEach(() => {
|
|
20
|
-
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "deadman-watchdog-int-"));
|
|
21
|
-
baselinePath = path.join(tmpDir, "baseline.json");
|
|
22
|
-
logDir = path.join(tmpDir, "logs");
|
|
23
|
-
|
|
24
|
-
fs.mkdirSync(path.dirname(baselinePath), { recursive: true });
|
|
25
|
-
fs.writeFileSync(baselinePath, JSON.stringify({
|
|
26
|
-
canary_ms: 5.0,
|
|
27
|
-
calibrated_at: new Date().toISOString(),
|
|
28
|
-
source: "test",
|
|
29
|
-
}));
|
|
30
|
-
|
|
31
|
-
monitor = new Monitor({ baselinePath, logDir });
|
|
32
|
-
});
|
|
33
|
-
|
|
34
|
-
afterEach(() => {
|
|
35
|
-
monitor.stop();
|
|
36
|
-
if (childProc && !childProc.killed) {
|
|
37
|
-
childProc.kill("SIGKILL");
|
|
38
|
-
}
|
|
39
|
-
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
40
|
-
});
|
|
41
|
-
|
|
42
|
-
/**
|
|
43
|
-
* Seed the watchdog's snapshot history with fake growth data for a PID.
|
|
44
|
-
* This simulates the monitor having observed the process growing over time.
|
|
45
|
-
*/
|
|
46
|
-
function seedGrowthHistory(pid: number, name: string) {
|
|
47
|
-
const state = monitor.getWatchdogState();
|
|
48
|
-
state.snapshotHistory = [
|
|
49
|
-
[{ pid, name, footprint_mb: 200, age_seconds: 10 }],
|
|
50
|
-
[{ pid, name, footprint_mb: 400, age_seconds: 15 }],
|
|
51
|
-
[{ pid, name, footprint_mb: 600, age_seconds: 20 }],
|
|
52
|
-
[{ pid, name, footprint_mb: 800, age_seconds: 25 }],
|
|
53
|
-
];
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
it("kills a growing child process within 6 seconds of confirmed RED", async () => {
|
|
57
|
-
childProc = spawn("sleep", ["60"], { stdio: "ignore" });
|
|
58
|
-
const childPid = childProc.pid!;
|
|
59
|
-
expect(childPid).toBeGreaterThan(0);
|
|
60
|
-
|
|
61
|
-
const kills: any[] = [];
|
|
62
|
-
monitor.onAutoKill((decision) => {
|
|
63
|
-
kills.push(decision);
|
|
64
|
-
});
|
|
65
|
-
|
|
66
|
-
monitor.start();
|
|
67
|
-
await new Promise(r => setTimeout(r, 1500));
|
|
68
|
-
|
|
69
|
-
// Seed growth evidence for the child — simulates monitor having tracked it
|
|
70
|
-
seedGrowthHistory(childPid, "sleep");
|
|
71
|
-
|
|
72
|
-
// Force RED confirmed
|
|
73
|
-
monitor._forceZone(Zone.RED, true);
|
|
74
|
-
|
|
75
|
-
// Wait up to 6 seconds for the fast watchdog to fire
|
|
76
|
-
const deadline = Date.now() + 6000;
|
|
77
|
-
while (Date.now() < deadline) {
|
|
78
|
-
await new Promise(r => setTimeout(r, 200));
|
|
79
|
-
try {
|
|
80
|
-
process.kill(childPid, 0);
|
|
81
|
-
} catch {
|
|
82
|
-
break;
|
|
83
|
-
}
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
let alive = true;
|
|
87
|
-
try {
|
|
88
|
-
process.kill(childPid, 0);
|
|
89
|
-
} catch {
|
|
90
|
-
alive = false;
|
|
91
|
-
}
|
|
92
|
-
|
|
93
|
-
expect(alive).toBe(false);
|
|
94
|
-
expect(kills.length).toBeGreaterThanOrEqual(1);
|
|
95
|
-
expect(kills[0].targets.some((t: any) => t.pid === childPid)).toBe(true);
|
|
96
|
-
|
|
97
|
-
// Verify decision log
|
|
98
|
-
const decisionsPath = path.join(logDir, "decisions.jsonl");
|
|
99
|
-
expect(fs.existsSync(decisionsPath)).toBe(true);
|
|
100
|
-
const entries = fs.readFileSync(decisionsPath, "utf-8").trim().split("\n").map(l => JSON.parse(l));
|
|
101
|
-
const killEntry = entries.find((e: any) => e.action === "auto_kill_fast");
|
|
102
|
-
expect(killEntry).toBeDefined();
|
|
103
|
-
expect(killEntry.targets.some((t: any) => t.pid === childPid)).toBe(true);
|
|
104
|
-
});
|
|
105
|
-
|
|
106
|
-
it("does NOT kill when zone is GREEN even with growth evidence", async () => {
|
|
107
|
-
childProc = spawn("sleep", ["60"], { stdio: "ignore" });
|
|
108
|
-
const childPid = childProc.pid!;
|
|
109
|
-
|
|
110
|
-
const kills: any[] = [];
|
|
111
|
-
monitor.onAutoKill((decision) => {
|
|
112
|
-
kills.push(decision);
|
|
113
|
-
});
|
|
114
|
-
|
|
115
|
-
monitor.start();
|
|
116
|
-
await new Promise(r => setTimeout(r, 200));
|
|
117
|
-
|
|
118
|
-
seedGrowthHistory(childPid, "sleep");
|
|
119
|
-
|
|
120
|
-
// Zone stays GREEN
|
|
121
|
-
expect(monitor.currentZone).toBe(Zone.GREEN);
|
|
122
|
-
|
|
123
|
-
await new Promise(r => setTimeout(r, 3000));
|
|
124
|
-
|
|
125
|
-
let alive = true;
|
|
126
|
-
try {
|
|
127
|
-
process.kill(childPid, 0);
|
|
128
|
-
} catch {
|
|
129
|
-
alive = false;
|
|
130
|
-
}
|
|
131
|
-
expect(alive).toBe(true);
|
|
132
|
-
expect(kills).toHaveLength(0);
|
|
133
|
-
});
|
|
134
|
-
|
|
135
|
-
it("kills newest process in RED even without growth evidence", async () => {
|
|
136
|
-
// The "newest" filter catches processes that appeared after the last
|
|
137
|
-
// non-RED snapshot. Since the child spawned during warm-up (when system
|
|
138
|
-
// was GREEN), forcing RED should trigger "newest" detection.
|
|
139
|
-
// We seed a cached snapshot with non-zero footprint so the process
|
|
140
|
-
// passes the footprint_mb > 0 guard (0 MB processes are infrastructure noise).
|
|
141
|
-
childProc = spawn("sleep", ["60"], { stdio: "ignore" });
|
|
142
|
-
const childPid = childProc.pid!;
|
|
143
|
-
|
|
144
|
-
const kills: any[] = [];
|
|
145
|
-
monitor.onAutoKill((decision) => {
|
|
146
|
-
kills.push(decision);
|
|
147
|
-
});
|
|
148
|
-
|
|
149
|
-
monitor.start();
|
|
150
|
-
await new Promise(r => setTimeout(r, 1500));
|
|
151
|
-
|
|
152
|
-
// Seed a single snapshot with meaningful footprint (no growth pattern,
|
|
153
|
-
// just proves the process exists with real memory usage).
|
|
154
|
-
// Also set lastNonRedTimestamp explicitly — with hysteresis, the warm-up
|
|
155
|
-
// period (1.5s) isn't long enough for 3 consecutive non-RED polls.
|
|
156
|
-
const state = monitor.getWatchdogState();
|
|
157
|
-
state.snapshotHistory = [
|
|
158
|
-
[{ pid: childPid, name: "sleep", footprint_mb: 150, age_seconds: 3 }],
|
|
159
|
-
];
|
|
160
|
-
state.lastNonRedTimestamp = Date.now() / 1000 - 5; // system was healthy 5s ago
|
|
161
|
-
|
|
162
|
-
monitor._forceZone(Zone.RED, true);
|
|
163
|
-
|
|
164
|
-
// Wait for the fast watchdog to fire
|
|
165
|
-
const deadline = Date.now() + 6000;
|
|
166
|
-
while (Date.now() < deadline) {
|
|
167
|
-
await new Promise(r => setTimeout(r, 200));
|
|
168
|
-
try {
|
|
169
|
-
process.kill(childPid, 0);
|
|
170
|
-
} catch {
|
|
171
|
-
break;
|
|
172
|
-
}
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
let alive = true;
|
|
176
|
-
try {
|
|
177
|
-
process.kill(childPid, 0);
|
|
178
|
-
} catch {
|
|
179
|
-
alive = false;
|
|
180
|
-
}
|
|
181
|
-
expect(alive).toBe(false);
|
|
182
|
-
expect(kills.length).toBeGreaterThanOrEqual(1);
|
|
183
|
-
});
|
|
184
|
-
|
|
185
|
-
// Cooldown is tested at the unit level in watchdog.test.ts.
|
|
186
|
-
// Integration testing cooldown with real processes is timing-sensitive
|
|
187
|
-
// and flaky — the unit test covers the logic reliably.
|
|
188
|
-
});
|