residoo 0.4.14 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +43 -2
- package/package.json +1 -1
- package/src/cli.js +131 -0
- package/src/watch.js +640 -0
package/README.md
CHANGED
|
@@ -113,6 +113,9 @@ reproduction; everything needed to rerun it ships in this repo.
|
|
|
113
113
|
acknowledgement ledger. See [Rotation](#rotation-from-found-to-closed).
|
|
114
114
|
- `--project <dir>` scans a repository checkout instead of the machine, for
|
|
115
115
|
CI and pre-commit. See [CI and pre-commit](#ci-and-pre-commit).
|
|
116
|
+
- **`residoo watch`**: continuous scanning instead of one snapshot, alerting
|
|
117
|
+
the moment a new secret lands in a transcript. See
|
|
118
|
+
[Watch: continuous scanning](#watch-continuous-scanning) below.
|
|
116
119
|
|
|
117
120
|
## Beyond transcripts: configs and planted persistence
|
|
118
121
|
|
|
@@ -285,7 +288,7 @@ As a GitHub Action (this repo doubles as a composite action):
|
|
|
285
288
|
```yaml
|
|
286
289
|
steps:
|
|
287
290
|
- uses: actions/checkout@v4
|
|
288
|
-
- uses: dandovdub/residoo@v0.
|
|
291
|
+
- uses: dandovdub/residoo@v0.5.0
|
|
289
292
|
```
|
|
290
293
|
|
|
291
294
|
As a pre-commit hook:
|
|
@@ -293,7 +296,7 @@ As a pre-commit hook:
|
|
|
293
296
|
```yaml
|
|
294
297
|
repos:
|
|
295
298
|
- repo: https://github.com/dandovdub/residoo
|
|
296
|
-
rev: v0.
|
|
299
|
+
rev: v0.5.0
|
|
297
300
|
hooks:
|
|
298
301
|
- id: residoo
|
|
299
302
|
```
|
|
@@ -373,11 +376,49 @@ residoo ack <fingerprint> [--note <text>] mark one finding rotated
|
|
|
373
376
|
|
|
374
377
|
residoo unseal <vault-dir> list a vault's contents
|
|
375
378
|
residoo unseal <vault-dir> --restore <n> --out <p> restore one file, hash-verified
|
|
379
|
+
|
|
380
|
+
residoo watch [options]
|
|
381
|
+
|
|
382
|
+
--interval <seconds> how often to check for new content (default 5, minimum 1)
|
|
383
|
+
--json NDJSON events on stdout, one line per finding/re-exposure
|
|
384
|
+
--verify same opt-in vendor check as scan --verify, applied to
|
|
385
|
+
each newly found credential once
|
|
386
|
+
--include-noisy, --include-suppressed, --no-color same meaning as scan
|
|
376
387
|
```
|
|
377
388
|
|
|
378
389
|
The vault passphrase comes from `RESIDOO_PASSPHRASE` or a hidden interactive
|
|
379
390
|
prompt. There is no recovery if you lose it, so pick one you keep.
|
|
380
391
|
|
|
392
|
+
## Watch: continuous scanning
|
|
393
|
+
|
|
394
|
+
`residoo scan` is a snapshot. `residoo watch` is the same engine run
|
|
395
|
+
continuously: it polls every source `scan` already covers, and the moment a
|
|
396
|
+
new secret lands in a transcript, prints an alert with the redacted value,
|
|
397
|
+
the rule, the file, and the same rotation runbook a scan finding carries,
|
|
398
|
+
instead of waiting for you to remember to run `scan` again.
|
|
399
|
+
|
|
400
|
+
```
|
|
401
|
+
$ residoo watch
|
|
402
|
+
watching 43 sources, 118 files (61 tailed, 57 rescanned on change)
|
|
403
|
+
polling every 5s; fs.watch is not used, every alert comes from polling
|
|
404
|
+
|
|
405
|
+
2026-09-03 14:02:11 [high] AWS Access Key ID AKIA****ABCD
|
|
406
|
+
claude-code · session-9f2c.jsonl:214 · rf1-8a3e91 Rotate: https://.../access_keys
|
|
407
|
+
```
|
|
408
|
+
|
|
409
|
+
It is watch-from-**now**: run `residoo scan` first for anything already on
|
|
410
|
+
disk, since a fresh `residoo watch` baselines whatever it finds on its first
|
|
411
|
+
sweep silently and only alerts on content written after it starts. A finding
|
|
412
|
+
already acknowledged or dismissed (`residoo ack` / `residoo dismiss`) stays
|
|
413
|
+
suppressed, checked against the same `~/.residoo/rotations.json` ledger, and
|
|
414
|
+
a ledger change made mid-watch takes effect within one poll interval, no
|
|
415
|
+
restart needed. A findings-free sweep prints nothing at all, including to
|
|
416
|
+
its own watched Claude Code session, by design: no other tool in this
|
|
417
|
+
project's own benchmark ([`bench/`](bench/)) has a continuous mode at all,
|
|
418
|
+
verified directly against each one's own `--help` output rather than
|
|
419
|
+
assumed; see [docs/comparison.md](docs/comparison.md) for how the one
|
|
420
|
+
adjacent thing, GitGuardian's `ggshield` AI hook, works differently.
|
|
421
|
+
|
|
381
422
|
## Sources supported today
|
|
382
423
|
|
|
383
424
|
43 sources: 42 transcript stores plus the agent-config source above, in two
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "residoo",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.5.0",
|
|
4
4
|
"description": "Find secrets leaking through your AI coding agent's session history. Zero network calls in the scan path, zero dependencies.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "CloudRoam (https://cloudroam.io)",
|
package/src/cli.js
CHANGED
|
@@ -10,6 +10,7 @@ const { checkIntegrity } = require("./integrity");
|
|
|
10
10
|
const {
|
|
11
11
|
ROTATION_GUIDANCE, guidanceFor, loadAcks, loadDismissed, ackFinding, dismissFinding, renderRotation,
|
|
12
12
|
} = require("./rotation");
|
|
13
|
+
const { startWatch, isTailable } = require("./watch");
|
|
13
14
|
|
|
14
15
|
/**
|
|
15
16
|
* A source is unavailable for the ordinary reason (not installed — nothing
|
|
@@ -126,6 +127,26 @@ Scan options:
|
|
|
126
127
|
with no network call at all, on by default, not
|
|
127
128
|
part of --verify.
|
|
128
129
|
|
|
130
|
+
Watch:
|
|
131
|
+
residoo watch continuous scanning instead of one snapshot:
|
|
132
|
+
alerts the moment a new secret lands in a
|
|
133
|
+
transcript, instead of waiting for the next
|
|
134
|
+
"residoo scan". Covers the same sources as scan;
|
|
135
|
+
run scan first for anything already on disk,
|
|
136
|
+
since watch only ever looks at content written
|
|
137
|
+
AFTER it starts. No other tool in this project's
|
|
138
|
+
own benchmark (bench/) has anything like it.
|
|
139
|
+
--interval <seconds> how often to check for new content (default 5,
|
|
140
|
+
minimum 1)
|
|
141
|
+
--json NDJSON events on stdout instead of plain text,
|
|
142
|
+
one line per finding/re-exposure
|
|
143
|
+
--verify same opt-in vendor check as scan --verify,
|
|
144
|
+
applied to each newly found credential once,
|
|
145
|
+
never to one already seen
|
|
146
|
+
--include-noisy, --include-suppressed, --no-color same meaning as scan
|
|
147
|
+
Ctrl+C stops cleanly and prints a session summary (skipped with --json,
|
|
148
|
+
where the same information is one final NDJSON event).
|
|
149
|
+
|
|
129
150
|
Rotation:
|
|
130
151
|
residoo explain <rule-id> full rotation runbook for one detection rule
|
|
131
152
|
(where to revoke, steps, what revocation does)
|
|
@@ -441,6 +462,115 @@ function runDismiss(args) {
|
|
|
441
462
|
return 0;
|
|
442
463
|
}
|
|
443
464
|
|
|
465
|
+
/**
|
|
466
|
+
* One pass over each source's files() purely to describe what's about to
|
|
467
|
+
* be watched -- how many files will be tailed (byte-offset, only new
|
|
468
|
+
* content ever read) versus polled (whole-file rescan on any change). A
|
|
469
|
+
* separate, cheap enumeration from the watch itself; not wired into
|
|
470
|
+
* startWatch() so a banner-rendering concern never has to live inside the
|
|
471
|
+
* engine.
|
|
472
|
+
*/
|
|
473
|
+
function printWatchBanner(sources) {
|
|
474
|
+
process.stderr.write("residoo watch: establishing a baseline over each source (this covers NEW content only -- run `residoo scan` first for anything already on disk)...\n");
|
|
475
|
+
for (const source of sources) {
|
|
476
|
+
let tail = 0, rescan = 0;
|
|
477
|
+
try {
|
|
478
|
+
for (const entry of source.files()) {
|
|
479
|
+
if (entry.broken) continue;
|
|
480
|
+
if (isTailable(entry.file)) tail++; else rescan++;
|
|
481
|
+
}
|
|
482
|
+
} catch {
|
|
483
|
+
// A source erroring while just being counted for the banner is not
|
|
484
|
+
// fatal to the watch itself -- sweepOnce has its own try/catch
|
|
485
|
+
// around files() and reports it there instead.
|
|
486
|
+
}
|
|
487
|
+
const parts = [];
|
|
488
|
+
if (tail) parts.push(`${tail} tailed`);
|
|
489
|
+
if (rescan) parts.push(`${rescan} polled (rescanned on change)`);
|
|
490
|
+
process.stderr.write(` ${source.label()} (${source.id()}): ${parts.join(", ") || "no files yet"}\n`);
|
|
491
|
+
}
|
|
492
|
+
process.stderr.write(
|
|
493
|
+
"fs.watch is not used; every alert comes from polling, so an alert can lag\n" +
|
|
494
|
+
"up to one --interval behind the actual write. SQLite-backed sources are\n" +
|
|
495
|
+
"always polled in full (no incremental read exists for them). Ctrl+C to stop.\n\n"
|
|
496
|
+
);
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
function printWatchSummary(stats) {
|
|
500
|
+
process.stderr.write(
|
|
501
|
+
`\nresidoo watch: stopped. ${stats.sweeps} sweep${stats.sweeps === 1 ? "" : "s"}, ` +
|
|
502
|
+
`${stats.loud} new finding${stats.loud === 1 ? "" : "s"}, ${stats.quiet} re-exposure${stats.quiet === 1 ? "" : "s"}` +
|
|
503
|
+
(stats.suppressedByLedger ? `, ${stats.suppressedByLedger} already acked or dismissed` : "") +
|
|
504
|
+
(stats.errors ? `, ${stats.errors} sweep error${stats.errors === 1 ? "" : "s"}` : "") + ".\n"
|
|
505
|
+
);
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
/**
|
|
509
|
+
* `residoo watch`: continuous scanning instead of one snapshot. See
|
|
510
|
+
* src/watch.js for the engine; this function is only argument parsing,
|
|
511
|
+
* the startup/shutdown banners, and wiring SIGINT/SIGTERM to a clean stop.
|
|
512
|
+
*/
|
|
513
|
+
async function runWatch(args) {
|
|
514
|
+
const wantsJson = args.includes("--json");
|
|
515
|
+
const includeNoisy = args.includes("--include-noisy");
|
|
516
|
+
const includeSuppressed = args.includes("--include-suppressed");
|
|
517
|
+
const verify = args.includes("--verify");
|
|
518
|
+
const noColor = args.includes("--no-color");
|
|
519
|
+
|
|
520
|
+
let intervalSeconds = 5;
|
|
521
|
+
const intervalArg = argValue(args, "--interval");
|
|
522
|
+
if (intervalArg !== null) {
|
|
523
|
+
const n = Number(intervalArg);
|
|
524
|
+
if (!Number.isFinite(n) || n < 1) {
|
|
525
|
+
process.stderr.write(`--interval must be a number of seconds, at least 1; got "${intervalArg}".\n`);
|
|
526
|
+
return 2;
|
|
527
|
+
}
|
|
528
|
+
intervalSeconds = n;
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
const sources = availableSources();
|
|
532
|
+
if (sources.length === 0) {
|
|
533
|
+
process.stderr.write(
|
|
534
|
+
"No known transcript sources found on this machine; nothing to watch.\n" +
|
|
535
|
+
`Checked: ${sourceStatusList()}.\n`
|
|
536
|
+
);
|
|
537
|
+
return 0;
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
if (!wantsJson) printWatchBanner(sources);
|
|
541
|
+
|
|
542
|
+
const { promise, stop } = startWatch({
|
|
543
|
+
sources,
|
|
544
|
+
options: { includeNoisy, includeSuppressed, verify, noColor, json: wantsJson, pollMs: intervalSeconds * 1000 },
|
|
545
|
+
});
|
|
546
|
+
|
|
547
|
+
const printFinalSummary = (stats) => {
|
|
548
|
+
if (wantsJson) process.stdout.write(JSON.stringify({ type: "summary", at: new Date(), ...stats }) + "\n");
|
|
549
|
+
else printWatchSummary(stats);
|
|
550
|
+
};
|
|
551
|
+
|
|
552
|
+
// A second Ctrl+C must not hang waiting on a graceful stop that already
|
|
553
|
+
// started -- stop() itself is idempotent, but the listener only needs to
|
|
554
|
+
// act once.
|
|
555
|
+
let signalled = false;
|
|
556
|
+
const onSignal = () => {
|
|
557
|
+
if (signalled) return;
|
|
558
|
+
signalled = true;
|
|
559
|
+
printFinalSummary(stop());
|
|
560
|
+
};
|
|
561
|
+
process.once("SIGINT", onSignal);
|
|
562
|
+
process.once("SIGTERM", onSignal);
|
|
563
|
+
|
|
564
|
+
const stats = await promise;
|
|
565
|
+
process.removeListener("SIGINT", onSignal);
|
|
566
|
+
process.removeListener("SIGTERM", onSignal);
|
|
567
|
+
// promise can also resolve because something ELSE called stop() (not
|
|
568
|
+
// possible from outside this function today, but the contract allows
|
|
569
|
+
// it) -- print the summary exactly once regardless of which path got here.
|
|
570
|
+
if (!signalled) printFinalSummary(stats);
|
|
571
|
+
return 0;
|
|
572
|
+
}
|
|
573
|
+
|
|
444
574
|
async function main(argv) {
|
|
445
575
|
const args = argv.slice(2);
|
|
446
576
|
if (args.includes("-h") || args.includes("--help") || args.length === 0) {
|
|
@@ -453,6 +583,7 @@ async function main(argv) {
|
|
|
453
583
|
if (cmd === "explain") return runExplain(args);
|
|
454
584
|
if (cmd === "ack") return runAck(args);
|
|
455
585
|
if (cmd === "dismiss") return runDismiss(args);
|
|
586
|
+
if (cmd === "watch") return runWatch(args);
|
|
456
587
|
if (cmd !== "scan") {
|
|
457
588
|
process.stderr.write(`Unknown command "${cmd}". Try "residoo --help".\n`);
|
|
458
589
|
return 2;
|
package/src/watch.js
ADDED
|
@@ -0,0 +1,640 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
const fs = require("fs");
|
|
4
|
+
const path = require("path");
|
|
5
|
+
const crypto = require("crypto");
|
|
6
|
+
const { scan } = require("./scan");
|
|
7
|
+
const { guidanceFor, fingerprintFinding, loadAcks, loadDismissed, statePath } = require("./rotation");
|
|
8
|
+
const { c, makePaint } = require("./color");
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* `residoo watch`: continuous, near-real-time scanning of the same
|
|
12
|
+
* transcript stores `residoo scan` covers, instead of a single snapshot.
|
|
13
|
+
* No competitor in this project's own benchmark (bench/) has anything like
|
|
14
|
+
* it, verified directly against the installed tools' own --help output,
|
|
15
|
+
* not assumed. ggshield's AI hook is the one adjacent thing, and it works
|
|
16
|
+
* a different way entirely: a per-agent-tool hook that ships content to
|
|
17
|
+
* GitGuardian's server. This is one local process, zero network, covering
|
|
18
|
+
* every known agent source at once.
|
|
19
|
+
*
|
|
20
|
+
* Design in one sentence: a poll sweep re-runs each source's own files()
|
|
21
|
+
* generator (every adapter already knows its own topology) and diffs
|
|
22
|
+
* {sizeBytes, mtimeMs} against an in-memory offset table; new bytes on a
|
|
23
|
+
* `.jsonl` file are TAILED (only the new bytes are ever read); a change to
|
|
24
|
+
* any other file (a config rewritten in place, a SQLite-backed source's
|
|
25
|
+
* files, a legacy whole-file-per-turn format) triggers a full re-read via
|
|
26
|
+
* the real source's own readLines(), made idempotent by dedup. fs.watch is
|
|
27
|
+
* layered on top purely as a latency hint (an early trigger for the next
|
|
28
|
+
* sweep) and is never load-bearing -- it is inert on Linux for files
|
|
29
|
+
* written to subdirectories of a non-recursively-watched root, and
|
|
30
|
+
* unavailable on some filesystems entirely; every guarantee here comes
|
|
31
|
+
* from polling.
|
|
32
|
+
*
|
|
33
|
+
* scan() (see scan.js) never touches disk itself: it is driven entirely by
|
|
34
|
+
* the {id, files, readLines} source contract. A synthetic, in-memory
|
|
35
|
+
* source built from a batch of newly-read lines gets suppression, AWS/
|
|
36
|
+
* PlanetScale/MongoDB-Atlas pairing, base64 decoding, and split-line
|
|
37
|
+
* joining for free, identically to a real `residoo scan`.
|
|
38
|
+
*/
|
|
39
|
+
|
|
40
|
+
// ── small pure helpers, duplicated rather than imported ─────────────────────
|
|
41
|
+
// report.js and scan.js each keep their own copy of a couple of tiny,
|
|
42
|
+
// stable helpers rather than exporting them for one extra call site; this
|
|
43
|
+
// file follows the same precedent (see scan.js's own comment on this).
|
|
44
|
+
|
|
45
|
+
function localTimestamp(d) {
|
|
46
|
+
const p2 = (n) => String(n).padStart(2, "0");
|
|
47
|
+
return `${d.getFullYear()}-${p2(d.getMonth() + 1)}-${p2(d.getDate())} ${p2(d.getHours())}:${p2(d.getMinutes())}`;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
const INVISIBLES_RE = /[\u{e0000}-\u{e007f}]/gu;
|
|
51
|
+
function safeBasename(file) {
|
|
52
|
+
return path.basename(String(file))
|
|
53
|
+
.replace(/[\x00-\x1f\x7f]/g, "")
|
|
54
|
+
.replace(INVISIBLES_RE, (ch) => "\\u{" + ch.codePointAt(0).toString(16).toUpperCase() + "}");
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* `.jsonl` is append-only by construction across every source in
|
|
59
|
+
* src/sources/ (verified: each adapter that emits `.jsonl` either only
|
|
60
|
+
* ever appends, or -- openclaw session resets, qwen-code archiving --
|
|
61
|
+
* renames/moves the whole file rather than rewriting its content in
|
|
62
|
+
* place). Every other extension a source can yield (settings JSON,
|
|
63
|
+
* SQLite-backed session state, legacy whole-file-per-turn formats like
|
|
64
|
+
* gemini-cli's session-*.json) gets rewritten, not appended, so it is
|
|
65
|
+
* always fully re-read on change. This is a property of the FILE, not the
|
|
66
|
+
* source: gemini-cli alone yields both kinds from one adapter, which is
|
|
67
|
+
* exactly why the split is per-file, not per-source.
|
|
68
|
+
*/
|
|
69
|
+
function isTailable(file) {
|
|
70
|
+
return file.endsWith(".jsonl");
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* First 64 bytes, hashed, to catch "the file was truncated and rewritten
|
|
75
|
+
* to a size at or past the old offset between two sweeps" -- the one case
|
|
76
|
+
* a plain size/mtime comparison cannot distinguish from ordinary growth.
|
|
77
|
+
* Read only when a file looks like it grew; the common case (genuinely
|
|
78
|
+
* nothing changed, or genuinely just appended to) never pays this cost.
|
|
79
|
+
*/
|
|
80
|
+
function prefixHash(file) {
|
|
81
|
+
let fd;
|
|
82
|
+
try {
|
|
83
|
+
fd = fs.openSync(file, "r");
|
|
84
|
+
const buf = Buffer.alloc(64);
|
|
85
|
+
const n = fs.readSync(fd, buf, 0, 64, 0);
|
|
86
|
+
return crypto.createHash("sha256").update(buf.subarray(0, n)).digest("hex");
|
|
87
|
+
} catch {
|
|
88
|
+
return null;
|
|
89
|
+
} finally {
|
|
90
|
+
if (fd !== undefined) { try { fs.closeSync(fd); } catch { /* already gone */ } }
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Whole-file SHA-256, used only for rescan-class (non-tailable) files to
|
|
96
|
+
* tell "the file was touched but the bytes are identical" apart from "the
|
|
97
|
+
* bytes actually changed." Many CLIs rewrite their entire settings/config
|
|
98
|
+
* file on every run even when there is nothing new to say, which would
|
|
99
|
+
* otherwise force a real rescan (and, worse, risk re-surfacing a secret
|
|
100
|
+
* that was already there at baseline) on pure churn. A hash can only gate
|
|
101
|
+
* whether to rescan; unlike the scan itself it never sees redacted output
|
|
102
|
+
* or alerts, so computing it unconditionally on a same-size touch is safe.
|
|
103
|
+
*/
|
|
104
|
+
function wholeFileHash(file) {
|
|
105
|
+
try {
|
|
106
|
+
return crypto.createHash("sha256").update(fs.readFileSync(file)).digest("hex");
|
|
107
|
+
} catch {
|
|
108
|
+
return null;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Split a just-read Buffer on raw newline bytes (0x0A) BEFORE decoding any
|
|
114
|
+
* text, so a multi-byte UTF-8 sequence straddling the read boundary is
|
|
115
|
+
* never partially decoded (0x0A cannot appear as a continuation byte of a
|
|
116
|
+
* multi-byte sequence, so byte-level splitting is always safe). Returns
|
|
117
|
+
* only COMPLETE lines and how many bytes they consumed; an incomplete
|
|
118
|
+
* trailing fragment (no `\n` yet) is deliberately left unread and
|
|
119
|
+
* unreturned -- the caller must NOT advance its offset past
|
|
120
|
+
* `consumedBytes`, so the next sweep re-reads the still-incomplete tail
|
|
121
|
+
* from scratch rather than holding fragile partial-line state across
|
|
122
|
+
* sweeps.
|
|
123
|
+
*/
|
|
124
|
+
function splitCompleteLines(buf) {
|
|
125
|
+
const lines = [];
|
|
126
|
+
let start = 0;
|
|
127
|
+
for (let i = 0; i < buf.length; i++) {
|
|
128
|
+
if (buf[i] === 0x0a) {
|
|
129
|
+
let end = i;
|
|
130
|
+
if (end > start && buf[end - 1] === 0x0d) end--; // trailing \r, matches readLines' crlfDelay handling
|
|
131
|
+
lines.push(buf.toString("utf-8", start, end));
|
|
132
|
+
start = i + 1;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
return { lines, consumedBytes: start };
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* Read new bytes from `fromOffset` to EOF (capped per sweep, leftover
|
|
140
|
+
* carries to the next one) via a plain fd, not a stream: a watcher issues
|
|
141
|
+
* this far more often, on far smaller reads, than scan.js's one-shot
|
|
142
|
+
* whole-file readLines(), so the lighter-weight sync primitive fits
|
|
143
|
+
* better here. Returns null if the file vanished or became unreadable
|
|
144
|
+
* between the caller's stat and this open.
|
|
145
|
+
*/
|
|
146
|
+
const MAX_TAIL_BYTES_PER_SWEEP = 64 * 1024 * 1024;
|
|
147
|
+
function tailRead(file, fromOffset) {
|
|
148
|
+
let fd;
|
|
149
|
+
try {
|
|
150
|
+
fd = fs.openSync(file, "r");
|
|
151
|
+
const stat = fs.fstatSync(fd);
|
|
152
|
+
const available = stat.size - fromOffset;
|
|
153
|
+
if (available <= 0) return { buffer: Buffer.alloc(0), size: stat.size };
|
|
154
|
+
const toRead = Math.min(available, MAX_TAIL_BYTES_PER_SWEEP);
|
|
155
|
+
const buf = Buffer.alloc(toRead);
|
|
156
|
+
const n = fs.readSync(fd, buf, 0, toRead, fromOffset);
|
|
157
|
+
return { buffer: buf.subarray(0, n), size: stat.size, mtimeMs: stat.mtimeMs };
|
|
158
|
+
} catch {
|
|
159
|
+
return null;
|
|
160
|
+
} finally {
|
|
161
|
+
if (fd !== undefined) { try { fs.closeSync(fd); } catch { /* already gone */ } }
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* Advance one TAILABLE file's tracked entry by whatever new complete lines
|
|
167
|
+
* are available, mutating `entry` in place. Returns a batch object for
|
|
168
|
+
* scan(), or null when there was nothing new to read yet (grew by less
|
|
169
|
+
* than one full line -- normal for a session mid-write).
|
|
170
|
+
*
|
|
171
|
+
* The overlap: when this file already produced a batch on some earlier
|
|
172
|
+
* sweep, `entry.lastLine` holds that batch's final line, prepended here so
|
|
173
|
+
* scan()'s own split-across-lines boundary pass (see decode.js) can join a
|
|
174
|
+
* secret that straddles the seam between sweeps. That line inevitably gets
|
|
175
|
+
* independently re-matched too (scan() matches every line in a batch, not
|
|
176
|
+
* just the boundary pass), which is exactly why `lineNumberFor` returns
|
|
177
|
+
* null for it below: the caller drops any finding that resolves to null,
|
|
178
|
+
* since it was already alerted on last sweep.
|
|
179
|
+
*/
|
|
180
|
+
function advanceTail(entry, file, sizeBytes) {
|
|
181
|
+
const tail = tailRead(file, entry.offset);
|
|
182
|
+
if (tail === null) { entry.vanished = true; return null; }
|
|
183
|
+
const { lines: newLines, consumedBytes } = splitCompleteLines(tail.buffer);
|
|
184
|
+
if (entry.prefixHash === null) entry.prefixHash = prefixHash(file);
|
|
185
|
+
if (newLines.length === 0) return null;
|
|
186
|
+
|
|
187
|
+
const hasOverlap = entry.lastLine !== null;
|
|
188
|
+
const lines = hasOverlap ? [entry.lastLine, ...newLines] : newLines;
|
|
189
|
+
const lineCountBefore = entry.lineCount;
|
|
190
|
+
|
|
191
|
+
entry.offset += consumedBytes;
|
|
192
|
+
entry.sizeBytes = sizeBytes;
|
|
193
|
+
entry.lineCount += newLines.length;
|
|
194
|
+
entry.lastLine = newLines[newLines.length - 1];
|
|
195
|
+
|
|
196
|
+
return {
|
|
197
|
+
lines, bytesRead: consumedBytes, sizeBytes, mtimeMs: tail.mtimeMs, absolute: false,
|
|
198
|
+
// batch-relative line k (1-based) -> "line since watch started for
|
|
199
|
+
// this file"; null on the overlap line (batch line 1, when present)
|
|
200
|
+
// since that line was already reported last sweep. A genuine boundary
|
|
201
|
+
// match spanning the seam still comes through on its SECOND line.
|
|
202
|
+
lineNumberFor(batchLine) {
|
|
203
|
+
const k = batchLine - 1;
|
|
204
|
+
if (hasOverlap && k === 0) return null;
|
|
205
|
+
return lineCountBefore + (hasOverlap ? k : k + 1);
|
|
206
|
+
},
|
|
207
|
+
};
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/**
|
|
211
|
+
* Full re-read of a non-tailable (rewritten-in-place) file, through the
|
|
212
|
+
* REAL source's own readLines() -- its size cap, read timeout, and
|
|
213
|
+
* partial/too-large/failed handling all apply exactly as they would in a
|
|
214
|
+
* real `residoo scan`. The whole file is fresh content every time, so
|
|
215
|
+
* scan()'s own line numbers are already true absolute positions; no
|
|
216
|
+
* remapping needed.
|
|
217
|
+
*/
|
|
218
|
+
async function readWholeFile(source, file, sizeBytes, mtimeMs) {
|
|
219
|
+
let result;
|
|
220
|
+
try {
|
|
221
|
+
result = await source.readLines(file);
|
|
222
|
+
} catch {
|
|
223
|
+
return null;
|
|
224
|
+
}
|
|
225
|
+
if (result.status === "failed" || result.status === "too-large") return null;
|
|
226
|
+
if (result.lines.length === 0) return null;
|
|
227
|
+
return {
|
|
228
|
+
lines: result.lines, bytesRead: result.bytesRead, sizeBytes, mtimeMs, absolute: true,
|
|
229
|
+
lineNumberFor(batchLine) { return batchLine; },
|
|
230
|
+
};
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/**
|
|
234
|
+
* One synthetic source per REAL source per sweep, wrapping whatever new
|
|
235
|
+
* line batches were collected for it. `id()` delegates to the real
|
|
236
|
+
* source's id so finding.source, rotation guidance, and fingerprints all
|
|
237
|
+
* come out identical to a real `residoo scan` of the same content.
|
|
238
|
+
*/
|
|
239
|
+
function makeSyntheticSource(realId, batchesByFile) {
|
|
240
|
+
return {
|
|
241
|
+
id: () => realId,
|
|
242
|
+
label: () => realId,
|
|
243
|
+
available: () => true,
|
|
244
|
+
*files() {
|
|
245
|
+
for (const [file, b] of batchesByFile) {
|
|
246
|
+
yield { file, mtimeMs: b.mtimeMs, sizeBytes: b.sizeBytes, broken: false };
|
|
247
|
+
}
|
|
248
|
+
},
|
|
249
|
+
async readLines(file) {
|
|
250
|
+
const b = batchesByFile.get(file);
|
|
251
|
+
return { lines: b.lines, status: "complete", bytesRead: b.bytesRead };
|
|
252
|
+
},
|
|
253
|
+
};
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
/**
|
|
257
|
+
* Silently seed the dedup ledger from a rescan-class file's CURRENT
|
|
258
|
+
* content the first time watch ever sees it, without alerting on any of
|
|
259
|
+
* it. Baseline never alerts (that promise is unchanged), but for a
|
|
260
|
+
* whole-file source specifically, a later edit ANYWHERE in the file
|
|
261
|
+
* forces a full rescan of everything in it, including a secret that was
|
|
262
|
+
* already sitting there at baseline -- without this seeding step, that
|
|
263
|
+
* unrelated edit would make an old, already-known secret look brand new.
|
|
264
|
+
* (Tailable files don't need this: their baseline skips straight to EOF,
|
|
265
|
+
* so past content is never re-read at all, let alone re-surfaced.)
|
|
266
|
+
* `verify` is always forced off here: seeding a dedup cache must never be
|
|
267
|
+
* the reason a live vendor API gets hit.
|
|
268
|
+
*/
|
|
269
|
+
async function baselineSeed(source, sourceId, file, sizeBytes, mtimeMs, seen, includeNoisy, includeSuppressed, noColor) {
|
|
270
|
+
const batch = await readWholeFile(source, file, sizeBytes, mtimeMs);
|
|
271
|
+
if (!batch) return;
|
|
272
|
+
let result;
|
|
273
|
+
try {
|
|
274
|
+
result = await scan({
|
|
275
|
+
sources: [makeSyntheticSource(sourceId, new Map([[file, batch]]))],
|
|
276
|
+
includeNoisy, includeSuppressed, verify: false, noColor,
|
|
277
|
+
});
|
|
278
|
+
} catch {
|
|
279
|
+
return; // best-effort: a failure here just leaves this file's dedup
|
|
280
|
+
// slate empty, no worse off than before baseline seeding existed
|
|
281
|
+
}
|
|
282
|
+
for (const finding of result.findings) {
|
|
283
|
+
const dedupKey = finding.ruleId + "\0" + finding.preview;
|
|
284
|
+
if (!seen.has(dedupKey)) seen.set(dedupKey, { count: 1, fingerprint: fingerprintFinding(finding) });
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
/**
|
|
289
|
+
* One sweep: re-enumerate every source's files(), tail/rescan whatever
|
|
290
|
+
* changed, feed the results through the real scan() engine, and emit
|
|
291
|
+
* events. Pure enough to unit test directly with no timers: call it
|
|
292
|
+
* yourself in a loop, or let startWatch() below drive it.
|
|
293
|
+
*
|
|
294
|
+
* `tracked` (path -> per-file state) and `seen` (dedup key -> alert count)
|
|
295
|
+
* are the state that persists ACROSS calls, both plain Maps owned by the
|
|
296
|
+
* caller so tests can inspect them directly. `ledger` is
|
|
297
|
+
* `{ acks, dismissed }` (fingerprint -> entry), reloaded by the caller on
|
|
298
|
+
* the rotations.json mtime changing, so a mid-watch `residoo ack`/
|
|
299
|
+
* `dismiss` takes effect without a restart.
|
|
300
|
+
*/
|
|
301
|
+
async function sweepOnce({ sources, tracked, seen, ledger, options, emit }) {
|
|
302
|
+
const { includeNoisy, includeSuppressed, verify, noColor } = options || {};
|
|
303
|
+
let loud = 0;
|
|
304
|
+
let quiet = 0;
|
|
305
|
+
let suppressedByLedger = 0;
|
|
306
|
+
|
|
307
|
+
for (const source of sources) {
|
|
308
|
+
const sourceId = source.id();
|
|
309
|
+
const batchesByFile = new Map();
|
|
310
|
+
const stillPresent = new Set();
|
|
311
|
+
|
|
312
|
+
let entries;
|
|
313
|
+
try {
|
|
314
|
+
entries = [...source.files()];
|
|
315
|
+
} catch (err) {
|
|
316
|
+
emit({ type: "watch-error", at: new Date(), source: sourceId, detail: "files() failed: " + (err && err.message) });
|
|
317
|
+
continue;
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
for (const entry of entries) {
|
|
321
|
+
if (entry.broken) continue; // same as scan.js: a dangling symlink, nothing to tail
|
|
322
|
+
const { file, mtimeMs, sizeBytes } = entry;
|
|
323
|
+
stillPresent.add(file);
|
|
324
|
+
const prev = tracked.get(file);
|
|
325
|
+
const tailable = isTailable(file);
|
|
326
|
+
|
|
327
|
+
if (!prev) {
|
|
328
|
+
const migrated = migrateVanishedEntry(tracked, sourceId, file, sizeBytes);
|
|
329
|
+
if (migrated) {
|
|
330
|
+
tracked.set(file, migrated);
|
|
331
|
+
migrated.mtimeMs = mtimeMs;
|
|
332
|
+
migrated.sizeBytes = sizeBytes;
|
|
333
|
+
// A rename alone is not new content -- if the file ALSO grew or
|
|
334
|
+
// (for a tailable file) changed since the old name's last known
|
|
335
|
+
// state, that's handled the normal way on this sweep's tail/
|
|
336
|
+
// rescan branches below via `prev`/`tracked`, not here.
|
|
337
|
+
if (tailable && sizeBytes > migrated.offset) {
|
|
338
|
+
const batch = advanceTail(migrated, file, sizeBytes);
|
|
339
|
+
if (batch) batchesByFile.set(file, batch);
|
|
340
|
+
}
|
|
341
|
+
continue;
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
// Genuinely new file, of EITHER class: baseline WITHOUT alerting.
|
|
345
|
+
// Watch-from-now is a promise about every source uniformly --
|
|
346
|
+
// `residoo scan` is what covers whatever is already on disk,
|
|
347
|
+
// including in a settings file this is the first time seeing.
|
|
348
|
+
// Emitting an alert for a rescan-class file's CURRENT content
|
|
349
|
+
// immediately would quietly break that promise for exactly one
|
|
350
|
+
// class of source and duplicate what scan already does, the first
|
|
351
|
+
// time `residoo watch` ever runs on a machine with existing
|
|
352
|
+
// config-file findings -- caught live, not merely reasoned about,
|
|
353
|
+
// running this for real. For that same class, content IS read once
|
|
354
|
+
// here (never for tailable files, whose baseline skips straight to
|
|
355
|
+
// EOF): not to alert, only to seed the dedup ledger, so a later
|
|
356
|
+
// edit ELSEWHERE in the file doesn't make an already-known secret
|
|
357
|
+
// look newly discovered (see baselineSeed's own doc comment).
|
|
358
|
+
let ino = null, dev = null;
|
|
359
|
+
try { const st = fs.statSync(file); ino = st.ino; dev = st.dev; } catch { /* raced with deletion */ }
|
|
360
|
+
tracked.set(file, {
|
|
361
|
+
sourceId, tailable, offset: sizeBytes, sizeBytes, mtimeMs,
|
|
362
|
+
ino, dev, lineCount: 0, prefixHash: null, lastLine: null, vanished: false,
|
|
363
|
+
contentHash: tailable ? null : wholeFileHash(file),
|
|
364
|
+
});
|
|
365
|
+
if (!tailable) {
|
|
366
|
+
await baselineSeed(source, sourceId, file, sizeBytes, mtimeMs, seen, includeNoisy, includeSuppressed, noColor);
|
|
367
|
+
}
|
|
368
|
+
continue;
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
prev.vanished = false;
|
|
372
|
+
|
|
373
|
+
if (!prev.tailable) {
|
|
374
|
+
if (sizeBytes !== prev.sizeBytes) {
|
|
375
|
+
// The byte count itself changed: unambiguous, always a real
|
|
376
|
+
// rescan (no need to hash-gate first; something is different by
|
|
377
|
+
// definition).
|
|
378
|
+
const batch = await readWholeFile(source, file, sizeBytes, mtimeMs);
|
|
379
|
+
if (batch) batchesByFile.set(file, batch);
|
|
380
|
+
prev.mtimeMs = mtimeMs;
|
|
381
|
+
prev.sizeBytes = sizeBytes;
|
|
382
|
+
prev.contentHash = wholeFileHash(file);
|
|
383
|
+
} else if (mtimeMs !== prev.mtimeMs) {
|
|
384
|
+
// Same size, but touched: could be a same-length content swap (a
|
|
385
|
+
// rotated secret of equal length) or pure mtime churn. The whole-
|
|
386
|
+
// file hash tells the two apart without ever running the
|
|
387
|
+
// detection engine on bytes that did not change.
|
|
388
|
+
const h = wholeFileHash(file);
|
|
389
|
+
if (h !== null && prev.contentHash !== null && h === prev.contentHash) {
|
|
390
|
+
prev.mtimeMs = mtimeMs;
|
|
391
|
+
continue;
|
|
392
|
+
}
|
|
393
|
+
const batch = await readWholeFile(source, file, sizeBytes, mtimeMs);
|
|
394
|
+
if (batch) batchesByFile.set(file, batch);
|
|
395
|
+
prev.mtimeMs = mtimeMs;
|
|
396
|
+
prev.contentHash = h;
|
|
397
|
+
}
|
|
398
|
+
continue;
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
// Tailable file, already tracked.
|
|
402
|
+
if (sizeBytes < prev.offset) {
|
|
403
|
+
rebaseline(prev, sizeBytes, mtimeMs); // shrank: truncated/reset, not new content
|
|
404
|
+
continue;
|
|
405
|
+
}
|
|
406
|
+
if (sizeBytes > prev.offset) {
|
|
407
|
+
const h = prefixHash(file);
|
|
408
|
+
if (prev.prefixHash !== null && h !== null && h !== prev.prefixHash) {
|
|
409
|
+
rebaseline(prev, sizeBytes, mtimeMs); // grew, but the START of the file changed too: a rewrite, not a real append
|
|
410
|
+
continue;
|
|
411
|
+
}
|
|
412
|
+
const batch = advanceTail(prev, file, sizeBytes);
|
|
413
|
+
prev.mtimeMs = mtimeMs;
|
|
414
|
+
if (batch) batchesByFile.set(file, batch);
|
|
415
|
+
}
|
|
416
|
+
// sizeBytes === prev.offset: no new bytes; mtime-only churn ignored.
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
// Anything tracked for this source that files() did not yield this
|
|
420
|
+
// sweep either vanished or was renamed/moved. Keep its offset one more
|
|
421
|
+
// sweep so a rename showing up as a "new" path above can migrate it,
|
|
422
|
+
// then let it drop out of `tracked` for good.
|
|
423
|
+
for (const [file, trackedEntry] of tracked) {
|
|
424
|
+
if (trackedEntry.sourceId !== sourceId || stillPresent.has(file)) continue;
|
|
425
|
+
if (trackedEntry.vanished) tracked.delete(file);
|
|
426
|
+
else trackedEntry.vanished = true;
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
if (batchesByFile.size === 0) continue;
|
|
430
|
+
|
|
431
|
+
let result;
|
|
432
|
+
try {
|
|
433
|
+
result = await scan({
|
|
434
|
+
sources: [makeSyntheticSource(sourceId, batchesByFile)],
|
|
435
|
+
includeNoisy, includeSuppressed, verify, noColor,
|
|
436
|
+
});
|
|
437
|
+
} catch (err) {
|
|
438
|
+
emit({ type: "watch-error", at: new Date(), source: sourceId, detail: "scan failed: " + (err && err.message) });
|
|
439
|
+
continue;
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
for (const finding of result.findings) {
|
|
443
|
+
const b = batchesByFile.get(finding.file);
|
|
444
|
+
const line = b.lineNumberFor(finding.line);
|
|
445
|
+
if (line === null) continue; // seam re-report: lies entirely on the overlap line, already alerted last sweep
|
|
446
|
+
|
|
447
|
+
const fp = fingerprintFinding(finding);
|
|
448
|
+
if ((ledger.acks && ledger.acks[fp]) || (ledger.dismissed && ledger.dismissed[fp])) {
|
|
449
|
+
suppressedByLedger++;
|
|
450
|
+
continue;
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
const dedupKey = finding.ruleId + "\0" + finding.preview;
|
|
454
|
+
const already = seen.get(dedupKey);
|
|
455
|
+
if (already) {
|
|
456
|
+
already.count++;
|
|
457
|
+
quiet++;
|
|
458
|
+
emit({ type: "reexposure", at: new Date(), ruleId: finding.ruleId, preview: finding.preview, count: already.count });
|
|
459
|
+
continue;
|
|
460
|
+
}
|
|
461
|
+
seen.set(dedupKey, { count: 1, fingerprint: fp });
|
|
462
|
+
loud++;
|
|
463
|
+
emit({
|
|
464
|
+
type: "finding", at: new Date(), ruleId: finding.ruleId, label: finding.label,
|
|
465
|
+
confidence: finding.confidence, source: finding.source, relFile: safeBasename(finding.relFile),
|
|
466
|
+
line, lineIsAbsolute: b.absolute, preview: finding.preview,
|
|
467
|
+
fingerprint: fp, guidance: guidanceFor(finding.ruleId),
|
|
468
|
+
});
|
|
469
|
+
}
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
return { loud, quiet, suppressedByLedger };
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
function rebaseline(entry, sizeBytes, mtimeMs) {
|
|
476
|
+
entry.offset = sizeBytes;
|
|
477
|
+
entry.sizeBytes = sizeBytes;
|
|
478
|
+
entry.mtimeMs = mtimeMs;
|
|
479
|
+
entry.lineCount = 0;
|
|
480
|
+
entry.lastLine = null;
|
|
481
|
+
entry.prefixHash = null; // re-hashed lazily next time the file looks like it grew
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
/**
|
|
485
|
+
* A path files() just yielded that was never tracked before might be a
|
|
486
|
+
* rename/move of a path that WAS tracked and then vanished (openclaw
|
|
487
|
+
* session resets rename to `*.jsonl.reset.<ts>`; qwen-code moves inactive
|
|
488
|
+
* sessions into chats/archive/). Recognized by inode identity, not name,
|
|
489
|
+
* and only when the new size is at least the old offset -- a smaller
|
|
490
|
+
* "match" is coincidence (inode reuse after deletion), not a real rename.
|
|
491
|
+
* Migrating carries the offset forward so the content already alerted on
|
|
492
|
+
* under the old name is never re-scanned under the new one.
|
|
493
|
+
*/
|
|
494
|
+
function migrateVanishedEntry(tracked, sourceId, file, sizeBytes) {
|
|
495
|
+
let st;
|
|
496
|
+
try { st = fs.statSync(file); } catch { return null; }
|
|
497
|
+
for (const [oldPath, entry] of tracked) {
|
|
498
|
+
if (!entry.vanished || entry.sourceId !== sourceId) continue;
|
|
499
|
+
if (entry.ino === st.ino && entry.dev === st.dev && sizeBytes >= entry.offset) {
|
|
500
|
+
tracked.delete(oldPath);
|
|
501
|
+
entry.vanished = false;
|
|
502
|
+
return entry;
|
|
503
|
+
}
|
|
504
|
+
}
|
|
505
|
+
return null;
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
// ── rendering ────────────────────────────────────────────────────────────
|
|
509
|
+
|
|
510
|
+
/** Human-readable line for one event, or null for an event that prints
|
|
511
|
+
* nothing (there is no such event today, but keeping the contract explicit
|
|
512
|
+
* matches the "nothing on a findings-free sweep" rule this file is built
|
|
513
|
+
* around: silence is the default, every printed line is deliberate). */
|
|
514
|
+
function renderHumanLine(event, paint) {
|
|
515
|
+
const ts = localTimestamp(event.at);
|
|
516
|
+
if (event.type === "finding") {
|
|
517
|
+
const g = event.guidance || {};
|
|
518
|
+
const rotate = g.rotateUrl || g.consolePath || "no rotation guidance shipped for this rule yet";
|
|
519
|
+
const where = event.lineIsAbsolute ? `line ${event.line}` : `line ${event.line} since watch started`;
|
|
520
|
+
return paint(c.red + c.bold, `⚠ ${ts} ${event.label}`) +
|
|
521
|
+
` ${event.preview} in ${event.relFile} (${event.source}), ${where}\n` +
|
|
522
|
+
paint(c.dim, ` ${event.fingerprint} rotate: ${rotate}`);
|
|
523
|
+
}
|
|
524
|
+
if (event.type === "reexposure") {
|
|
525
|
+
return paint(c.dim, ` ${ts} ${event.ruleId} ${event.preview} re-exposed (seen ${event.count}x so far)`);
|
|
526
|
+
}
|
|
527
|
+
if (event.type === "watch-error") {
|
|
528
|
+
return paint(c.yellow, `residoo watch: ${event.source ? event.source + ": " : ""}${event.detail}`);
|
|
529
|
+
}
|
|
530
|
+
return null;
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
/** `{ acks, dismissed }`, reloaded only when rotations.json's own mtime
|
|
534
|
+
* changed since the last check -- cheap (one stat per tick) and means a
|
|
535
|
+
* mid-watch `residoo ack`/`dismiss` takes effect within one poll interval,
|
|
536
|
+
* not only on restart. Missing ledger file is not an error: it just means
|
|
537
|
+
* nothing has been triaged yet, same as loadAcks()/loadDismissed() already
|
|
538
|
+
* treat it.
|
|
539
|
+
*/
|
|
540
|
+
function reloadLedgerIfChanged(prev) {
|
|
541
|
+
let mtimeMs = null;
|
|
542
|
+
try { mtimeMs = fs.statSync(statePath()).mtimeMs; } catch { /* no ledger yet */ }
|
|
543
|
+
if (prev && prev.mtimeMs === mtimeMs) return prev;
|
|
544
|
+
return { acks: loadAcks(), dismissed: loadDismissed(), mtimeMs };
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
/**
|
|
548
|
+
* Start watching. Returns `{ promise, stop, stats }`: `promise` resolves
|
|
549
|
+
* with the final `stats` when `stop()` is called (by a caller, or by the
|
|
550
|
+
* SIGINT/SIGTERM handlers cli.js wires up); `stats` is also readable live.
|
|
551
|
+
* All I/O is injectable (`sources`, `out`, `errOut`) so tests drive this
|
|
552
|
+
* in-process with a tiny `options.pollMs` and real temp
|
|
553
|
+
* directories, never a spawned child process.
|
|
554
|
+
*
|
|
555
|
+
* fs.watch is deliberately NOT used in v1: every source's public contract
|
|
556
|
+
* ({id, label, available, files, readLines}) has no way to ask "what root
|
|
557
|
+
* directory do you watch", so there is no clean hook to attach an
|
|
558
|
+
* fs.watch() to without reaching into adapter internals. Polling alone is
|
|
559
|
+
* not a degraded fallback here, it is the correctness backstop this whole
|
|
560
|
+
* design already leans on (fs.watch is well known to be inert on Linux for
|
|
561
|
+
* writes to subdirectories of a non-recursively-watched root, and
|
|
562
|
+
* unavailable on some filesystems entirely) -- v1 ships with exactly that
|
|
563
|
+
* backstop and nothing riding on top of it. `options.fsWatch` is accepted
|
|
564
|
+
* and ignored, reserved for when a future version adds real hints.
|
|
565
|
+
*/
|
|
566
|
+
function startWatch({ sources, options = {}, out = process.stdout, errOut = process.stderr } = {}) {
|
|
567
|
+
const paint = makePaint(options.noColor, out);
|
|
568
|
+
const tracked = new Map();
|
|
569
|
+
const seen = new Map();
|
|
570
|
+
let ledger = reloadLedgerIfChanged(null);
|
|
571
|
+
let sweeping = false;
|
|
572
|
+
let pendingTick = false;
|
|
573
|
+
let stopped = false;
|
|
574
|
+
let timer = null;
|
|
575
|
+
const stats = { loud: 0, quiet: 0, suppressedByLedger: 0, sweeps: 0, errors: 0 };
|
|
576
|
+
|
|
577
|
+
function emit(event) {
|
|
578
|
+
if (event.type === "watch-error") stats.errors++;
|
|
579
|
+
if (options.json) {
|
|
580
|
+
out.write(JSON.stringify(event) + "\n");
|
|
581
|
+
return;
|
|
582
|
+
}
|
|
583
|
+
const line = renderHumanLine(event, paint);
|
|
584
|
+
if (line !== null) out.write(line + "\n");
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
async function tick() {
|
|
588
|
+
if (sweeping) { pendingTick = true; return; }
|
|
589
|
+
sweeping = true;
|
|
590
|
+
try {
|
|
591
|
+
ledger = reloadLedgerIfChanged(ledger);
|
|
592
|
+
const result = await sweepOnce({ sources, tracked, seen, ledger, options, emit });
|
|
593
|
+
stats.loud += result.loud;
|
|
594
|
+
stats.quiet += result.quiet;
|
|
595
|
+
stats.suppressedByLedger += result.suppressedByLedger;
|
|
596
|
+
stats.sweeps++;
|
|
597
|
+
} catch (err) {
|
|
598
|
+
// An uncaught throw here would be an unhandled promise rejection
|
|
599
|
+
// (process crash on modern Node) if it ever escaped the timer
|
|
600
|
+
// callback that drives this in the real CLI. One stderr note, the
|
|
601
|
+
// loop keeps running -- one bad sweep must not end the whole watch.
|
|
602
|
+
stats.errors++;
|
|
603
|
+
errOut.write(`residoo watch: sweep failed unexpectedly: ${err && err.message}\n`);
|
|
604
|
+
} finally {
|
|
605
|
+
sweeping = false;
|
|
606
|
+
if (pendingTick && !stopped) { pendingTick = false; tick(); }
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
|
|
610
|
+
// No floor enforced here: that belongs to CLI argument validation
|
|
611
|
+
// (runWatch in cli.js, --interval's stated 1-second floor), not this
|
|
612
|
+
// engine. Tests need real sub-second polling to stay fast; `pollMs` is
|
|
613
|
+
// the raw, unclamped knob for that.
|
|
614
|
+
const intervalMs = options.pollMs || 5000;
|
|
615
|
+
let resolvePromise;
|
|
616
|
+
const promise = new Promise((resolve) => { resolvePromise = resolve; });
|
|
617
|
+
|
|
618
|
+
function scheduleNext() {
|
|
619
|
+
if (stopped) return;
|
|
620
|
+
timer = setTimeout(() => { tick().finally(scheduleNext); }, intervalMs);
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
function stop() {
|
|
624
|
+
if (stopped) return stats;
|
|
625
|
+
stopped = true;
|
|
626
|
+
if (timer) clearTimeout(timer);
|
|
627
|
+
resolvePromise(stats);
|
|
628
|
+
return stats;
|
|
629
|
+
}
|
|
630
|
+
|
|
631
|
+
// First sweep runs immediately (this is the baseline sweep -- see
|
|
632
|
+
// sweepOnce's own doc comment), not after waiting a full interval.
|
|
633
|
+
tick().finally(scheduleNext);
|
|
634
|
+
|
|
635
|
+
return { promise, stop, stats };
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
module.exports = {
|
|
639
|
+
sweepOnce, startWatch, isTailable, splitCompleteLines, localTimestamp, safeBasename,
|
|
640
|
+
};
|