residoo 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -3
- package/package.json +1 -1
- package/src/cli.js +67 -18
- package/src/guard.js +102 -13
- package/src/ocr.js +172 -0
- package/src/report.js +8 -2
- package/src/scan.js +47 -4
package/README.md
CHANGED
|
@@ -51,8 +51,10 @@ into the conversation at all — which also means a long session compacting
|
|
|
51
51
|
away the exact value you pasted days ago can't force you to paste it
|
|
52
52
|
again, since there's nothing to lose. `residoo guard` blocks an obviously
|
|
53
53
|
sensitive file read before it happens (100% recall, 0% false positives on
|
|
54
|
-
its own [scored 81-case corpus](bench/guard/RESULTS.md))
|
|
55
|
-
|
|
54
|
+
its own [scored 81-case corpus](bench/guard/RESULTS.md)) and, via a second
|
|
55
|
+
hook, a secret typed directly into the prompt itself — confirmed against
|
|
56
|
+
Claude Code's own docs to block before the model ever processes it. All
|
|
57
|
+
four are covered in [docs/features.md](docs/features.md).
|
|
56
58
|
|
|
57
59
|
> [!NOTE]
|
|
58
60
|
> gitleaks and trufflehog scan **commits**. residoo scans the **conversation
|
|
@@ -121,6 +123,11 @@ while losing rows, then fixed in public against the classes it was losing
|
|
|
121
123
|
output, no external CSS/JS, nothing to open it needs the network.
|
|
122
124
|
- `--seal --keychain` encrypts every transcript with a finding into a
|
|
123
125
|
local vault. See [docs/architecture.md](docs/architecture.md#sealing-what-it-finds).
|
|
126
|
+
- `--ocr` reads secrets out of a pasted or tool-returned screenshot, too —
|
|
127
|
+
a real, verified-unclaimed gap: nobody else in this space has shipped
|
|
128
|
+
this. Opt-in, needs `tesseract` installed, 100% local, best-effort (OCR
|
|
129
|
+
can misread a character and miss an exact-format match). See
|
|
130
|
+
[docs/architecture.md](docs/architecture.md#reading-secrets-out-of-pasted-screenshots).
|
|
124
131
|
- Tells you how many **distinct** secrets it found versus how many times
|
|
125
132
|
one got echoed back across tool calls, so the headline number reflects
|
|
126
133
|
real exposure, not repetition.
|
|
@@ -133,7 +140,8 @@ while losing rows, then fixed in public against the classes it was losing
|
|
|
133
140
|
for CI and pre-commit. See [docs/ci.md](docs/ci.md).
|
|
134
141
|
- `residoo watch` / `residoo mcp` / `residoo cred` / `residoo guard`:
|
|
135
142
|
continuous scanning, conversational queries, credential injection
|
|
136
|
-
without pasting, and
|
|
143
|
+
without pasting, and blocking a sensitive file read or a sensitive
|
|
144
|
+
prompt before either happens. See [docs/features.md](docs/features.md).
|
|
137
145
|
|
|
138
146
|
## What it does not do
|
|
139
147
|
|
|
@@ -203,6 +211,8 @@ residoo scan [options]
|
|
|
203
211
|
--no-color disable ANSI colour
|
|
204
212
|
--verify ask each credential's own vendor if it still authenticates
|
|
205
213
|
(real network call; see docs/architecture.md)
|
|
214
|
+
--ocr also OCR pasted/tool-returned images and scan the text
|
|
215
|
+
(needs tesseract installed; no network call; best-effort)
|
|
206
216
|
|
|
207
217
|
--seal encrypt every transcript with findings into a local vault
|
|
208
218
|
--vault-dir <dir> vault location (default ./residoo-vault-<stamp>)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "residoo",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.11.0",
|
|
4
4
|
"description": "Find secrets leaking through your AI coding agent's session history. Zero network calls in the scan path, zero dependencies.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "CloudRoam (https://cloudroam.io)",
|
package/src/cli.js
CHANGED
|
@@ -138,6 +138,18 @@ Scan options:
|
|
|
138
138
|
JWT's own signed exp claim is checked locally
|
|
139
139
|
with no network call at all, on by default, not
|
|
140
140
|
part of --verify.
|
|
141
|
+
--ocr also OCR every pasted or tool-returned image found
|
|
142
|
+
in a transcript (a screenshot of a .env file or a
|
|
143
|
+
cloud console, for example) and scan the extracted
|
|
144
|
+
text the same way. Off by default: needs
|
|
145
|
+
tesseract installed (e.g. brew install tesseract;
|
|
146
|
+
residoo does not bundle it), and it is real CPU
|
|
147
|
+
work per image. No network call -- OCR runs
|
|
148
|
+
100% locally, same as everything else. OCR is
|
|
149
|
+
lossy by nature: a real test found visually
|
|
150
|
+
similar characters (0/O, Y/*) can be misread,
|
|
151
|
+
which breaks an exact-format match, so this is
|
|
152
|
+
best-effort additional coverage, not a guarantee.
|
|
141
153
|
|
|
142
154
|
Watch:
|
|
143
155
|
residoo watch continuous scanning instead of one snapshot:
|
|
@@ -206,24 +218,47 @@ Cred:
|
|
|
206
218
|
tool, present only when RESIDOO_CRED_ALLOWED_COMMANDS is configured.
|
|
207
219
|
|
|
208
220
|
Guard:
|
|
209
|
-
residoo guard
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
221
|
+
residoo guard one binary, two Claude Code hooks, dispatched on
|
|
222
|
+
the payload's own hook_event_name:
|
|
223
|
+
|
|
224
|
+
PreToolUse blocks an obviously-sensitive file read
|
|
225
|
+
(.env, id_rsa, .aws/credentials, and similar)
|
|
226
|
+
before it can be written to the session
|
|
227
|
+
transcript at all. Narrower than it sounds:
|
|
228
|
+
Claude Code's hooks API can see a proposed Bash
|
|
229
|
+
command or Read path before it runs, but never
|
|
230
|
+
the command's OUTPUT, so this alone cannot catch
|
|
231
|
+
a secret typed into a prompt or one arriving
|
|
232
|
+
through an unrelated command's output.
|
|
233
|
+
|
|
234
|
+
UserPromptSubmit closes exactly that gap: it
|
|
235
|
+
checks the user's own typed prompt against
|
|
236
|
+
residoo's 79 high-confidence rules (never
|
|
237
|
+
--verify, never --ocr -- this hook has no matcher
|
|
238
|
+
and fires on every single prompt, so it must stay
|
|
239
|
+
fast) and can block it before Claude processes it
|
|
240
|
+
at all -- confirmed directly against Claude
|
|
241
|
+
Code's own docs, not assumed. A block ERASES the
|
|
242
|
+
user's whole typed message, so this path is
|
|
243
|
+
extra-conservative: a documented vendor-example
|
|
244
|
+
key or an obvious placeholder is suppressed, same
|
|
245
|
+
as residoo scan itself. Best-effort, not a
|
|
246
|
+
guarantee: per Claude Code's own docs, a slow
|
|
247
|
+
hook here fails OPEN (the prompt goes through
|
|
248
|
+
unscanned) rather than stalling the session,
|
|
249
|
+
consistent with this hook never blocking on
|
|
250
|
+
anything it doesn't recognize.
|
|
251
|
+
|
|
252
|
+
scan/watch/mcp remain the real safety net either
|
|
253
|
+
way -- this is prevention on top of detection,
|
|
254
|
+
not a replacement for it. Add both to
|
|
223
255
|
.claude/settings.json:
|
|
224
|
-
{"hooks":{
|
|
225
|
-
|
|
226
|
-
|
|
256
|
+
{"hooks":{
|
|
257
|
+
"PreToolUse":[{"matcher":"Bash|Read",
|
|
258
|
+
"hooks":[{"type":"command",
|
|
259
|
+
"command":"residoo guard"}]}],
|
|
260
|
+
"UserPromptSubmit":[{"hooks":[{"type":"command",
|
|
261
|
+
"command":"residoo guard"}]}]}}
|
|
227
262
|
|
|
228
263
|
Rotation:
|
|
229
264
|
residoo explain <rule-id> full rotation runbook for one detection rule
|
|
@@ -876,6 +911,13 @@ async function main(argv) {
|
|
|
876
911
|
// API call; see verify.js). Off by default; every other flag here only
|
|
877
912
|
// changes what is READ or how it is DISPLAYED.
|
|
878
913
|
const verify = args.includes("--verify");
|
|
914
|
+
// --ocr: no network call (tesseract runs 100% locally), but it does shell
|
|
915
|
+
// out to a binary this project doesn't ship and do real per-image CPU
|
|
916
|
+
// work, unlike every other flag here — same "off by default, opt in for
|
|
917
|
+
// a reason" posture as --verify, different reason. See ocr.js for the
|
|
918
|
+
// exact confirmed image shape this looks for and its honest accuracy
|
|
919
|
+
// limitations.
|
|
920
|
+
const wantsOcr = args.includes("--ocr");
|
|
879
921
|
|
|
880
922
|
// --project [dir]: the dir is optional (CI passes ".", a bare --project
|
|
881
923
|
// means the current directory). null means machine mode.
|
|
@@ -987,7 +1029,7 @@ async function main(argv) {
|
|
|
987
1029
|
|
|
988
1030
|
const progress = makeProgressReporter(noColor);
|
|
989
1031
|
const result = await scan({
|
|
990
|
-
sources, includeNoisy, includeSuppressed, verify, noColor,
|
|
1032
|
+
sources, includeNoisy, includeSuppressed, verify, noColor, ocr: wantsOcr,
|
|
991
1033
|
onProgress: progress.onProgress,
|
|
992
1034
|
// Clears the spinner's last frame before --verify's own stderr lines
|
|
993
1035
|
// print; without this the last spinner line sits uncleared on screen
|
|
@@ -998,6 +1040,13 @@ async function main(argv) {
|
|
|
998
1040
|
onBeforeVerify: progress.stop,
|
|
999
1041
|
});
|
|
1000
1042
|
progress.stop();
|
|
1043
|
+
// Always stderr, never gated on --json/--sarif: those formats' stdout
|
|
1044
|
+
// contract is machine-readable output only, but a user who asked for
|
|
1045
|
+
// --ocr and got silently zero image findings because tesseract isn't
|
|
1046
|
+
// installed needs to know that, not infer it from an empty result.
|
|
1047
|
+
if (result.ocrRequestedButMissing) {
|
|
1048
|
+
process.stderr.write("--ocr was requested but tesseract is not installed or not on PATH; no images were scanned. Install it (e.g. \"brew install tesseract\") and rerun.\n");
|
|
1049
|
+
}
|
|
1001
1050
|
const integrity = wantsIntegrity ? runIntegrity() : null;
|
|
1002
1051
|
const rotation = renderRotation(result.findings, acks, dismissed);
|
|
1003
1052
|
process.stdout.write((wantsSarif
|
package/src/guard.js
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
|
-
* `residoo guard`:
|
|
5
|
-
*
|
|
6
|
-
*
|
|
4
|
+
* `residoo guard`: two Claude Code hooks in one binary, dispatched on the
|
|
5
|
+
* payload's own `hook_event_name` field.
|
|
6
|
+
*
|
|
7
|
+
* PreToolUse blocks an obviously-sensitive file read before it happens,
|
|
8
|
+
* instead of finding the leak in the transcript afterward.
|
|
7
9
|
*
|
|
8
10
|
* Scope, stated plainly because it is much narrower than "prevent secrets
|
|
9
11
|
* from leaking": Claude Code's hooks API gives a PreToolUse hook the
|
|
@@ -17,17 +19,53 @@
|
|
|
17
19
|
* and similar) -- it cannot catch a secret typed directly into a prompt, a
|
|
18
20
|
* secret arriving in the output of an otherwise-unremarkable command
|
|
19
21
|
* (curl, a build log), or any file path this pattern list does not name.
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
+
*
|
|
23
|
+
* UserPromptSubmit closes exactly the "typed directly into a prompt" gap
|
|
24
|
+
* named above: Claude Code's own docs (code.claude.com/docs/en/hooks,
|
|
25
|
+
* fetched and read directly, not summarized secondhand) confirm this hook
|
|
26
|
+
* "runs before every prompt and blocks model processing until it
|
|
27
|
+
* completes" -- a real prevention point, not an after-the-fact alert.
|
|
28
|
+
* Verified precisely, not assumed: the payload's `prompt` field carries the
|
|
29
|
+
* exact submitted text (the field is `prompt`, NOT `user_prompt` -- a
|
|
30
|
+
* specific claim to the contrary did not survive this session's own
|
|
31
|
+
* adversarial verification pass), and a JSON response with
|
|
32
|
+
* `"decision": "block"` "prevents the prompt from being processed and
|
|
33
|
+
* erases it from context." Checked against residoo's existing 79
|
|
34
|
+
* high-confidence PATTERNS.js rules ONLY -- never NOISY_PATTERNS, never
|
|
35
|
+
* --verify (a network call), never --ocr (tesseract, and irrelevant to a
|
|
36
|
+
* text prompt anyway) -- because this hook has NO matcher support (it
|
|
37
|
+
* fires on every single prompt in every session, confirmed in the same
|
|
38
|
+
* docs) and must stay fast: Claude Code's own default timeout for this
|
|
39
|
+
* event's command hooks is 30s, down from 600s elsewhere, specifically
|
|
40
|
+
* because "a stuck hook stalls the session." A false positive here also
|
|
41
|
+
* costs more than a PreToolUse block: it erases the user's entire typed
|
|
42
|
+
* message, not one denied tool call, which is why this path additionally
|
|
43
|
+
* runs the same vendor-example/placeholder suppression `residoo scan`
|
|
44
|
+
* itself uses (see scan.js's VENDOR_EXAMPLE_VALUES/zeroEntropyTail) before
|
|
45
|
+
* ever blocking -- a documented AWS example key or an obvious placeholder
|
|
46
|
+
* must never eat a real prompt.
|
|
47
|
+
*
|
|
48
|
+
* Same fail-open timeout reality as scan.js's whole 30-vendor --verify
|
|
49
|
+
* surface, disclosed rather than hidden: per Claude Code's own docs, a
|
|
50
|
+
* command/http/mcp_tool hook (this one) that times out on UserPromptSubmit
|
|
51
|
+
* has its output discarded and "the prompt still reaches Claude" unblocked
|
|
52
|
+
* -- a genuinely slow scan cannot stall the session, but that also means
|
|
53
|
+
* this is best-effort, additive coverage, not an absolute guarantee, the
|
|
54
|
+
* same honesty already applied to --ocr.
|
|
22
55
|
*
|
|
23
56
|
* Fails safe in the direction of NOT blocking on any uncertainty: a
|
|
24
57
|
* malformed hook payload, an unrecognized tool name, or a parse error all
|
|
25
58
|
* fall through to "allow" (no stdout, exit 0) rather than denying a call
|
|
26
59
|
* this module does not understand. The one thing this module must never do
|
|
27
|
-
* is silently hang or crash the agent's turn over a tool call
|
|
28
|
-
* always going to be fine.
|
|
60
|
+
* is silently hang or crash the agent's turn over a tool call -- or a
|
|
61
|
+
* prompt -- that was always going to be fine. `residoo scan`/`watch`/`mcp`
|
|
62
|
+
* remain the actual safety net; this is a narrower, best-effort tripwire on
|
|
63
|
+
* top, not a replacement for them.
|
|
29
64
|
*/
|
|
30
65
|
|
|
66
|
+
const { PATTERNS, redact } = require("./patterns");
|
|
67
|
+
const { VENDOR_EXAMPLE_VALUES, zeroEntropyTail } = require("./scan");
|
|
68
|
+
|
|
31
69
|
// A matched path fragment must be preceded by a path separator or the start
|
|
32
70
|
// of the string, and followed by either the end of the string (the common
|
|
33
71
|
// case for Read's file_path) or a shell metacharacter/whitespace (the case
|
|
@@ -127,12 +165,49 @@ function evaluateToolInput(toolName, toolInput) {
|
|
|
127
165
|
};
|
|
128
166
|
}
|
|
129
167
|
|
|
168
|
+
// High-confidence only, computed once at module load (PATTERNS is a static
|
|
169
|
+
// array): never NOISY_PATTERNS, matching the same elevated bar decodeLine/
|
|
170
|
+
// ocrLine already use in scan.js for content one step removed from a plain
|
|
171
|
+
// file line -- a user's own live-typed prompt deserves at least that same
|
|
172
|
+
// bar, arguably a higher one given what a wrong block costs here (see this
|
|
173
|
+
// file's own docstring).
|
|
174
|
+
const PROMPT_GUARD_RULES = PATTERNS.filter((r) => r.confidence === "high");
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* Pure decision function: given a UserPromptSubmit hook payload's raw
|
|
178
|
+
* `prompt` string, decide whether to block. No I/O, fully unit-testable.
|
|
179
|
+
* Suppresses the same way `residoo scan` does (a documented vendor-example
|
|
180
|
+
* key, or an obviously-placeholder zero-entropy tail) before ever blocking
|
|
181
|
+
* -- a false positive here erases the user's entire typed message, not one
|
|
182
|
+
* denied tool call, so this path is deliberately more conservative than
|
|
183
|
+
* evaluateToolInput above, not just a copy of the same bar.
|
|
184
|
+
*/
|
|
185
|
+
function evaluatePromptText(promptText) {
|
|
186
|
+
if (typeof promptText !== "string" || !promptText) return { block: false, reason: null };
|
|
187
|
+
for (const rule of PROMPT_GUARD_RULES) {
|
|
188
|
+
rule.re.lastIndex = 0;
|
|
189
|
+
const m = rule.re.exec(promptText);
|
|
190
|
+
if (!m) continue;
|
|
191
|
+
const value = m[0];
|
|
192
|
+
if (VENDOR_EXAMPLE_VALUES.has(value) || zeroEntropyTail(value)) continue;
|
|
193
|
+
return {
|
|
194
|
+
block: true,
|
|
195
|
+
reason: `residoo guard: this prompt looks like it contains ${rule.label} (${redact(value)}). ` +
|
|
196
|
+
`Blocked before it could be sent. If this is a false positive, rephrase or remove it, or disable this hook in .claude/settings.json.`,
|
|
197
|
+
};
|
|
198
|
+
}
|
|
199
|
+
return { block: false, reason: null };
|
|
200
|
+
}
|
|
201
|
+
|
|
130
202
|
/**
|
|
131
|
-
* Reads one PreToolUse hook payload from `input`
|
|
132
|
-
*
|
|
133
|
-
*
|
|
134
|
-
*
|
|
135
|
-
*
|
|
203
|
+
* Reads one PreToolUse OR UserPromptSubmit hook payload from `input`
|
|
204
|
+
* (default stdin) -- distinguished by the payload's own `hook_event_name`
|
|
205
|
+
* common field, a single binary handling both the way Claude Code's own
|
|
206
|
+
* hook registration allows -- decides, and writes the hook's own JSON
|
|
207
|
+
* response protocol to `output` (default stdout) -- exit code is the
|
|
208
|
+
* caller's job (bin/residoo.js), this returns the intended process exit
|
|
209
|
+
* code instead of calling process.exit itself, matching every other run*
|
|
210
|
+
* function in cli.js.
|
|
136
211
|
*/
|
|
137
212
|
async function runGuard({ input = process.stdin, output = process.stdout } = {}) {
|
|
138
213
|
const chunks = [];
|
|
@@ -146,6 +221,20 @@ async function runGuard({ input = process.stdin, output = process.stdout } = {})
|
|
|
146
221
|
return 0; // malformed payload: fail open, never block on something we can't parse
|
|
147
222
|
}
|
|
148
223
|
|
|
224
|
+
// hook_event_name is a documented COMMON field present on every hook
|
|
225
|
+
// event's payload (code.claude.com/docs/en/hooks), not specific to one
|
|
226
|
+
// event -- checked first so a UserPromptSubmit payload (which has no
|
|
227
|
+
// tool_name/tool_input at all) never falls through to evaluateToolInput
|
|
228
|
+
// and is instead routed to its own decision function and its own
|
|
229
|
+
// response shape (decision:"block", not hookSpecificOutput.
|
|
230
|
+
// permissionDecision -- the two events do not share a response schema).
|
|
231
|
+
if (payload.hook_event_name === "UserPromptSubmit") {
|
|
232
|
+
const decision = evaluatePromptText(payload.prompt);
|
|
233
|
+
if (!decision.block) return 0;
|
|
234
|
+
output.write(JSON.stringify({ decision: "block", reason: decision.reason }) + "\n");
|
|
235
|
+
return 0;
|
|
236
|
+
}
|
|
237
|
+
|
|
149
238
|
const decision = evaluateToolInput(payload.tool_name, payload.tool_input);
|
|
150
239
|
if (!decision.block) return 0;
|
|
151
240
|
|
|
@@ -159,4 +248,4 @@ async function runGuard({ input = process.stdin, output = process.stdout } = {})
|
|
|
159
248
|
return 0;
|
|
160
249
|
}
|
|
161
250
|
|
|
162
|
-
module.exports = { evaluateToolInput, matchSensitivePath, runGuard, SENSITIVE_PATH_PATTERNS };
|
|
251
|
+
module.exports = { evaluateToolInput, matchSensitivePath, evaluatePromptText, runGuard, SENSITIVE_PATH_PATTERNS };
|
package/src/ocr.js
ADDED
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Opt-in OCR of pasted-screenshot images inside a transcript (--ocr).
|
|
5
|
+
*
|
|
6
|
+
* Everything else in residoo detects secrets in TEXT already sitting in a
|
|
7
|
+
* transcript line. This module covers a real, verified-unclaimed gap: a
|
|
8
|
+
* user pastes a screenshot of a .env file, a cloud console page, or a
|
|
9
|
+
* terminal into their AI agent, and the credential in that image is
|
|
10
|
+
* invisible to every text-based rule in patterns.js. Real, ground-truth
|
|
11
|
+
* inspection of this machine's own Claude Code session files confirms the
|
|
12
|
+
* exact shape a pasted or tool-returned image takes in the JSONL transcript
|
|
13
|
+
* (both as a direct message content block and nested inside a tool_result):
|
|
14
|
+
* {"type":"image","source":{"type":"base64","media_type":"image/png","data":"<base64>"}}
|
|
15
|
+
* This module's job stops at extracting that data and turning it into text;
|
|
16
|
+
* the text then flows through the exact same PATTERNS rules and redact()
|
|
17
|
+
* every other line in a transcript does — no new detection logic, no new
|
|
18
|
+
* false-positive surface, just a new place text can come from.
|
|
19
|
+
*
|
|
20
|
+
* Same shell-out posture as verify.js's AWS check, for the same reason:
|
|
21
|
+
* residoo ships zero runtime dependencies, and a correct from-scratch OCR
|
|
22
|
+
* engine is not something this project could build or verify. tesseract is
|
|
23
|
+
* the mature, widely-packaged, offline OCR engine every major distro and
|
|
24
|
+
* Homebrew ships; shelling out to an already-installed copy costs nothing
|
|
25
|
+
* at install time and adds no dependency residoo itself carries. Off by
|
|
26
|
+
* default: it requires tesseract to be installed, and it is real CPU work
|
|
27
|
+
* per image, unlike every other rule in this file which is a regex over
|
|
28
|
+
* text already in memory.
|
|
29
|
+
*
|
|
30
|
+
* Image bytes go to tesseract over stdin and its output is read back over
|
|
31
|
+
* stdout -- never written to a file, matching every other credential-
|
|
32
|
+
* bearing value in this codebase never touching disk unless --seal is
|
|
33
|
+
* explicitly asked for. Nothing here makes a network call; tesseract's own
|
|
34
|
+
* OCR is 100% local.
|
|
35
|
+
*
|
|
36
|
+
* HONEST LIMITATION, found by testing this module against a real rendered
|
|
37
|
+
* image before shipping it (not assumed): OCR is lossy. A real test against
|
|
38
|
+
* a clean, large, monospace "AKIASM0KETESTFAKEKEY"-shaped string produced
|
|
39
|
+
* "AKIASM@KETESTFAKEKE*" at low resolution and "AKIASMOKETESTFAKEKE*" (0
|
|
40
|
+
* misread as O, trailing Y misread as *) even at 2x resolution -- visually
|
|
41
|
+
* similar characters (0/O, Y/*) are a real, inherent tesseract failure
|
|
42
|
+
* mode, not a bug in how this module invokes it. A single misread character
|
|
43
|
+
* breaks an exact-format regex match. This means --ocr is best-effort
|
|
44
|
+
* additive coverage on a previously-zero-coverage surface, not a guarantee
|
|
45
|
+
* every credential in every screenshot will be caught -- documented here so
|
|
46
|
+
* that claim is never overstated in the CLI help text or README either.
|
|
47
|
+
*/
|
|
48
|
+
|
|
49
|
+
const { spawnSync } = require("child_process");
|
|
50
|
+
|
|
51
|
+
// Test-only escape hatch, same pattern as verify.js's RESIDOO_TEST_AWS_CLI:
|
|
52
|
+
// when set, every spawnSync call below runs that path instead of
|
|
53
|
+
// "tesseract" on PATH, so tests exercise the real spawnSync + stdin/stdout
|
|
54
|
+
// plumbing against a small fixture script rather than requiring the real
|
|
55
|
+
// tesseract binary (or the network) on every machine that runs `npm test`.
|
|
56
|
+
function tesseractBinary() {
|
|
57
|
+
return process.env.RESIDOO_TEST_TESSERACT || "tesseract";
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function isTesseractAvailable(spawnFn = spawnSync) {
|
|
61
|
+
try {
|
|
62
|
+
const r = spawnFn(tesseractBinary(), ["--version"], {
|
|
63
|
+
timeout: 5000,
|
|
64
|
+
env: { PATH: process.env.PATH || "" },
|
|
65
|
+
stdio: ["ignore", "ignore", "ignore"],
|
|
66
|
+
});
|
|
67
|
+
return !r.error && r.status === 0;
|
|
68
|
+
} catch {
|
|
69
|
+
return false;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
// A real screenshot is rarely more than a few MB; this is a generous
|
|
74
|
+
// ceiling against a maliciously or accidentally huge "image" field in an
|
|
75
|
+
// attacker-plantable transcript, not a real-world limit. Base64 is ~4/3
|
|
76
|
+
// the decoded size, hence the larger character-count bound.
|
|
77
|
+
const MAX_BASE64_CHARS = 30_000_000; // ~22 MB decoded
|
|
78
|
+
// A single line with hundreds of embedded images (crafted or corrupted)
|
|
79
|
+
// must not turn --ocr into a hang; cap how many this module will even
|
|
80
|
+
// attempt per line. Real transcripts have at most a handful of images per
|
|
81
|
+
// message.
|
|
82
|
+
const MAX_BLOCKS_PER_LINE = 8;
|
|
83
|
+
const MAX_WALK_DEPTH = 12; // defensive bound against pathological nesting
|
|
84
|
+
|
|
85
|
+
const KNOWN_IMAGE_MEDIA_TYPES = new Set(["image/png", "image/jpeg", "image/webp", "image/gif"]);
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Find every {type:"image", source:{type:"base64", data, media_type}}
|
|
89
|
+
* block in a transcript line, at any nesting depth (a direct message
|
|
90
|
+
* content block and a tool_result's nested content block are both real,
|
|
91
|
+
* confirmed shapes -- see this file's own docstring). Returns
|
|
92
|
+
* [{ data, mediaType }], capped at MAX_BLOCKS_PER_LINE.
|
|
93
|
+
*
|
|
94
|
+
* Not every source's lines are JSON (or valid JSON) -- a malformed or
|
|
95
|
+
* partial line fails JSON.parse and this returns [] rather than throwing,
|
|
96
|
+
* the same fail-quiet-on-this-one-line posture decode.js's contentProjection
|
|
97
|
+
* already has for the exact same reason.
|
|
98
|
+
*/
|
|
99
|
+
function extractImageBlocks(line) {
|
|
100
|
+
const t = typeof line === "string" ? line.trim() : "";
|
|
101
|
+
if (t[0] !== "{" && t[0] !== "[") return [];
|
|
102
|
+
let parsed;
|
|
103
|
+
try { parsed = JSON.parse(t); } catch { return []; }
|
|
104
|
+
|
|
105
|
+
const out = [];
|
|
106
|
+
const walk = (node, depth) => {
|
|
107
|
+
if (out.length >= MAX_BLOCKS_PER_LINE || depth > MAX_WALK_DEPTH || node == null || typeof node !== "object") return;
|
|
108
|
+
if (Array.isArray(node)) {
|
|
109
|
+
for (const item of node) { if (out.length >= MAX_BLOCKS_PER_LINE) return; walk(item, depth + 1); }
|
|
110
|
+
return;
|
|
111
|
+
}
|
|
112
|
+
const source = node.source;
|
|
113
|
+
if (
|
|
114
|
+
node.type === "image" && source && typeof source === "object" &&
|
|
115
|
+
source.type === "base64" && typeof source.data === "string" && source.data.length > 0 &&
|
|
116
|
+
source.data.length <= MAX_BASE64_CHARS &&
|
|
117
|
+
KNOWN_IMAGE_MEDIA_TYPES.has(source.media_type)
|
|
118
|
+
) {
|
|
119
|
+
out.push({ data: source.data, mediaType: source.media_type });
|
|
120
|
+
return; // an image block's own fields are never themselves nested image blocks
|
|
121
|
+
}
|
|
122
|
+
for (const key of Object.keys(node)) { if (out.length >= MAX_BLOCKS_PER_LINE) return; walk(node[key], depth + 1); }
|
|
123
|
+
};
|
|
124
|
+
walk(parsed, 0);
|
|
125
|
+
return out;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
const DEFAULT_TIMEOUT_MS = 20_000;
|
|
129
|
+
|
|
130
|
+
/** Strip control bytes: OCR output flows into the exact same matching/redaction path as any other text, but must never carry a raw control byte into a terminal. */
|
|
131
|
+
function stripControlChars(s) { return String(s || "").replace(/[\x00-\x1f\x7f]/g, ""); }
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* OCR one image's base64 data via tesseract over stdin/stdout. Returns
|
|
135
|
+
* { text, error }: text is "" (never null) on any failure, so a caller
|
|
136
|
+
* never needs a null check before feeding it through the pattern-matching
|
|
137
|
+
* loop; error names why when text is empty, for --ocr's own diagnostics,
|
|
138
|
+
* never surfaced as a scan failure (an unreadable or corrupt image is not
|
|
139
|
+
* a reason to fail the whole scan).
|
|
140
|
+
*/
|
|
141
|
+
function ocrImageBase64(base64Data, { spawnFn = spawnSync, timeoutMs = DEFAULT_TIMEOUT_MS } = {}) {
|
|
142
|
+
let buf;
|
|
143
|
+
try {
|
|
144
|
+
buf = Buffer.from(base64Data, "base64");
|
|
145
|
+
} catch (e) {
|
|
146
|
+
return { text: "", error: `could not decode base64 image data (${e && e.message})` };
|
|
147
|
+
}
|
|
148
|
+
if (buf.length === 0) return { text: "", error: "decoded image was empty" };
|
|
149
|
+
|
|
150
|
+
let r;
|
|
151
|
+
try {
|
|
152
|
+
r = spawnFn(tesseractBinary(), ["stdin", "stdout"], {
|
|
153
|
+
input: buf,
|
|
154
|
+
timeout: timeoutMs,
|
|
155
|
+
maxBuffer: 10 * 1024 * 1024,
|
|
156
|
+
env: { PATH: process.env.PATH || "" },
|
|
157
|
+
});
|
|
158
|
+
} catch (e) {
|
|
159
|
+
return { text: "", error: `tesseract failed to run (${e && e.message})` };
|
|
160
|
+
}
|
|
161
|
+
if (r.error) {
|
|
162
|
+
if (r.error.code === "ENOENT") return { text: "", error: "tesseract not found on PATH" };
|
|
163
|
+
if (r.error.code === "ETIMEDOUT") return { text: "", error: `tesseract timed out after ${timeoutMs}ms` };
|
|
164
|
+
return { text: "", error: `tesseract failed to run (${r.error.code || r.error.message})` };
|
|
165
|
+
}
|
|
166
|
+
if (r.status !== 0) {
|
|
167
|
+
return { text: "", error: `tesseract exited ${r.status}` };
|
|
168
|
+
}
|
|
169
|
+
return { text: stripControlChars((r.stdout || "").toString("utf-8")), error: null };
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
module.exports = { isTesseractAvailable, extractImageBlocks, ocrImageBase64, tesseractBinary };
|
package/src/report.js
CHANGED
|
@@ -466,9 +466,11 @@ function render({ findings, filesScanned, sourcesScanned, bytesScanned, suppress
|
|
|
466
466
|
// so the reader should know the value was hidden.
|
|
467
467
|
const encoded = items.filter((f) => f.encoding).length;
|
|
468
468
|
const split = items.filter((f) => f.spanLines).length;
|
|
469
|
+
const ocrd = items.filter((f) => f.ocr).length;
|
|
469
470
|
const marks = [];
|
|
470
471
|
if (encoded) marks.push(`${encoded} base64-wrapped`);
|
|
471
472
|
if (split) marks.push(`${split} split across lines`);
|
|
473
|
+
if (ocrd) marks.push(`${ocrd} read from a pasted image (--ocr)`);
|
|
472
474
|
const markNote = marks.length ? paint(c.yellow, ` [${marks.join(", ")}]`) : "";
|
|
473
475
|
const paddedLabel = label.length <= labelWidth ? label.padEnd(labelWidth) : label;
|
|
474
476
|
push(` ${paint(color + c.bold, String(items.length).padStart(4))} [${tag}] ${paddedLabel}${distinctNote}${markNote}`);
|
|
@@ -526,17 +528,21 @@ function renderJson(result, integrity = null, rotation = null) {
|
|
|
526
528
|
bytesScanned: result.bytesScanned,
|
|
527
529
|
suppressedCount: result.suppressedCount || 0,
|
|
528
530
|
unreadableFiles: result.unreadableFiles || [],
|
|
531
|
+
ocrRequestedButMissing: result.ocrRequestedButMissing || false,
|
|
529
532
|
},
|
|
530
533
|
findings: result.findings.map((f) => ({
|
|
531
534
|
rule: f.ruleId, label: f.label, confidence: f.confidence,
|
|
532
535
|
source: f.source, file: f.relFile, line: f.line, preview: f.preview,
|
|
533
536
|
fileMTimeMs: f.fileMTimeMs,
|
|
534
|
-
// Markers for the
|
|
537
|
+
// Markers for the decode/reconstruct/OCR passes (absent on ordinary
|
|
535
538
|
// findings). `encoding` names how the value was wrapped ("base64" /
|
|
536
539
|
// "base64url"); `spanLines` names the adjacent line pair a split value
|
|
537
|
-
// was reconstructed across
|
|
540
|
+
// was reconstructed across; `ocr` means the value was never plain
|
|
541
|
+
// text at all -- it was read out of a pasted or tool-returned image
|
|
542
|
+
// (see ocr.js).
|
|
538
543
|
...(f.encoding ? { encoding: f.encoding } : {}),
|
|
539
544
|
...(f.spanLines ? { spanLines: f.spanLines } : {}),
|
|
545
|
+
...(f.ocr ? { ocr: true } : {}),
|
|
540
546
|
fingerprint: fingerprintFinding(f),
|
|
541
547
|
// Only present on an --include-suppressed run: says WHY this finding
|
|
542
548
|
// is low-confidence, so a JSON consumer doesn't have to guess.
|
package/src/scan.js
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
const path = require("path");
|
|
4
4
|
const { PATTERNS, NOISY_PATTERNS, redact } = require("./patterns");
|
|
5
5
|
const { findDecodedMatches, findBoundaryMatches, contentProjection } = require("./decode");
|
|
6
|
+
const { isTesseractAvailable, extractImageBlocks, ocrImageBase64 } = require("./ocr");
|
|
6
7
|
const { findPairedSecret, findNearbyCandidate } = require("./pairing");
|
|
7
8
|
const { looksRandom } = require("./rarity");
|
|
8
9
|
const { decodeJwtExpiryMs } = require("./jwtExpiry");
|
|
@@ -266,8 +267,15 @@ function localTimestamp(d) {
|
|
|
266
267
|
* absolute path can itself carry a username or a project name the rest of
|
|
267
268
|
* this report is careful never to print.
|
|
268
269
|
*/
|
|
269
|
-
async function scan({ sources, includeNoisy = false, includeSuppressed = false, onProgress = null, verify = false, verifyOnlyFingerprint = null, onBeforeVerify = null, noColor = false } = {}) {
|
|
270
|
+
async function scan({ sources, includeNoisy = false, includeSuppressed = false, onProgress = null, verify = false, verifyOnlyFingerprint = null, onBeforeVerify = null, noColor = false, ocr = false } = {}) {
|
|
270
271
|
const rules = includeNoisy ? PATTERNS.concat(NOISY_PATTERNS) : PATTERNS;
|
|
272
|
+
// --ocr: checked once, not per line/image -- isTesseractAvailable shells
|
|
273
|
+
// out, and this scan can touch thousands of lines. ocrRequestedButMissing
|
|
274
|
+
// flows back to the caller (see the return value below) so a user who
|
|
275
|
+
// asked for --ocr without tesseract installed gets a clear, once-per-scan
|
|
276
|
+
// message, not silence and zero image findings.
|
|
277
|
+
const ocrReady = ocr && isTesseractAvailable();
|
|
278
|
+
const ocrRequestedButMissing = ocr && !ocrReady;
|
|
271
279
|
// The decode pass (see decode.js) only applies high-confidence, vendor-
|
|
272
280
|
// prefixed rules to decoded bytes: random binary that decodes to printable
|
|
273
281
|
// text can shape-match a generic rule, but not a vendor prefix. NOISY rules
|
|
@@ -559,6 +567,34 @@ async function scan({ sources, includeNoisy = false, includeSuppressed = false,
|
|
|
559
567
|
}
|
|
560
568
|
};
|
|
561
569
|
|
|
570
|
+
// --ocr only: a line whose JSON shape holds a pasted or tool-returned
|
|
571
|
+
// image (see ocr.js's docstring for the exact confirmed shape) gets each
|
|
572
|
+
// image block decoded and OCR'd, and the extracted text runs through the
|
|
573
|
+
// same high-confidence rules the decode pass above uses, for the same
|
|
574
|
+
// reason -- a step removed from literal transcript text deserves the
|
|
575
|
+
// higher bar. Every finding carries an `ocr: true` marker so a report can
|
|
576
|
+
// say where the value actually came from.
|
|
577
|
+
const ocrLine = (line, file, relFile, lineNo, mtimeMs) => {
|
|
578
|
+
if (!ocrReady) return;
|
|
579
|
+
for (const block of extractImageBlocks(line)) {
|
|
580
|
+
const { text } = ocrImageBase64(block.data);
|
|
581
|
+
if (!text) continue;
|
|
582
|
+
for (const rule of highRules) {
|
|
583
|
+
rule.re.lastIndex = 0;
|
|
584
|
+
let m;
|
|
585
|
+
while ((m = rule.re.exec(text)) !== null) {
|
|
586
|
+
const suppressedReason = suppressionReason(m[0], null, rule.id);
|
|
587
|
+
if (suppressedReason && !includeSuppressed) {
|
|
588
|
+
suppressedCount++;
|
|
589
|
+
continue;
|
|
590
|
+
}
|
|
591
|
+
record(rule, m[0], relFile, file, lineNo, mtimeMs,
|
|
592
|
+
suppressedReason ? "low" : rule.confidence, suppressedReason, { ocr: true });
|
|
593
|
+
}
|
|
594
|
+
}
|
|
595
|
+
}
|
|
596
|
+
};
|
|
597
|
+
|
|
562
598
|
// Feature 2: split-line boundary join. A finding here means one credential
|
|
563
599
|
// was split across this line and the next and is contiguous on neither. It
|
|
564
600
|
// is recorded against BOTH contributing lines (each holds a fragment of the
|
|
@@ -669,6 +705,13 @@ async function scan({ sources, includeNoisy = false, includeSuppressed = false,
|
|
|
669
705
|
} catch (err) {
|
|
670
706
|
flagFailed();
|
|
671
707
|
}
|
|
708
|
+
if (ocrReady) {
|
|
709
|
+
try {
|
|
710
|
+
ocrLine(line, file, relFile, i + 1, mtimeMs);
|
|
711
|
+
} catch (err) {
|
|
712
|
+
flagFailed();
|
|
713
|
+
}
|
|
714
|
+
}
|
|
672
715
|
try {
|
|
673
716
|
const content = contentProjection(line);
|
|
674
717
|
// Boundary join with the previous line (2-way splits only; see
|
|
@@ -872,7 +915,7 @@ async function scan({ sources, includeNoisy = false, includeSuppressed = false,
|
|
|
872
915
|
|
|
873
916
|
const distinctCounts = {};
|
|
874
917
|
for (const [ruleId, set] of distinctByRule) distinctCounts[ruleId] = set.size;
|
|
875
|
-
return { findings, filesScanned, sourcesScanned, bytesScanned, suppressedCount, distinctCounts, unreadableFiles };
|
|
918
|
+
return { findings, filesScanned, sourcesScanned, bytesScanned, suppressedCount, distinctCounts, unreadableFiles, ocrRequestedButMissing };
|
|
876
919
|
}
|
|
877
920
|
|
|
878
921
|
/**
|
|
@@ -884,7 +927,7 @@ async function scan({ sources, includeNoisy = false, includeSuppressed = false,
|
|
|
884
927
|
function emptyResult() {
|
|
885
928
|
return {
|
|
886
929
|
findings: [], filesScanned: 0, sourcesScanned: [], bytesScanned: 0,
|
|
887
|
-
suppressedCount: 0, distinctCounts: {}, unreadableFiles: [],
|
|
930
|
+
suppressedCount: 0, distinctCounts: {}, unreadableFiles: [], ocrRequestedButMissing: false,
|
|
888
931
|
};
|
|
889
932
|
}
|
|
890
933
|
|
|
@@ -900,4 +943,4 @@ function emptyResult() {
|
|
|
900
943
|
// so residoo_verify_finding's v1 only supports the single-token vendors below.
|
|
901
944
|
const VERIFIABLE_RULE_IDS = new Set(Object.keys(SIMPLE_VERIFY_FNS));
|
|
902
945
|
|
|
903
|
-
module.exports = { scan, emptyResult, VENDOR_EXAMPLE_VALUES, VERIFIABLE_RULE_IDS };
|
|
946
|
+
module.exports = { scan, emptyResult, VENDOR_EXAMPLE_VALUES, VERIFIABLE_RULE_IDS, zeroEntropyTail };
|