residoo 0.8.8 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -0
- package/package.json +1 -1
- package/src/cli.js +57 -2
- package/src/ocr.js +172 -0
- package/src/report.js +207 -3
- package/src/scan.js +46 -3
package/README.md
CHANGED
|
@@ -116,8 +116,16 @@ while losing rows, then fixed in public against the classes it was losing
|
|
|
116
116
|
- Redacts everything in its own output, including `--json`: you get a
|
|
117
117
|
shape and a first/last-4 preview, never the real value.
|
|
118
118
|
- `--sarif` emits SARIF 2.1.0 for GitHub code scanning.
|
|
119
|
+
- `--html [path]` writes a self-contained, filterable HTML report with a
|
|
120
|
+
rotation guide per finding — same redaction guarantee as every other
|
|
121
|
+
output, no external CSS/JS, nothing to open it needs the network.
|
|
119
122
|
- `--seal --keychain` encrypts every transcript with a finding into a
|
|
120
123
|
local vault. See [docs/architecture.md](docs/architecture.md#sealing-what-it-finds).
|
|
124
|
+
- `--ocr` reads secrets out of a pasted or tool-returned screenshot, too —
|
|
125
|
+
a real, verified-unclaimed gap: nobody else in this space has shipped
|
|
126
|
+
this. Opt-in, needs `tesseract` installed, 100% local, best-effort (OCR
|
|
127
|
+
can misread a character and miss an exact-format match). See
|
|
128
|
+
[docs/architecture.md](docs/architecture.md#reading-secrets-out-of-pasted-screenshots).
|
|
121
129
|
- Tells you how many **distinct** secrets it found versus how many times
|
|
122
130
|
one got echoed back across tool calls, so the headline number reflects
|
|
123
131
|
real exposure, not repetition.
|
|
@@ -186,6 +194,8 @@ faith.
|
|
|
186
194
|
residoo scan [options]
|
|
187
195
|
|
|
188
196
|
--json machine-readable output (full detail, still redacted)
|
|
197
|
+
--html [path] also write a self-contained HTML report (default:
|
|
198
|
+
residoo-report-<stamp>.html); combines with --json
|
|
189
199
|
--project [dir] scan a repository checkout instead of this machine
|
|
190
200
|
(committed transcripts, agent configs, root .env)
|
|
191
201
|
--include-noisy also run broad, false-positive-prone rules
|
|
@@ -198,6 +208,8 @@ residoo scan [options]
|
|
|
198
208
|
--no-color disable ANSI colour
|
|
199
209
|
--verify ask each credential's own vendor if it still authenticates
|
|
200
210
|
(real network call; see docs/architecture.md)
|
|
211
|
+
--ocr also OCR pasted/tool-returned images and scan the text
|
|
212
|
+
(needs tesseract installed; no network call; best-effort)
|
|
201
213
|
|
|
202
214
|
--seal encrypt every transcript with findings into a local vault
|
|
203
215
|
--vault-dir <dir> vault location (default ./residoo-vault-<stamp>)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "residoo",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.10.0",
|
|
4
4
|
"description": "Find secrets leaking through your AI coding agent's session history. Zero network calls in the scan path, zero dependencies.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "CloudRoam (https://cloudroam.io)",
|
package/src/cli.js
CHANGED
|
@@ -5,7 +5,7 @@ const fs = require("fs");
|
|
|
5
5
|
const crypto = require("crypto");
|
|
6
6
|
const { availableSources, ALL_SOURCES } = require("./sources");
|
|
7
7
|
const { scan, emptyResult } = require("./scan");
|
|
8
|
-
const { render, renderIntegrity, renderJson, renderSarif, makeProgressReporter, printIntro } = require("./report");
|
|
8
|
+
const { render, renderIntegrity, renderJson, renderSarif, renderHtml, makeProgressReporter, printIntro } = require("./report");
|
|
9
9
|
const { checkIntegrity } = require("./integrity");
|
|
10
10
|
const {
|
|
11
11
|
ROTATION_GUIDANCE, guidanceFor, loadAcks, loadDismissed, ackFinding, dismissFinding, renderRotation,
|
|
@@ -78,6 +78,15 @@ Scan options:
|
|
|
78
78
|
GitHub code scanning's Security tab and inline PR
|
|
79
79
|
annotations. Use --json for the full picture
|
|
80
80
|
(findings + integrity + rotation) instead.
|
|
81
|
+
--html [path] also write a self-contained HTML report (default:
|
|
82
|
+
residoo-report-<timestamp>.html in the current
|
|
83
|
+
directory) -- a filterable table with a rotation
|
|
84
|
+
guide per finding, safe to screenshot or share:
|
|
85
|
+
same redacted preview as every other output, no
|
|
86
|
+
raw value in any code path, no external CSS/JS/
|
|
87
|
+
fonts, no network access needed to open it.
|
|
88
|
+
Independent of --json/--sarif; can combine with
|
|
89
|
+
either.
|
|
81
90
|
--project [dir] scan a repository checkout instead of this machine
|
|
82
91
|
(default dir: current directory). Covers committed
|
|
83
92
|
agent transcripts, agent config/rules files, and
|
|
@@ -129,6 +138,18 @@ Scan options:
|
|
|
129
138
|
JWT's own signed exp claim is checked locally
|
|
130
139
|
with no network call at all, on by default, not
|
|
131
140
|
part of --verify.
|
|
141
|
+
--ocr also OCR every pasted or tool-returned image found
|
|
142
|
+
in a transcript (a screenshot of a .env file or a
|
|
143
|
+
cloud console, for example) and scan the extracted
|
|
144
|
+
text the same way. Off by default: needs
|
|
145
|
+
tesseract installed (e.g. brew install tesseract;
|
|
146
|
+
residoo does not bundle it), and it is real CPU
|
|
147
|
+
work per image. No network call -- OCR runs
|
|
148
|
+
100% locally, same as everything else. OCR is
|
|
149
|
+
lossy by nature: a real test found visually
|
|
150
|
+
similar characters (0/O, Y/*) can be misread,
|
|
151
|
+
which breaks an exact-format match, so this is
|
|
152
|
+
best-effort additional coverage, not a guarantee.
|
|
132
153
|
|
|
133
154
|
Watch:
|
|
134
155
|
residoo watch continuous scanning instead of one snapshot:
|
|
@@ -340,6 +361,23 @@ async function resolveUnsealSecret(args, vaultDir) {
|
|
|
340
361
|
}
|
|
341
362
|
}
|
|
342
363
|
|
|
364
|
+
// --html [path]: writes the self-contained HTML report to disk (default
|
|
365
|
+
// residoo-report-<timestamp>.html in the cwd, same naming convention as
|
|
366
|
+
// --seal's default vault dir) and prints where it went. Independent of
|
|
367
|
+
// which stdout format was chosen (text/--json/--sarif), same relationship
|
|
368
|
+
// --seal already has to those — this is a side effect, not another
|
|
369
|
+
// mutually-exclusive output mode.
|
|
370
|
+
function writeHtmlReport(result, integrity, rotation, args) {
|
|
371
|
+
const stamp = new Date().toISOString().replace(/[:.]/g, "-").slice(0, 19);
|
|
372
|
+
// Bare --html is valid (auto-named), so the next token is only taken as a
|
|
373
|
+
// path when it doesn't itself look like another flag — same guard --project
|
|
374
|
+
// uses for the same reason.
|
|
375
|
+
const next = argValue(args, "--html");
|
|
376
|
+
const out = next && !next.startsWith("--") ? path.resolve(next) : path.resolve(`residoo-report-${stamp}.html`);
|
|
377
|
+
fs.writeFileSync(out, renderHtml(result, integrity, rotation));
|
|
378
|
+
process.stdout.write(`HTML report written to ${out}\n`);
|
|
379
|
+
}
|
|
380
|
+
|
|
343
381
|
async function runSeal(result, args) {
|
|
344
382
|
const { sealFindings, uploadVaultToCloudRoam } = require("./sealvault");
|
|
345
383
|
|
|
@@ -850,6 +888,13 @@ async function main(argv) {
|
|
|
850
888
|
// API call; see verify.js). Off by default; every other flag here only
|
|
851
889
|
// changes what is READ or how it is DISPLAYED.
|
|
852
890
|
const verify = args.includes("--verify");
|
|
891
|
+
// --ocr: no network call (tesseract runs 100% locally), but it does shell
|
|
892
|
+
// out to a binary this project doesn't ship and do real per-image CPU
|
|
893
|
+
// work, unlike every other flag here — same "off by default, opt in for
|
|
894
|
+
// a reason" posture as --verify, different reason. See ocr.js for the
|
|
895
|
+
// exact confirmed image shape this looks for and its honest accuracy
|
|
896
|
+
// limitations.
|
|
897
|
+
const wantsOcr = args.includes("--ocr");
|
|
853
898
|
|
|
854
899
|
// --project [dir]: the dir is optional (CI passes ".", a bare --project
|
|
855
900
|
// means the current directory). null means machine mode.
|
|
@@ -955,12 +1000,13 @@ async function main(argv) {
|
|
|
955
1000
|
// a planted repo-level hook in the CWD is exactly as dangerous here.
|
|
956
1001
|
if (integrity) process.stdout.write(renderIntegrity(integrity, { noColor }) + "\n");
|
|
957
1002
|
}
|
|
1003
|
+
if (args.includes("--html")) writeHtmlReport(empty, integrity, renderRotation([], acks, dismissed), args);
|
|
958
1004
|
return failOnFind && integrityWarnCount(integrity) > 0 ? 1 : 0;
|
|
959
1005
|
}
|
|
960
1006
|
|
|
961
1007
|
const progress = makeProgressReporter(noColor);
|
|
962
1008
|
const result = await scan({
|
|
963
|
-
sources, includeNoisy, includeSuppressed, verify, noColor,
|
|
1009
|
+
sources, includeNoisy, includeSuppressed, verify, noColor, ocr: wantsOcr,
|
|
964
1010
|
onProgress: progress.onProgress,
|
|
965
1011
|
// Clears the spinner's last frame before --verify's own stderr lines
|
|
966
1012
|
// print; without this the last spinner line sits uncleared on screen
|
|
@@ -971,6 +1017,13 @@ async function main(argv) {
|
|
|
971
1017
|
onBeforeVerify: progress.stop,
|
|
972
1018
|
});
|
|
973
1019
|
progress.stop();
|
|
1020
|
+
// Always stderr, never gated on --json/--sarif: those formats' stdout
|
|
1021
|
+
// contract is machine-readable output only, but a user who asked for
|
|
1022
|
+
// --ocr and got silently zero image findings because tesseract isn't
|
|
1023
|
+
// installed needs to know that, not infer it from an empty result.
|
|
1024
|
+
if (result.ocrRequestedButMissing) {
|
|
1025
|
+
process.stderr.write("--ocr was requested but tesseract is not installed or not on PATH; no images were scanned. Install it (e.g. \"brew install tesseract\") and rerun.\n");
|
|
1026
|
+
}
|
|
974
1027
|
const integrity = wantsIntegrity ? runIntegrity() : null;
|
|
975
1028
|
const rotation = renderRotation(result.findings, acks, dismissed);
|
|
976
1029
|
process.stdout.write((wantsSarif
|
|
@@ -984,6 +1037,8 @@ async function main(argv) {
|
|
|
984
1037
|
if (sealExit !== 0) return sealExit;
|
|
985
1038
|
}
|
|
986
1039
|
|
|
1040
|
+
if (args.includes("--html")) writeHtmlReport(result, integrity, rotation, args);
|
|
1041
|
+
|
|
987
1042
|
// --allow-acked narrows the SECRET gate only: an acknowledged rotation says
|
|
988
1043
|
// nothing about a planted hook, so integrity warnings always fail. Without
|
|
989
1044
|
// the flag, acks change what the report says, never what CI does — a gate
|
package/src/ocr.js
ADDED
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Opt-in OCR of pasted-screenshot images inside a transcript (--ocr).
|
|
5
|
+
*
|
|
6
|
+
* Everything else in residoo detects secrets in TEXT already sitting in a
|
|
7
|
+
* transcript line. This module covers a real, verified-unclaimed gap: a
|
|
8
|
+
* user pastes a screenshot of a .env file, a cloud console page, or a
|
|
9
|
+
* terminal into their AI agent, and the credential in that image is
|
|
10
|
+
* invisible to every text-based rule in patterns.js. Real, ground-truth
|
|
11
|
+
* inspection of this machine's own Claude Code session files confirms the
|
|
12
|
+
* exact shape a pasted or tool-returned image takes in the JSONL transcript
|
|
13
|
+
* (both as a direct message content block and nested inside a tool_result):
|
|
14
|
+
* {"type":"image","source":{"type":"base64","media_type":"image/png","data":"<base64>"}}
|
|
15
|
+
* This module's job stops at extracting that data and turning it into text;
|
|
16
|
+
* the text then flows through the exact same PATTERNS rules and redact()
|
|
17
|
+
* every other line in a transcript does — no new detection logic, no new
|
|
18
|
+
* false-positive surface, just a new place text can come from.
|
|
19
|
+
*
|
|
20
|
+
* Same shell-out posture as verify.js's AWS check, for the same reason:
|
|
21
|
+
* residoo ships zero runtime dependencies, and a correct from-scratch OCR
|
|
22
|
+
* engine is not something this project could build or verify. tesseract is
|
|
23
|
+
* the mature, widely-packaged, offline OCR engine every major distro and
|
|
24
|
+
* Homebrew ships; shelling out to an already-installed copy costs nothing
|
|
25
|
+
* at install time and adds no dependency residoo itself carries. Off by
|
|
26
|
+
* default: it requires tesseract to be installed, and it is real CPU work
|
|
27
|
+
* per image, unlike every other rule in this file which is a regex over
|
|
28
|
+
* text already in memory.
|
|
29
|
+
*
|
|
30
|
+
* Image bytes go to tesseract over stdin and its output is read back over
|
|
31
|
+
* stdout -- never written to a file, matching every other credential-
|
|
32
|
+
* bearing value in this codebase never touching disk unless --seal is
|
|
33
|
+
* explicitly asked for. Nothing here makes a network call; tesseract's own
|
|
34
|
+
* OCR is 100% local.
|
|
35
|
+
*
|
|
36
|
+
* HONEST LIMITATION, found by testing this module against a real rendered
|
|
37
|
+
* image before shipping it (not assumed): OCR is lossy. A real test against
|
|
38
|
+
* a clean, large, monospace "AKIASM0KETESTFAKEKEY"-shaped string produced
|
|
39
|
+
* "AKIASM@KETESTFAKEKE*" at low resolution and "AKIASMOKETESTFAKEKE*" (0
|
|
40
|
+
* misread as O, trailing Y misread as *) even at 2x resolution -- visually
|
|
41
|
+
* similar characters (0/O, Y/*) are a real, inherent tesseract failure
|
|
42
|
+
* mode, not a bug in how this module invokes it. A single misread character
|
|
43
|
+
* breaks an exact-format regex match. This means --ocr is best-effort
|
|
44
|
+
* additive coverage on a previously-zero-coverage surface, not a guarantee
|
|
45
|
+
* every credential in every screenshot will be caught -- documented here so
|
|
46
|
+
* that claim is never overstated in the CLI help text or README either.
|
|
47
|
+
*/
|
|
48
|
+
|
|
49
|
+
const { spawnSync } = require("child_process");
|
|
50
|
+
|
|
51
|
+
// Test-only escape hatch, same pattern as verify.js's RESIDOO_TEST_AWS_CLI:
|
|
52
|
+
// when set, every spawnSync call below runs that path instead of
|
|
53
|
+
// "tesseract" on PATH, so tests exercise the real spawnSync + stdin/stdout
|
|
54
|
+
// plumbing against a small fixture script rather than requiring the real
|
|
55
|
+
// tesseract binary (or the network) on every machine that runs `npm test`.
|
|
56
|
+
function tesseractBinary() {
|
|
57
|
+
return process.env.RESIDOO_TEST_TESSERACT || "tesseract";
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function isTesseractAvailable(spawnFn = spawnSync) {
|
|
61
|
+
try {
|
|
62
|
+
const r = spawnFn(tesseractBinary(), ["--version"], {
|
|
63
|
+
timeout: 5000,
|
|
64
|
+
env: { PATH: process.env.PATH || "" },
|
|
65
|
+
stdio: ["ignore", "ignore", "ignore"],
|
|
66
|
+
});
|
|
67
|
+
return !r.error && r.status === 0;
|
|
68
|
+
} catch {
|
|
69
|
+
return false;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
// A real screenshot is rarely more than a few MB; this is a generous
|
|
74
|
+
// ceiling against a maliciously or accidentally huge "image" field in an
|
|
75
|
+
// attacker-plantable transcript, not a real-world limit. Base64 is ~4/3
|
|
76
|
+
// the decoded size, hence the larger character-count bound.
|
|
77
|
+
const MAX_BASE64_CHARS = 30_000_000; // ~22 MB decoded
|
|
78
|
+
// A single line with hundreds of embedded images (crafted or corrupted)
|
|
79
|
+
// must not turn --ocr into a hang; cap how many this module will even
|
|
80
|
+
// attempt per line. Real transcripts have at most a handful of images per
|
|
81
|
+
// message.
|
|
82
|
+
const MAX_BLOCKS_PER_LINE = 8;
|
|
83
|
+
const MAX_WALK_DEPTH = 12; // defensive bound against pathological nesting
|
|
84
|
+
|
|
85
|
+
const KNOWN_IMAGE_MEDIA_TYPES = new Set(["image/png", "image/jpeg", "image/webp", "image/gif"]);
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Find every {type:"image", source:{type:"base64", data, media_type}}
|
|
89
|
+
* block in a transcript line, at any nesting depth (a direct message
|
|
90
|
+
* content block and a tool_result's nested content block are both real,
|
|
91
|
+
* confirmed shapes -- see this file's own docstring). Returns
|
|
92
|
+
* [{ data, mediaType }], capped at MAX_BLOCKS_PER_LINE.
|
|
93
|
+
*
|
|
94
|
+
* Not every source's lines are JSON (or valid JSON) -- a malformed or
|
|
95
|
+
* partial line fails JSON.parse and this returns [] rather than throwing,
|
|
96
|
+
* the same fail-quiet-on-this-one-line posture decode.js's contentProjection
|
|
97
|
+
* already has for the exact same reason.
|
|
98
|
+
*/
|
|
99
|
+
function extractImageBlocks(line) {
|
|
100
|
+
const t = typeof line === "string" ? line.trim() : "";
|
|
101
|
+
if (t[0] !== "{" && t[0] !== "[") return [];
|
|
102
|
+
let parsed;
|
|
103
|
+
try { parsed = JSON.parse(t); } catch { return []; }
|
|
104
|
+
|
|
105
|
+
const out = [];
|
|
106
|
+
const walk = (node, depth) => {
|
|
107
|
+
if (out.length >= MAX_BLOCKS_PER_LINE || depth > MAX_WALK_DEPTH || node == null || typeof node !== "object") return;
|
|
108
|
+
if (Array.isArray(node)) {
|
|
109
|
+
for (const item of node) { if (out.length >= MAX_BLOCKS_PER_LINE) return; walk(item, depth + 1); }
|
|
110
|
+
return;
|
|
111
|
+
}
|
|
112
|
+
const source = node.source;
|
|
113
|
+
if (
|
|
114
|
+
node.type === "image" && source && typeof source === "object" &&
|
|
115
|
+
source.type === "base64" && typeof source.data === "string" && source.data.length > 0 &&
|
|
116
|
+
source.data.length <= MAX_BASE64_CHARS &&
|
|
117
|
+
KNOWN_IMAGE_MEDIA_TYPES.has(source.media_type)
|
|
118
|
+
) {
|
|
119
|
+
out.push({ data: source.data, mediaType: source.media_type });
|
|
120
|
+
return; // an image block's own fields are never themselves nested image blocks
|
|
121
|
+
}
|
|
122
|
+
for (const key of Object.keys(node)) { if (out.length >= MAX_BLOCKS_PER_LINE) return; walk(node[key], depth + 1); }
|
|
123
|
+
};
|
|
124
|
+
walk(parsed, 0);
|
|
125
|
+
return out;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
const DEFAULT_TIMEOUT_MS = 20_000;
|
|
129
|
+
|
|
130
|
+
/** Strip control bytes: OCR output flows into the exact same matching/redaction path as any other text, but must never carry a raw control byte into a terminal. */
|
|
131
|
+
function stripControlChars(s) { return String(s || "").replace(/[\x00-\x1f\x7f]/g, ""); }
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* OCR one image's base64 data via tesseract over stdin/stdout. Returns
|
|
135
|
+
* { text, error }: text is "" (never null) on any failure, so a caller
|
|
136
|
+
* never needs a null check before feeding it through the pattern-matching
|
|
137
|
+
* loop; error names why when text is empty, for --ocr's own diagnostics,
|
|
138
|
+
* never surfaced as a scan failure (an unreadable or corrupt image is not
|
|
139
|
+
* a reason to fail the whole scan).
|
|
140
|
+
*/
|
|
141
|
+
function ocrImageBase64(base64Data, { spawnFn = spawnSync, timeoutMs = DEFAULT_TIMEOUT_MS } = {}) {
|
|
142
|
+
let buf;
|
|
143
|
+
try {
|
|
144
|
+
buf = Buffer.from(base64Data, "base64");
|
|
145
|
+
} catch (e) {
|
|
146
|
+
return { text: "", error: `could not decode base64 image data (${e && e.message})` };
|
|
147
|
+
}
|
|
148
|
+
if (buf.length === 0) return { text: "", error: "decoded image was empty" };
|
|
149
|
+
|
|
150
|
+
let r;
|
|
151
|
+
try {
|
|
152
|
+
r = spawnFn(tesseractBinary(), ["stdin", "stdout"], {
|
|
153
|
+
input: buf,
|
|
154
|
+
timeout: timeoutMs,
|
|
155
|
+
maxBuffer: 10 * 1024 * 1024,
|
|
156
|
+
env: { PATH: process.env.PATH || "" },
|
|
157
|
+
});
|
|
158
|
+
} catch (e) {
|
|
159
|
+
return { text: "", error: `tesseract failed to run (${e && e.message})` };
|
|
160
|
+
}
|
|
161
|
+
if (r.error) {
|
|
162
|
+
if (r.error.code === "ENOENT") return { text: "", error: "tesseract not found on PATH" };
|
|
163
|
+
if (r.error.code === "ETIMEDOUT") return { text: "", error: `tesseract timed out after ${timeoutMs}ms` };
|
|
164
|
+
return { text: "", error: `tesseract failed to run (${r.error.code || r.error.message})` };
|
|
165
|
+
}
|
|
166
|
+
if (r.status !== 0) {
|
|
167
|
+
return { text: "", error: `tesseract exited ${r.status}` };
|
|
168
|
+
}
|
|
169
|
+
return { text: stripControlChars((r.stdout || "").toString("utf-8")), error: null };
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
module.exports = { isTesseractAvailable, extractImageBlocks, ocrImageBase64, tesseractBinary };
|
package/src/report.js
CHANGED
|
@@ -466,9 +466,11 @@ function render({ findings, filesScanned, sourcesScanned, bytesScanned, suppress
|
|
|
466
466
|
// so the reader should know the value was hidden.
|
|
467
467
|
const encoded = items.filter((f) => f.encoding).length;
|
|
468
468
|
const split = items.filter((f) => f.spanLines).length;
|
|
469
|
+
const ocrd = items.filter((f) => f.ocr).length;
|
|
469
470
|
const marks = [];
|
|
470
471
|
if (encoded) marks.push(`${encoded} base64-wrapped`);
|
|
471
472
|
if (split) marks.push(`${split} split across lines`);
|
|
473
|
+
if (ocrd) marks.push(`${ocrd} read from a pasted image (--ocr)`);
|
|
472
474
|
const markNote = marks.length ? paint(c.yellow, ` [${marks.join(", ")}]`) : "";
|
|
473
475
|
const paddedLabel = label.length <= labelWidth ? label.padEnd(labelWidth) : label;
|
|
474
476
|
push(` ${paint(color + c.bold, String(items.length).padStart(4))} [${tag}] ${paddedLabel}${distinctNote}${markNote}`);
|
|
@@ -526,17 +528,21 @@ function renderJson(result, integrity = null, rotation = null) {
|
|
|
526
528
|
bytesScanned: result.bytesScanned,
|
|
527
529
|
suppressedCount: result.suppressedCount || 0,
|
|
528
530
|
unreadableFiles: result.unreadableFiles || [],
|
|
531
|
+
ocrRequestedButMissing: result.ocrRequestedButMissing || false,
|
|
529
532
|
},
|
|
530
533
|
findings: result.findings.map((f) => ({
|
|
531
534
|
rule: f.ruleId, label: f.label, confidence: f.confidence,
|
|
532
535
|
source: f.source, file: f.relFile, line: f.line, preview: f.preview,
|
|
533
536
|
fileMTimeMs: f.fileMTimeMs,
|
|
534
|
-
// Markers for the
|
|
537
|
+
// Markers for the decode/reconstruct/OCR passes (absent on ordinary
|
|
535
538
|
// findings). `encoding` names how the value was wrapped ("base64" /
|
|
536
539
|
// "base64url"); `spanLines` names the adjacent line pair a split value
|
|
537
|
-
// was reconstructed across
|
|
540
|
+
// was reconstructed across; `ocr` means the value was never plain
|
|
541
|
+
// text at all -- it was read out of a pasted or tool-returned image
|
|
542
|
+
// (see ocr.js).
|
|
538
543
|
...(f.encoding ? { encoding: f.encoding } : {}),
|
|
539
544
|
...(f.spanLines ? { spanLines: f.spanLines } : {}),
|
|
545
|
+
...(f.ocr ? { ocr: true } : {}),
|
|
540
546
|
fingerprint: fingerprintFinding(f),
|
|
541
547
|
// Only present on an --include-suppressed run: says WHY this finding
|
|
542
548
|
// is low-confidence, so a JSON consumer doesn't have to guess.
|
|
@@ -643,4 +649,202 @@ function renderSarif(result) {
|
|
|
643
649
|
}, null, 2);
|
|
644
650
|
}
|
|
645
651
|
|
|
646
|
-
|
|
652
|
+
function escapeHtml(s) {
|
|
653
|
+
return String(s == null ? "" : s)
|
|
654
|
+
.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">")
|
|
655
|
+
.replace(/"/g, """).replace(/'/g, "'");
|
|
656
|
+
}
|
|
657
|
+
|
|
658
|
+
const HTML_REPORT_STYLE = `
|
|
659
|
+
:root { color-scheme: dark; --bg:#0d1117; --panel:#161b22; --border:#30363d; --text:#c9d1d9;
|
|
660
|
+
--dim:#8b949e; --red:#f85149; --yellow:#d29922; --green:#3fb950; --cyan:#58a6ff; --accent:#238636; }
|
|
661
|
+
* { box-sizing: border-box; }
|
|
662
|
+
body { margin:0; padding:32px; background:var(--bg); color:var(--text);
|
|
663
|
+
font:14px/1.5 -apple-system,BlinkMacSystemFont,"Segoe UI",Helvetica,Arial,sans-serif; }
|
|
664
|
+
.wrap { max-width: 1100px; margin: 0 auto; }
|
|
665
|
+
h1 { font-size:20px; margin:0 0 4px; }
|
|
666
|
+
.meta { color:var(--dim); font-size:12px; margin-bottom:24px; }
|
|
667
|
+
.cards { display:flex; gap:12px; flex-wrap:wrap; margin-bottom:24px; }
|
|
668
|
+
.card { background:var(--panel); border:1px solid var(--border); border-radius:8px;
|
|
669
|
+
padding:14px 18px; min-width:140px; }
|
|
670
|
+
.card .n { font-size:24px; font-weight:700; }
|
|
671
|
+
.card .l { color:var(--dim); font-size:12px; margin-top:2px; }
|
|
672
|
+
.clean { background:var(--panel); border:1px solid var(--accent); border-radius:8px;
|
|
673
|
+
padding:20px; color:var(--green); font-weight:600; }
|
|
674
|
+
input#filter { width:100%; padding:10px 12px; margin-bottom:14px; background:var(--panel);
|
|
675
|
+
border:1px solid var(--border); border-radius:8px; color:var(--text); font-size:14px; }
|
|
676
|
+
input#filter:focus { outline:1px solid var(--cyan); }
|
|
677
|
+
table { width:100%; border-collapse:collapse; background:var(--panel); border:1px solid var(--border);
|
|
678
|
+
border-radius:8px; overflow:hidden; margin-bottom:24px; }
|
|
679
|
+
th, td { text-align:left; padding:10px 12px; border-bottom:1px solid var(--border); font-size:13px; }
|
|
680
|
+
th { color:var(--dim); font-weight:600; font-size:11px; text-transform:uppercase; letter-spacing:.03em; }
|
|
681
|
+
tr:last-child td { border-bottom:none; }
|
|
682
|
+
tr.row:hover { background:#1c232c; cursor:pointer; }
|
|
683
|
+
.conf-high { color:var(--red); font-weight:600; } .conf-medium { color:var(--yellow); font-weight:600; }
|
|
684
|
+
.conf-low { color:var(--dim); }
|
|
685
|
+
.status-pending { color:var(--yellow); } .status-acked { color:var(--green); } .status-dismissed { color:var(--dim); }
|
|
686
|
+
code.preview { font-family:ui-monospace,SFMono-Regular,Menlo,monospace; background:#0000004d;
|
|
687
|
+
padding:1px 6px; border-radius:4px; }
|
|
688
|
+
.guide { display:none; background:#0000004d; padding:12px 16px; }
|
|
689
|
+
.guide.open { display:table-row; }
|
|
690
|
+
.guide td { border-bottom:1px solid var(--border); }
|
|
691
|
+
.guide ol { margin:6px 0; padding-left:20px; }
|
|
692
|
+
.guide a { color:var(--cyan); }
|
|
693
|
+
.note { color:var(--dim); font-size:12px; margin-top:2px; }
|
|
694
|
+
.section-title { font-size:15px; font-weight:600; margin:24px 0 10px; }
|
|
695
|
+
.warn-badge { display:inline-block; background:#f8514922; color:var(--red); border:1px solid var(--red);
|
|
696
|
+
border-radius:4px; padding:1px 8px; font-size:11px; font-weight:600; margin-left:8px; }
|
|
697
|
+
.footer { color:var(--dim); font-size:12px; margin-top:32px; border-top:1px solid var(--border); padding-top:16px; }
|
|
698
|
+
`;
|
|
699
|
+
|
|
700
|
+
const HTML_REPORT_SCRIPT = `
|
|
701
|
+
document.getElementById("filter")?.addEventListener("input", function (e) {
|
|
702
|
+
var q = e.target.value.toLowerCase();
|
|
703
|
+
document.querySelectorAll("tr.row").forEach(function (row) {
|
|
704
|
+
var hit = row.getAttribute("data-search").includes(q);
|
|
705
|
+
row.style.display = hit ? "" : "none";
|
|
706
|
+
var g = row.nextElementSibling;
|
|
707
|
+
if (g && g.classList.contains("guide") && !hit) g.classList.remove("open");
|
|
708
|
+
});
|
|
709
|
+
});
|
|
710
|
+
document.querySelectorAll("tr.row").forEach(function (row) {
|
|
711
|
+
row.addEventListener("click", function () {
|
|
712
|
+
var g = row.nextElementSibling;
|
|
713
|
+
if (g && g.classList.contains("guide")) g.classList.toggle("open");
|
|
714
|
+
});
|
|
715
|
+
});
|
|
716
|
+
`;
|
|
717
|
+
|
|
718
|
+
/**
|
|
719
|
+
* Self-contained, single-file HTML report (residoo scan --html). Same data
|
|
720
|
+
* as renderJson (findings deduped by rotation.js into distinct-value rows,
|
|
721
|
+
* plus integrity), presented for the audience --json/--sarif don't serve
|
|
722
|
+
* well: a screenshot for an incident channel, or a non-CLI teammate.
|
|
723
|
+
*
|
|
724
|
+
* Every value shown is `f.preview`/`entry.preview` — already redacted by
|
|
725
|
+
* patterns.js's redact() before it ever reaches this function, the same
|
|
726
|
+
* guarantee every other output format has. Unlike a competitor's own HTML
|
|
727
|
+
* report (which explicitly notes only its HTML mode masks values, and its
|
|
728
|
+
* JSON mode ships full raw secrets "for incident response"), residoo has
|
|
729
|
+
* no output mode, in any format, that ever writes a raw value — this
|
|
730
|
+
* function has no code path that could regress that, since it never
|
|
731
|
+
* receives the raw value in the first place.
|
|
732
|
+
*
|
|
733
|
+
* No external CSS/JS/fonts/images: everything is inlined below, so the
|
|
734
|
+
* file opens correctly with no network access, matching residoo's own
|
|
735
|
+
* "no network calls in the default path" posture for the report itself,
|
|
736
|
+
* not just the scan that produced it.
|
|
737
|
+
*/
|
|
738
|
+
function renderHtml(result, integrity = null, rotation = null) {
|
|
739
|
+
const { version } = require("../package.json");
|
|
740
|
+
const findings = result.findings || [];
|
|
741
|
+
const scannedAt = localTimestamp(new Date());
|
|
742
|
+
const distinct = rotation && rotation.counts ? rotation.counts.distinct : 0;
|
|
743
|
+
const pending = rotation && rotation.counts ? rotation.counts.pending : 0;
|
|
744
|
+
const confirmedDead = rotation && rotation.counts ? rotation.counts.confirmedDead || 0 : 0;
|
|
745
|
+
const needsReview = Math.max(0, pending - confirmedDead);
|
|
746
|
+
const byFile = new Set(findings.map((f) => f.file)).size;
|
|
747
|
+
const integrityWarns = integrity ? integrity.findings.filter((f) => f.severity === "warn").length : 0;
|
|
748
|
+
|
|
749
|
+
const head =
|
|
750
|
+
`<!doctype html><html><head><meta charset="utf-8">` +
|
|
751
|
+
`<meta name="viewport" content="width=device-width, initial-scale=1">` +
|
|
752
|
+
`<meta name="robots" content="noindex">` +
|
|
753
|
+
`<title>residoo report -- ${escapeHtml(scannedAt)}</title>` +
|
|
754
|
+
`<style>${HTML_REPORT_STYLE}</style></head><body><div class="wrap">` +
|
|
755
|
+
`<h1>residoo report</h1>` +
|
|
756
|
+
`<div class="meta">v${escapeHtml(version)} · scanned ${escapeHtml(scannedAt)} · ` +
|
|
757
|
+
`generated locally, never uploaded — safe to share, values below are redacted</div>`;
|
|
758
|
+
|
|
759
|
+
if (findings.length === 0) {
|
|
760
|
+
const body =
|
|
761
|
+
`<div class="clean">✓ No exposed secrets found: ${filesScannedLine(result)}</div>` +
|
|
762
|
+
(integrity ? renderIntegrityHtml(integrity) : "") +
|
|
763
|
+
`</div></body></html>`;
|
|
764
|
+
return head + body;
|
|
765
|
+
}
|
|
766
|
+
|
|
767
|
+
const cards =
|
|
768
|
+
`<div class="cards">` +
|
|
769
|
+
card(String(findings.length), `finding${findings.length === 1 ? "" : "s"} across ${byFile} file${byFile === 1 ? "" : "s"}`) +
|
|
770
|
+
card(String(distinct), `distinct value${distinct === 1 ? "" : "s"}`) +
|
|
771
|
+
card(String(needsReview), `need${needsReview === 1 ? "s" : ""} review`) +
|
|
772
|
+
card(String(result.filesScanned), "files scanned") +
|
|
773
|
+
`</div>`;
|
|
774
|
+
|
|
775
|
+
const rows = (rotation && rotation.entries ? [...rotation.entries] : [])
|
|
776
|
+
.sort((a, b) => b.occurrences - a.occurrences)
|
|
777
|
+
.map((e) => rotationRowHtml(e))
|
|
778
|
+
.join("");
|
|
779
|
+
|
|
780
|
+
const table =
|
|
781
|
+
`<input id="filter" type="text" placeholder="Filter by rule, file, or preview...">` +
|
|
782
|
+
`<table><thead><tr><th>Rule</th><th>Status</th><th>Preview</th><th>Seen</th><th>Files</th></tr></thead>` +
|
|
783
|
+
`<tbody>${rows}</tbody></table>`;
|
|
784
|
+
|
|
785
|
+
const integritySection = integrity
|
|
786
|
+
? `<div class="section-title">Integrity checks${integrityWarns > 0 ? `<span class="warn-badge">${integrityWarns} warning${integrityWarns === 1 ? "" : "s"}</span>` : ""}</div>` +
|
|
787
|
+
renderIntegrityHtml(integrity)
|
|
788
|
+
: "";
|
|
789
|
+
|
|
790
|
+
const footer =
|
|
791
|
+
`<div class="footer">Values are redacted (first/last 4 characters only). Nothing in this file, or in the scan` +
|
|
792
|
+
` that produced it, ever left this machine. Generated by <code class="preview">residoo scan --html</code> --` +
|
|
793
|
+
` github.com/dandovdub/residoo</div>`;
|
|
794
|
+
|
|
795
|
+
return head + cards + table + integritySection + footer + `<script>${HTML_REPORT_SCRIPT}</script></div></body></html>`;
|
|
796
|
+
}
|
|
797
|
+
|
|
798
|
+
function card(n, label) {
|
|
799
|
+
return `<div class="card"><div class="n">${escapeHtml(n)}</div><div class="l">${escapeHtml(label)}</div></div>`;
|
|
800
|
+
}
|
|
801
|
+
|
|
802
|
+
function filesScannedLine(result) {
|
|
803
|
+
return `${result.filesScanned} file${result.filesScanned === 1 ? "" : "s"} scanned across ${(result.sourcesScanned || []).join(", ") || "no sources"}`;
|
|
804
|
+
}
|
|
805
|
+
|
|
806
|
+
const HTML_STATUS_LABEL = { pending: "pending", acked: "rotated", dismissed: "dismissed" };
|
|
807
|
+
|
|
808
|
+
function rotationRowHtml(e) {
|
|
809
|
+
const search = escapeHtml(`${e.label} ${e.ruleId} ${e.preview} ${(e.files || []).join(" ")}`.toLowerCase());
|
|
810
|
+
const filesShown = (e.files || []).slice(0, 3).map((f) => safeBasename(f));
|
|
811
|
+
const moreFiles = (e.files || []).length - filesShown.length;
|
|
812
|
+
const g = e.guidance || {};
|
|
813
|
+
const link = g.rotateUrl
|
|
814
|
+
? `<a href="${escapeHtml(g.rotateUrl)}" target="_blank" rel="noopener">${escapeHtml(g.rotateUrl)}</a>`
|
|
815
|
+
: escapeHtml(g.consolePath || "");
|
|
816
|
+
const steps = (g.steps || []).map((s) => `<li>${escapeHtml(s)}</li>`).join("");
|
|
817
|
+
return (
|
|
818
|
+
`<tr class="row" data-search="${search}">` +
|
|
819
|
+
`<td>${escapeHtml(e.label)}</td>` +
|
|
820
|
+
`<td class="status-${escapeHtml(e.status)}">${escapeHtml(HTML_STATUS_LABEL[e.status] || e.status)}</td>` +
|
|
821
|
+
`<td><code class="preview">${escapeHtml(e.preview)}</code></td>` +
|
|
822
|
+
`<td>${e.occurrences}×</td>` +
|
|
823
|
+
`<td>${escapeHtml(filesShown.join(", "))}${moreFiles > 0 ? ` +${moreFiles} more` : ""}</td>` +
|
|
824
|
+
`</tr>` +
|
|
825
|
+
`<tr class="guide"><td colspan="5">` +
|
|
826
|
+
`<div><strong>${escapeHtml(g.label || e.label)}</strong></div>` +
|
|
827
|
+
(link ? `<div class="note">${link}</div>` : "") +
|
|
828
|
+
(steps ? `<ol>${steps}</ol>` : "") +
|
|
829
|
+
(g.revokeNote ? `<div class="note">${escapeHtml(g.revokeNote)}</div>` : "") +
|
|
830
|
+
`</td></tr>`
|
|
831
|
+
);
|
|
832
|
+
}
|
|
833
|
+
|
|
834
|
+
function renderIntegrityHtml(integrity) {
|
|
835
|
+
if (!integrity.findings || integrity.findings.length === 0) {
|
|
836
|
+
return `<div class="clean" style="margin-bottom:24px">✓ No integrity findings.</div>`;
|
|
837
|
+
}
|
|
838
|
+
const rows = integrity.findings.map((f) =>
|
|
839
|
+
`<tr><td class="status-${f.severity === "warn" ? "pending" : "dismissed"}">${escapeHtml(f.severity)}</td>` +
|
|
840
|
+
`<td>${escapeHtml(f.kind)}</td><td>${escapeHtml(safeBasename(f.file))}</td>` +
|
|
841
|
+
`<td>${escapeHtml(f.detail || "")}</td></tr>`
|
|
842
|
+
).join("");
|
|
843
|
+
return (
|
|
844
|
+
`<table><thead><tr><th>Severity</th><th>Kind</th><th>File</th><th>Detail</th></tr></thead>` +
|
|
845
|
+
`<tbody>${rows}</tbody></table>` +
|
|
846
|
+
(integrity.scopeNote ? `<div class="note">${escapeHtml(integrity.scopeNote)}</div>` : "")
|
|
847
|
+
);
|
|
848
|
+
}
|
|
849
|
+
|
|
850
|
+
module.exports = { render, renderIntegrity, renderRotationSection, renderJson, renderSarif, renderHtml, makeProgressReporter, printIntro };
|
package/src/scan.js
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
const path = require("path");
|
|
4
4
|
const { PATTERNS, NOISY_PATTERNS, redact } = require("./patterns");
|
|
5
5
|
const { findDecodedMatches, findBoundaryMatches, contentProjection } = require("./decode");
|
|
6
|
+
const { isTesseractAvailable, extractImageBlocks, ocrImageBase64 } = require("./ocr");
|
|
6
7
|
const { findPairedSecret, findNearbyCandidate } = require("./pairing");
|
|
7
8
|
const { looksRandom } = require("./rarity");
|
|
8
9
|
const { decodeJwtExpiryMs } = require("./jwtExpiry");
|
|
@@ -266,8 +267,15 @@ function localTimestamp(d) {
|
|
|
266
267
|
* absolute path can itself carry a username or a project name the rest of
|
|
267
268
|
* this report is careful never to print.
|
|
268
269
|
*/
|
|
269
|
-
async function scan({ sources, includeNoisy = false, includeSuppressed = false, onProgress = null, verify = false, verifyOnlyFingerprint = null, onBeforeVerify = null, noColor = false } = {}) {
|
|
270
|
+
async function scan({ sources, includeNoisy = false, includeSuppressed = false, onProgress = null, verify = false, verifyOnlyFingerprint = null, onBeforeVerify = null, noColor = false, ocr = false } = {}) {
|
|
270
271
|
const rules = includeNoisy ? PATTERNS.concat(NOISY_PATTERNS) : PATTERNS;
|
|
272
|
+
// --ocr: checked once, not per line/image -- isTesseractAvailable shells
|
|
273
|
+
// out, and this scan can touch thousands of lines. ocrRequestedButMissing
|
|
274
|
+
// flows back to the caller (see the return value below) so a user who
|
|
275
|
+
// asked for --ocr without tesseract installed gets a clear, once-per-scan
|
|
276
|
+
// message, not silence and zero image findings.
|
|
277
|
+
const ocrReady = ocr && isTesseractAvailable();
|
|
278
|
+
const ocrRequestedButMissing = ocr && !ocrReady;
|
|
271
279
|
// The decode pass (see decode.js) only applies high-confidence, vendor-
|
|
272
280
|
// prefixed rules to decoded bytes: random binary that decodes to printable
|
|
273
281
|
// text can shape-match a generic rule, but not a vendor prefix. NOISY rules
|
|
@@ -559,6 +567,34 @@ async function scan({ sources, includeNoisy = false, includeSuppressed = false,
|
|
|
559
567
|
}
|
|
560
568
|
};
|
|
561
569
|
|
|
570
|
+
// --ocr only: a line whose JSON shape holds a pasted or tool-returned
|
|
571
|
+
// image (see ocr.js's docstring for the exact confirmed shape) gets each
|
|
572
|
+
// image block decoded and OCR'd, and the extracted text runs through the
|
|
573
|
+
// same high-confidence rules the decode pass above uses, for the same
|
|
574
|
+
// reason -- a step removed from literal transcript text deserves the
|
|
575
|
+
// higher bar. Every finding carries an `ocr: true` marker so a report can
|
|
576
|
+
// say where the value actually came from.
|
|
577
|
+
const ocrLine = (line, file, relFile, lineNo, mtimeMs) => {
|
|
578
|
+
if (!ocrReady) return;
|
|
579
|
+
for (const block of extractImageBlocks(line)) {
|
|
580
|
+
const { text } = ocrImageBase64(block.data);
|
|
581
|
+
if (!text) continue;
|
|
582
|
+
for (const rule of highRules) {
|
|
583
|
+
rule.re.lastIndex = 0;
|
|
584
|
+
let m;
|
|
585
|
+
while ((m = rule.re.exec(text)) !== null) {
|
|
586
|
+
const suppressedReason = suppressionReason(m[0], null, rule.id);
|
|
587
|
+
if (suppressedReason && !includeSuppressed) {
|
|
588
|
+
suppressedCount++;
|
|
589
|
+
continue;
|
|
590
|
+
}
|
|
591
|
+
record(rule, m[0], relFile, file, lineNo, mtimeMs,
|
|
592
|
+
suppressedReason ? "low" : rule.confidence, suppressedReason, { ocr: true });
|
|
593
|
+
}
|
|
594
|
+
}
|
|
595
|
+
}
|
|
596
|
+
};
|
|
597
|
+
|
|
562
598
|
// Feature 2: split-line boundary join. A finding here means one credential
|
|
563
599
|
// was split across this line and the next and is contiguous on neither. It
|
|
564
600
|
// is recorded against BOTH contributing lines (each holds a fragment of the
|
|
@@ -669,6 +705,13 @@ async function scan({ sources, includeNoisy = false, includeSuppressed = false,
|
|
|
669
705
|
} catch (err) {
|
|
670
706
|
flagFailed();
|
|
671
707
|
}
|
|
708
|
+
if (ocrReady) {
|
|
709
|
+
try {
|
|
710
|
+
ocrLine(line, file, relFile, i + 1, mtimeMs);
|
|
711
|
+
} catch (err) {
|
|
712
|
+
flagFailed();
|
|
713
|
+
}
|
|
714
|
+
}
|
|
672
715
|
try {
|
|
673
716
|
const content = contentProjection(line);
|
|
674
717
|
// Boundary join with the previous line (2-way splits only; see
|
|
@@ -872,7 +915,7 @@ async function scan({ sources, includeNoisy = false, includeSuppressed = false,
|
|
|
872
915
|
|
|
873
916
|
const distinctCounts = {};
|
|
874
917
|
for (const [ruleId, set] of distinctByRule) distinctCounts[ruleId] = set.size;
|
|
875
|
-
return { findings, filesScanned, sourcesScanned, bytesScanned, suppressedCount, distinctCounts, unreadableFiles };
|
|
918
|
+
return { findings, filesScanned, sourcesScanned, bytesScanned, suppressedCount, distinctCounts, unreadableFiles, ocrRequestedButMissing };
|
|
876
919
|
}
|
|
877
920
|
|
|
878
921
|
/**
|
|
@@ -884,7 +927,7 @@ async function scan({ sources, includeNoisy = false, includeSuppressed = false,
|
|
|
884
927
|
function emptyResult() {
|
|
885
928
|
return {
|
|
886
929
|
findings: [], filesScanned: 0, sourcesScanned: [], bytesScanned: 0,
|
|
887
|
-
suppressedCount: 0, distinctCounts: {}, unreadableFiles: [],
|
|
930
|
+
suppressedCount: 0, distinctCounts: {}, unreadableFiles: [], ocrRequestedButMissing: false,
|
|
888
931
|
};
|
|
889
932
|
}
|
|
890
933
|
|