shipclear 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +104 -0
- package/adapters/claude-code/.claude-plugin/plugin.json +7 -0
- package/adapters/claude-code/commands/safe-to-ship.md +37 -0
- package/adapters/claude-code/hooks/guard.cjs +49 -0
- package/adapters/claude-code/hooks/hooks.json +15 -0
- package/adapters/claude-code/skills/shipclear-prevention/SKILL.md +40 -0
- package/adapters/codex/AGENTS.md +40 -0
- package/adapters/cursor/shipclear.mdc +40 -0
- package/adapters/gemini/GEMINI.md +40 -0
- package/adapters/generic/INSTRUCTIONS.md +66 -0
- package/cli/bin/shipclear.js +182 -0
- package/cli/lib/checks.js +414 -0
- package/cli/lib/fix.js +160 -0
- package/cli/lib/git.js +112 -0
- package/cli/lib/ignore.js +33 -0
- package/cli/lib/patterns.js +215 -0
- package/cli/lib/report.js +316 -0
- package/cli/lib/scan.js +51 -0
- package/core/gate-checklist.md +44 -0
- package/core/prevention.md +33 -0
- package/docs/quickstart.md +87 -0
- package/docs/troubleshooting.md +75 -0
- package/package.json +45 -0
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import fs from 'node:fs';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
import { fileURLToPath } from 'node:url';
|
|
5
|
+
import { runGate } from '../lib/checks.js';
|
|
6
|
+
import { applyFixes, resetHistory, installHook, envExampleFrom, GITIGNORE_BLOCK } from '../lib/fix.js';
|
|
7
|
+
import { renderTerminal, renderMarkdown, verdictOf, VERDICT_TEXT } from '../lib/report.js';
|
|
8
|
+
import { findSecrets, maskSecret, lineOfIndex } from '../lib/patterns.js';
|
|
9
|
+
import { isGitRepo, stagedFiles, stagedContent } from '../lib/git.js';
|
|
10
|
+
import { loadIgnore } from '../lib/ignore.js';
|
|
11
|
+
|
|
12
|
+
const pkg = JSON.parse(
|
|
13
|
+
fs.readFileSync(path.join(path.dirname(fileURLToPath(import.meta.url)), '../../package.json'), 'utf8')
|
|
14
|
+
);
|
|
15
|
+
|
|
16
|
+
const HELP = `
|
|
17
|
+
ShipClear v${pkg.version} — the last command before you launch.
|
|
18
|
+
|
|
19
|
+
Usage:
|
|
20
|
+
shipclear setup Prepare this project: .env files, .gitignore, commit guard
|
|
21
|
+
shipclear scan Quick scan for secrets in this project
|
|
22
|
+
shipclear scan --staged Scan only what's about to be committed (used by the guard)
|
|
23
|
+
shipclear ship Full safe-to-ship gate → SHIP-REPORT.md + verdict
|
|
24
|
+
shipclear ship --no-fix Same, but don't apply automatic fixes
|
|
25
|
+
shipclear ship --fix-history
|
|
26
|
+
Also erase git history to remove a leaked secret from
|
|
27
|
+
old commits (only when nothing was ever pushed; refuses
|
|
28
|
+
if it would lose branches or stashes)
|
|
29
|
+
|
|
30
|
+
Everything runs locally. Nothing is uploaded, ever.
|
|
31
|
+
Docs: https://github.com/adamtimmerberg/ShipClear
|
|
32
|
+
`;
|
|
33
|
+
|
|
34
|
+
function cmdSetup(root) {
|
|
35
|
+
const done = [];
|
|
36
|
+
const warnings = [];
|
|
37
|
+
const gitignorePath = path.join(root, '.gitignore');
|
|
38
|
+
const gitignore = fs.existsSync(gitignorePath) ? fs.readFileSync(gitignorePath, 'utf8') : '';
|
|
39
|
+
if (!gitignore.split('\n').some((l) => /^(\.env(\.\*)?|\*\.env|\.env\*)\s*$/.test(l.trim()))) {
|
|
40
|
+
fs.writeFileSync(gitignorePath, gitignore + GITIGNORE_BLOCK);
|
|
41
|
+
done.push('.gitignore now excludes .env files and the ShipClear report — this protects future commits; if a secret was already committed earlier, `shipclear ship` will find it');
|
|
42
|
+
}
|
|
43
|
+
const envPath = path.join(root, '.env');
|
|
44
|
+
if (!fs.existsSync(envPath)) {
|
|
45
|
+
fs.writeFileSync(envPath, '# Real values live here. This file is git-ignored — never commit it.\n');
|
|
46
|
+
done.push('created .env (for real values — stays on your machine)');
|
|
47
|
+
}
|
|
48
|
+
if (!fs.existsSync(path.join(root, '.env.example'))) {
|
|
49
|
+
// Same derivation the auto-fix uses: variable names from .env, values stripped.
|
|
50
|
+
const example = envExampleFrom(envPath);
|
|
51
|
+
fs.writeFileSync(path.join(root, '.env.example'), example);
|
|
52
|
+
done.push(example.includes('=')
|
|
53
|
+
? 'created .env.example (your variable names, no values — safe to share)'
|
|
54
|
+
: 'created .env.example (a safe-to-share template — add a line per setting as you add them to .env)');
|
|
55
|
+
}
|
|
56
|
+
const inGit = isGitRepo(root);
|
|
57
|
+
if (inGit) {
|
|
58
|
+
if (installHook(root)) {
|
|
59
|
+
done.push('installed the commit guard (secrets can no longer be committed)');
|
|
60
|
+
} else {
|
|
61
|
+
warnings.push('this project already has a pre-commit hook, so ShipClear left it alone — your commits are NOT yet protected. Add `shipclear scan --staged` as a line in that existing hook to turn protection on.');
|
|
62
|
+
}
|
|
63
|
+
} else {
|
|
64
|
+
warnings.push("this folder isn't set up with git yet, so ShipClear couldn't install the commit guard — the safety net that stops a secret from ever being committed. Run `git init` in this folder, then run `shipclear setup` again.");
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
console.log(warnings.length ? '\n ShipClear setup finished — but read this first:' : '\n ShipClear setup complete.');
|
|
68
|
+
for (const d of done) console.log(` ✔ ${d}`);
|
|
69
|
+
if (!done.length && !warnings.length) console.log(' ✔ everything was already in place');
|
|
70
|
+
for (const w of warnings) console.log(` ⚠ ${w}`);
|
|
71
|
+
// The files above sit uncommitted otherwise, and "commit" is not an
|
|
72
|
+
// instruction to someone who has never run git — give the command.
|
|
73
|
+
if (inGit && done.length) console.log('\n These changes aren\'t saved into git yet — run: git add -A && git commit -m "add ShipClear protection"');
|
|
74
|
+
console.log('\n Before you launch or go public: shipclear ship\n');
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function cmdScanStaged(root, quiet) {
|
|
78
|
+
if (!isGitRepo(root)) {
|
|
79
|
+
console.error('Not a git repository.');
|
|
80
|
+
return 0;
|
|
81
|
+
}
|
|
82
|
+
const ignored = loadIgnore(root);
|
|
83
|
+
const blockers = [];
|
|
84
|
+
for (const rel of stagedFiles(root)) {
|
|
85
|
+
if (ignored(rel)) continue;
|
|
86
|
+
if (/^\.env(\..+)?$/.test(path.posix.basename(rel)) && !/\.(example|sample|template)$/.test(rel)) {
|
|
87
|
+
blockers.push({ rel, reason: 'this is a .env secrets file — it must never be committed' });
|
|
88
|
+
continue;
|
|
89
|
+
}
|
|
90
|
+
const content = stagedContent(root, rel);
|
|
91
|
+
if (content === null) continue;
|
|
92
|
+
for (const hit of findSecrets(content, { includeGeneric: false })) {
|
|
93
|
+
blockers.push({
|
|
94
|
+
rel: `${rel}:${lineOfIndex(content, hit.index)}`,
|
|
95
|
+
reason: `${hit.name} (\`${maskSecret(hit.match)}\`)`,
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
if (blockers.length === 0) {
|
|
100
|
+
if (!quiet) console.log('ShipClear: staged changes are clean.');
|
|
101
|
+
return 0;
|
|
102
|
+
}
|
|
103
|
+
console.error('\n🛑 ShipClear blocked this commit — it contains secrets:\n');
|
|
104
|
+
for (const b of blockers) console.error(` • ${b.rel} — ${b.reason}`);
|
|
105
|
+
console.error('\nMove secrets to .env, restage, and commit again.');
|
|
106
|
+
console.error('(If this is a confirmed false positive, add the path to .shipclearignore.)\n');
|
|
107
|
+
return 1;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
function cmdScan(root) {
|
|
111
|
+
const { findings, notes } = runGate(root, { quick: true });
|
|
112
|
+
const verdict = verdictOf(findings);
|
|
113
|
+
console.log(renderTerminal({ findings, notes, verdict, version: pkg.version, quick: true }));
|
|
114
|
+
return findings.some((f) => f.severity === 'critical') ? 1 : 0;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
function cmdShip(root, { fix = true, fixHistory = false } = {}) {
|
|
118
|
+
let { findings, notes } = runGate(root);
|
|
119
|
+
const fixes = fix ? applyFixes(root, findings) : [];
|
|
120
|
+
|
|
121
|
+
if (fixHistory) {
|
|
122
|
+
const { done, refused } = resetHistory(root, findings);
|
|
123
|
+
if (done) {
|
|
124
|
+
fixes.push(done);
|
|
125
|
+
// History is gone — re-run the gate so the verdict reflects the
|
|
126
|
+
// repo as it now is, not as it was before the reset.
|
|
127
|
+
({ findings, notes } = runGate(root));
|
|
128
|
+
} else {
|
|
129
|
+
notes.push(`--fix-history was not applied: ${refused}`);
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
const verdict = verdictOf(findings);
|
|
134
|
+
const result = { findings, notes, verdict, fixes, version: pkg.version };
|
|
135
|
+
fs.writeFileSync(path.join(root, 'SHIP-REPORT.md'), renderMarkdown(result));
|
|
136
|
+
console.log(renderTerminal(result));
|
|
137
|
+
return verdict === 'DO_NOT_SHIP' ? 1 : 0;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
const args = process.argv.slice(2);
|
|
141
|
+
const cmd = args[0];
|
|
142
|
+
const root = process.cwd();
|
|
143
|
+
|
|
144
|
+
let exitCode = 0;
|
|
145
|
+
try {
|
|
146
|
+
switch (cmd) {
|
|
147
|
+
case 'setup':
|
|
148
|
+
cmdSetup(root);
|
|
149
|
+
break;
|
|
150
|
+
case 'scan':
|
|
151
|
+
exitCode = args.includes('--staged')
|
|
152
|
+
? cmdScanStaged(root, args.includes('--quiet'))
|
|
153
|
+
: cmdScan(root);
|
|
154
|
+
break;
|
|
155
|
+
case 'ship': {
|
|
156
|
+
const fixHistory = args.includes('--fix-history');
|
|
157
|
+
// A history reset needs .gitignore in place first so the fresh commit
|
|
158
|
+
// doesn't re-add .env — so --fix-history implies the additive fixes.
|
|
159
|
+
exitCode = cmdShip(root, { fix: !args.includes('--no-fix') || fixHistory, fixHistory });
|
|
160
|
+
break;
|
|
161
|
+
}
|
|
162
|
+
case '--version':
|
|
163
|
+
case '-v':
|
|
164
|
+
console.log(pkg.version);
|
|
165
|
+
break;
|
|
166
|
+
default:
|
|
167
|
+
console.log(HELP);
|
|
168
|
+
if (cmd && cmd !== 'help' && cmd !== '--help' && cmd !== '-h') {
|
|
169
|
+
console.error(`Unknown command: ${cmd}`);
|
|
170
|
+
exitCode = 1;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
} catch (err) {
|
|
174
|
+
// A raw stack trace here is exactly the wrong first impression for a
|
|
175
|
+
// tool whose whole promise is "no security background needed" — this is
|
|
176
|
+
// the difference between an unreadable crash (permission errors on
|
|
177
|
+
// sandboxed/cloud IDEs are common) and a plain-English explanation.
|
|
178
|
+
console.error(`\n ShipClear hit an unexpected problem: ${err.message}`);
|
|
179
|
+
console.error(' If this keeps happening, please report it: https://github.com/adamtimmerberg/ShipClear/issues\n');
|
|
180
|
+
exitCode = 1;
|
|
181
|
+
}
|
|
182
|
+
process.exit(exitCode);
|
|
@@ -0,0 +1,414 @@
|
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
import os from 'node:os';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
import { execFileSync } from 'node:child_process';
|
|
5
|
+
import {
|
|
6
|
+
findSecrets, looksPlaceholder, entropy, maskSecret, lineOfIndex,
|
|
7
|
+
realLookingEmails, SSN, PASSWORD_ASSIGNMENT, SUSPICIOUS_ACCOUNT_EMAIL,
|
|
8
|
+
personalPaths, TLS_VERIFICATION_OFF, DEBUG_MODE_ON,
|
|
9
|
+
} from './patterns.js';
|
|
10
|
+
import { loadIgnore } from './ignore.js';
|
|
11
|
+
import { walkFiles, isTextCandidate, readFileSafe } from './scan.js';
|
|
12
|
+
import { isGitRepo, hasCommits, hasRemote, branchNames, trackedFiles, historyAddedLines } from './git.js';
|
|
13
|
+
|
|
14
|
+
const ENV_FILE = /^\.env(?:\..+)?$/;
|
|
15
|
+
// Covers .env.example but also .env.local.example, .env.production.sample,
|
|
16
|
+
// etc. — real starters (nextjs-subscription-payments) use the nested forms.
|
|
17
|
+
const ENV_TEMPLATE = /^\.env(?:\..+)?\.(?:example|sample|template)$|^\.env\.(?:example|sample|template)$/;
|
|
18
|
+
const KEY_FILE = /(\.pem|\.p12|\.pfx|\.ppk)$|(^|\/)id_(?:rsa|ed25519|ecdsa|dsa)(\.pub)?$/;
|
|
19
|
+
const DATA_FILE_EXT = new Set(['.csv', '.tsv', '.sql', '.jsonl', '.ndjson']);
|
|
20
|
+
// .mdc: Cursor rules files — prose instructions, not code.
|
|
21
|
+
const DOC_EXT = new Set(['.md', '.mdx', '.mdc', '.rst', '.txt']);
|
|
22
|
+
// .claude/skills, commands, agents, and settings.json are intentionally
|
|
23
|
+
// shareable project config — only the local/private pieces are artifacts.
|
|
24
|
+
const AGENT_ARTIFACTS = ['.claude/settings.local.json', '.codex/', '.aider', '.specstory/'];
|
|
25
|
+
|
|
26
|
+
const isEnvFile = (rel) => ENV_FILE.test(path.posix.basename(rel)) && !isEnvTemplate(rel);
|
|
27
|
+
const isEnvTemplate = (rel) => ENV_TEMPLATE.test(path.posix.basename(rel));
|
|
28
|
+
const isKeyFile = (rel) => KEY_FILE.test(rel);
|
|
29
|
+
const isDataFile = (rel) => DATA_FILE_EXT.has(path.extname(rel).toLowerCase());
|
|
30
|
+
const isDocFile = (rel) => DOC_EXT.has(path.extname(rel).toLowerCase());
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Run the deterministic layer of the gate. Returns { findings, notes, inGit }.
|
|
34
|
+
* Findings: { id, severity, file?, line?, detail?, commit? }.
|
|
35
|
+
* Severity: critical | high | medium | low | info.
|
|
36
|
+
*/
|
|
37
|
+
export function runGate(root, { quick = false } = {}) {
|
|
38
|
+
const findings = [];
|
|
39
|
+
const notes = [];
|
|
40
|
+
const ignored = loadIgnore(root);
|
|
41
|
+
const inGit = isGitRepo(root);
|
|
42
|
+
|
|
43
|
+
const allFiles = (inGit ? trackedFiles(root) : walkFiles(root)).filter((f) => !ignored(f));
|
|
44
|
+
if (!inGit) notes.push('Not a git repository — history and tracked-file checks were skipped.');
|
|
45
|
+
|
|
46
|
+
const read = (rel) => readFileSafe(path.join(root, rel));
|
|
47
|
+
|
|
48
|
+
// Raw values already reported from the working tree — used to suppress
|
|
49
|
+
// duplicate history findings for secrets that are still present (the
|
|
50
|
+
// in-code finding's fix advice already covers rotation).
|
|
51
|
+
const seenSecretValues = new Set();
|
|
52
|
+
|
|
53
|
+
// --- secrets in code (skipping .env files, which are *supposed* to hold them) ---
|
|
54
|
+
for (const rel of allFiles) {
|
|
55
|
+
if (!isTextCandidate(rel) || isEnvFile(rel) || isEnvTemplate(rel) || isKeyFile(rel)) continue;
|
|
56
|
+
const content = read(rel);
|
|
57
|
+
if (content === null) continue;
|
|
58
|
+
|
|
59
|
+
for (const hit of findSecrets(content, { includeGeneric: !isDocFile(rel) })) {
|
|
60
|
+
seenSecretValues.add(hit.match);
|
|
61
|
+
findings.push({
|
|
62
|
+
id: hit.patternId === 'generic-secret' ? 'generic-secret' : 'secret-in-code',
|
|
63
|
+
severity: hit.severity,
|
|
64
|
+
file: rel,
|
|
65
|
+
line: lineOfIndex(content, hit.index),
|
|
66
|
+
detail: `${hit.name}: \`${maskSecret(hit.value ?? hit.match)}\``,
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
if (!quick && !isDocFile(rel)) {
|
|
71
|
+
// Hardcoded account pairs: a suspicious email literal with a password
|
|
72
|
+
// assignment within three lines of it.
|
|
73
|
+
const lines = content.split('\n');
|
|
74
|
+
lines.forEach((lineText, i) => {
|
|
75
|
+
if (!SUSPICIOUS_ACCOUNT_EMAIL.test(lineText)) return;
|
|
76
|
+
const nearby = lines.slice(Math.max(0, i - 3), i + 4).join('\n');
|
|
77
|
+
if (PASSWORD_ASSIGNMENT.test(nearby)) {
|
|
78
|
+
findings.push({
|
|
79
|
+
id: 'test-account',
|
|
80
|
+
severity: 'high',
|
|
81
|
+
file: rel,
|
|
82
|
+
line: i + 1,
|
|
83
|
+
detail: 'An email + password pair is written directly in the code.',
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
TLS_VERIFICATION_OFF.lastIndex = 0;
|
|
89
|
+
let tls;
|
|
90
|
+
while ((tls = TLS_VERIFICATION_OFF.exec(content))) {
|
|
91
|
+
findings.push({
|
|
92
|
+
id: 'tls-verification-off',
|
|
93
|
+
severity: 'medium',
|
|
94
|
+
file: rel,
|
|
95
|
+
line: lineOfIndex(content, tls.index),
|
|
96
|
+
detail: `\`${tls[0].trim()}\``,
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
DEBUG_MODE_ON.lastIndex = 0;
|
|
101
|
+
let dbg;
|
|
102
|
+
while ((dbg = DEBUG_MODE_ON.exec(content))) {
|
|
103
|
+
findings.push({
|
|
104
|
+
id: 'debug-mode-on',
|
|
105
|
+
severity: 'medium',
|
|
106
|
+
file: rel,
|
|
107
|
+
line: lineOfIndex(content, dbg.index),
|
|
108
|
+
detail: `\`${dbg[0].trim()}\``,
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
for (const hit of personalPaths(content)) {
|
|
113
|
+
findings.push({
|
|
114
|
+
id: 'personal-path',
|
|
115
|
+
severity: 'low',
|
|
116
|
+
file: rel,
|
|
117
|
+
line: lineOfIndex(content, hit.index),
|
|
118
|
+
detail: `\`${hit.match}\``,
|
|
119
|
+
});
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// A generic-secret on the same line as a test-account is the same problem
|
|
125
|
+
// told twice — keep the clearer explanation.
|
|
126
|
+
const accountLines = new Set(
|
|
127
|
+
findings.filter((f) => f.id === 'test-account').map((f) => `${f.file}:${f.line}`)
|
|
128
|
+
);
|
|
129
|
+
const deduped = findings.filter(
|
|
130
|
+
(f) => !(f.id === 'generic-secret' && accountLines.has(`${f.file}:${f.line}`))
|
|
131
|
+
);
|
|
132
|
+
findings.length = 0;
|
|
133
|
+
findings.push(...deduped);
|
|
134
|
+
|
|
135
|
+
// --- tracked files that should never be tracked ---
|
|
136
|
+
if (inGit) {
|
|
137
|
+
for (const rel of allFiles) {
|
|
138
|
+
if (isEnvFile(rel)) {
|
|
139
|
+
findings.push({ id: 'env-tracked', severity: 'critical', file: rel });
|
|
140
|
+
} else if (isKeyFile(rel)) {
|
|
141
|
+
findings.push({ id: 'key-file-tracked', severity: 'critical', file: rel });
|
|
142
|
+
} else if (rel.startsWith('.shipclear/')) {
|
|
143
|
+
// The prevention rules tell the AI to keep generated test credentials
|
|
144
|
+
// here, git-ignored. If it's tracked, that ignore never happened.
|
|
145
|
+
findings.push({
|
|
146
|
+
id: 'test-account',
|
|
147
|
+
severity: 'high',
|
|
148
|
+
file: rel,
|
|
149
|
+
detail: 'This is where ShipClear\'s rules tell your AI to keep test logins — it must never be committed.',
|
|
150
|
+
});
|
|
151
|
+
} else {
|
|
152
|
+
const artifact = AGENT_ARTIFACTS.find((p) => rel.startsWith(p));
|
|
153
|
+
if (artifact) findings.push({ id: 'agent-artifact', severity: 'low', file: rel });
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
// --- .gitignore hygiene ---
|
|
159
|
+
const gitignorePath = path.join(root, '.gitignore');
|
|
160
|
+
const gitignore = fs.existsSync(gitignorePath) ? fs.readFileSync(gitignorePath, 'utf8') : null;
|
|
161
|
+
const coversEnv =
|
|
162
|
+
gitignore && gitignore.split('\n').some((l) => /^(\.env(\.\*)?|\*\.env|\.env\*)\s*$/.test(l.trim()));
|
|
163
|
+
if (!coversEnv) {
|
|
164
|
+
findings.push({
|
|
165
|
+
id: 'gitignore-incomplete',
|
|
166
|
+
severity: 'high',
|
|
167
|
+
file: '.gitignore',
|
|
168
|
+
detail: gitignore === null ? 'No .gitignore file exists.' : '`.env` is not ignored.',
|
|
169
|
+
});
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
if (quick) return { findings, notes, inGit };
|
|
173
|
+
|
|
174
|
+
// --- .env.example should contain shapes, not values ---
|
|
175
|
+
for (const rel of allFiles.filter(isEnvTemplate)) {
|
|
176
|
+
const content = read(rel);
|
|
177
|
+
if (content === null) continue;
|
|
178
|
+
content.split('\n').forEach((lineText, i) => {
|
|
179
|
+
const m = lineText.match(/^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*["']?([^"'\n#]*)/);
|
|
180
|
+
if (!m) return;
|
|
181
|
+
const value = m[2].trim();
|
|
182
|
+
if (!value || looksPlaceholder(value)) return;
|
|
183
|
+
const specific = findSecrets(value, { includeGeneric: false }).length > 0;
|
|
184
|
+
// Values the specific patterns already cleared don't get a second
|
|
185
|
+
// chance via entropy: scheme:// URLs (a connection string with a real
|
|
186
|
+
// password would have matched; without one it's a template) and JWTs
|
|
187
|
+
// (only service-role JWTs are secrets — anon keys are public by design).
|
|
188
|
+
if (!specific && /^[a-z][a-z0-9+.-]*:\/\//i.test(value)) return;
|
|
189
|
+
if (!specific && /^eyJ[A-Za-z0-9_-]+\./.test(value)) return;
|
|
190
|
+
if (specific || (value.length >= 12 && entropy(value) > 3.5)) {
|
|
191
|
+
seenSecretValues.add(value);
|
|
192
|
+
findings.push({
|
|
193
|
+
id: 'env-example-leak',
|
|
194
|
+
severity: 'high',
|
|
195
|
+
file: rel,
|
|
196
|
+
line: i + 1,
|
|
197
|
+
detail: `\`${m[1]}\` has a real-looking value (\`${maskSecret(value)}\`).`,
|
|
198
|
+
});
|
|
199
|
+
}
|
|
200
|
+
});
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
// --- real-looking personal data in tracked data files ---
|
|
204
|
+
for (const rel of allFiles.filter(isDataFile)) {
|
|
205
|
+
const content = read(rel);
|
|
206
|
+
if (content === null) continue;
|
|
207
|
+
const emails = realLookingEmails(content);
|
|
208
|
+
SSN.lastIndex = 0;
|
|
209
|
+
const ssns = content.match(SSN) || [];
|
|
210
|
+
if (emails.length >= 3 || ssns.length > 0) {
|
|
211
|
+
const parts = [];
|
|
212
|
+
if (emails.length >= 3) parts.push(`${emails.length} distinct email addresses`);
|
|
213
|
+
if (ssns.length > 0) parts.push(`${ssns.length} SSN-shaped values`);
|
|
214
|
+
findings.push({ id: 'pii-data-file', severity: 'high', file: rel, detail: parts.join(' and ') });
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
// --- secrets anywhere in git history, even if since deleted ---
|
|
219
|
+
if (inGit && hasCommits(root)) {
|
|
220
|
+
const remoteExists = hasRemote(root);
|
|
221
|
+
const currentBranch = branchNames(root)[0] || 'main';
|
|
222
|
+
const seen = new Set();
|
|
223
|
+
// One finding per file+commit, not per secret: two keys leaked in the
|
|
224
|
+
// same .env in the same commit are one problem with one fix, and two
|
|
225
|
+
// identical ten-line paragraphs read as "do this twice" to a beginner.
|
|
226
|
+
const byLocation = new Map();
|
|
227
|
+
for (const { commit, file, text } of historyAddedLines(root)) {
|
|
228
|
+
if (ignored(file)) continue;
|
|
229
|
+
for (const hit of findSecrets(text, { includeGeneric: false })) {
|
|
230
|
+
if (seen.has(hit.match)) continue;
|
|
231
|
+
seen.add(hit.match);
|
|
232
|
+
// Already reported from the working tree — that finding's advice
|
|
233
|
+
// covers rotation, so a history duplicate is just noise. History
|
|
234
|
+
// findings are for secrets that were *deleted* but still leak.
|
|
235
|
+
if (seenSecretValues.has(hit.match)) continue;
|
|
236
|
+
const key = `${file}@${commit}`;
|
|
237
|
+
const label = `${hit.name} (\`${maskSecret(hit.match)}\`)`;
|
|
238
|
+
const existing = byLocation.get(key);
|
|
239
|
+
if (existing) {
|
|
240
|
+
existing.names.push(label);
|
|
241
|
+
existing.detail = `${existing.names.join(' and ')} first added in commit ${commit}.`;
|
|
242
|
+
continue;
|
|
243
|
+
}
|
|
244
|
+
const finding = {
|
|
245
|
+
id: 'secret-in-history',
|
|
246
|
+
severity: 'critical',
|
|
247
|
+
file,
|
|
248
|
+
commit,
|
|
249
|
+
hasRemote: remoteExists,
|
|
250
|
+
branch: currentBranch,
|
|
251
|
+
names: [label],
|
|
252
|
+
detail: `${label} first added in commit ${commit}.`,
|
|
253
|
+
};
|
|
254
|
+
byLocation.set(key, finding);
|
|
255
|
+
findings.push(finding);
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
// --- generated test logins exist; the rules promise a launch-time reminder ---
|
|
261
|
+
if (fs.existsSync(path.join(root, '.shipclear', 'test-accounts.json'))) {
|
|
262
|
+
findings.push({ id: 'test-registry-present', severity: 'info', file: '.shipclear/test-accounts.json' });
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
// --- .env exists but no template for teammates ---
|
|
266
|
+
if (fs.existsSync(path.join(root, '.env')) && !allFilesHasTemplate(root, allFiles)) {
|
|
267
|
+
findings.push({ id: 'env-example-missing', severity: 'info', file: '.env.example' });
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
// --- optional deeper scanners, used automatically when installed ---
|
|
271
|
+
if (hasBinary('gitleaks')) runGitleaks(root, findings, notes, ignored);
|
|
272
|
+
const osvRan = hasBinary('osv-scanner') && runOsvScanner(root, findings, notes);
|
|
273
|
+
|
|
274
|
+
// --- known-vulnerable dependencies (best effort, npm projects only) ---
|
|
275
|
+
if (osvRan) {
|
|
276
|
+
// covered above, across all ecosystems
|
|
277
|
+
} else if (fs.existsSync(path.join(root, 'package-lock.json'))) {
|
|
278
|
+
try {
|
|
279
|
+
let auditJson;
|
|
280
|
+
try {
|
|
281
|
+
auditJson = execFileSync('npm', ['audit', '--json'], {
|
|
282
|
+
cwd: root, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], timeout: 60000,
|
|
283
|
+
});
|
|
284
|
+
} catch (err) {
|
|
285
|
+
// npm audit exits non-zero when vulnerabilities exist; the JSON is
|
|
286
|
+
// still on stdout.
|
|
287
|
+
auditJson = err.stdout;
|
|
288
|
+
}
|
|
289
|
+
const meta = JSON.parse(auditJson)?.metadata?.vulnerabilities || {};
|
|
290
|
+
const serious = (meta.high || 0) + (meta.critical || 0);
|
|
291
|
+
if (serious > 0) {
|
|
292
|
+
findings.push({
|
|
293
|
+
id: 'dependency-vulns',
|
|
294
|
+
severity: 'high',
|
|
295
|
+
file: 'package-lock.json',
|
|
296
|
+
detail: `${serious} high/critical known ${serious === 1 ? 'vulnerability' : 'vulnerabilities'} (npm audit).`,
|
|
297
|
+
});
|
|
298
|
+
}
|
|
299
|
+
} catch {
|
|
300
|
+
notes.push('Could not run `npm audit` (offline or npm unavailable) — dependency check skipped.');
|
|
301
|
+
}
|
|
302
|
+
} else if (fs.existsSync(path.join(root, 'package.json'))) {
|
|
303
|
+
notes.push('No package-lock.json, so the dependency vulnerability check was skipped. If your package.json has a "dependencies" section, run `npm install` once to create that file; otherwise there is nothing to do.');
|
|
304
|
+
} else {
|
|
305
|
+
// Non-npm project (Python, Ruby, Go, Rust, PHP…). The built-in check is
|
|
306
|
+
// npm-only; osv-scanner covers every ecosystem but isn't installed, so
|
|
307
|
+
// point the user at how to check their own stack rather than going
|
|
308
|
+
// silent — a vibe-coded Flask app can easily pin a flask version with
|
|
309
|
+
// known CVEs and never hear about it.
|
|
310
|
+
const PY = ['requirements.txt', 'Pipfile', 'pyproject.toml', 'poetry.lock'];
|
|
311
|
+
const other = { 'Gemfile': 'bundler-audit', 'go.mod': 'govulncheck', 'Cargo.toml': 'cargo audit', 'composer.json': 'composer audit' };
|
|
312
|
+
if (PY.some((f) => fs.existsSync(path.join(root, f)))) {
|
|
313
|
+
notes.push('This looks like a Python project. ShipClear\'s built-in dependency check is npm-only — to check your Python packages for known vulnerabilities, run `pip install pip-audit && pip-audit`, or install osv-scanner and ShipClear will use it automatically next time.');
|
|
314
|
+
} else {
|
|
315
|
+
const manifest = Object.keys(other).find((f) => fs.existsSync(path.join(root, f)));
|
|
316
|
+
if (manifest) {
|
|
317
|
+
notes.push(`Dependency check skipped — ShipClear's built-in check is npm-only. For this project, run \`${other[manifest]}\`, or install osv-scanner and ShipClear will check ${manifest} automatically next time.`);
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
return { findings, notes, inGit };
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
function allFilesHasTemplate(root, files) {
|
|
326
|
+
return files.some(isEnvTemplate) || fs.existsSync(path.join(root, '.env.example'));
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
function hasBinary(name) {
|
|
330
|
+
try {
|
|
331
|
+
execFileSync(process.platform === 'win32' ? 'where' : 'which', [name], {
|
|
332
|
+
stdio: ['ignore', 'ignore', 'ignore'],
|
|
333
|
+
});
|
|
334
|
+
return true;
|
|
335
|
+
} catch {
|
|
336
|
+
return false;
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
// When gitleaks is installed, run it as a second opinion on top of the
|
|
341
|
+
// built-in patterns — its rule set is deeper than ours. Everything it finds
|
|
342
|
+
// that we didn't becomes an additional secret-in-code finding.
|
|
343
|
+
function runGitleaks(root, findings, notes, ignored) {
|
|
344
|
+
const reportPath = path.join(os.tmpdir(), `shipclear-gitleaks-${process.pid}.json`);
|
|
345
|
+
const attempts = [
|
|
346
|
+
['dir', root, '--no-banner', '--exit-code', '0', '--report-format', 'json', '--report-path', reportPath],
|
|
347
|
+
['detect', '--source', root, '--no-git', '--no-banner', '--exit-code', '0', '--report-format', 'json', '--report-path', reportPath],
|
|
348
|
+
];
|
|
349
|
+
try {
|
|
350
|
+
let ran = false;
|
|
351
|
+
for (const args of attempts) {
|
|
352
|
+
try {
|
|
353
|
+
execFileSync('gitleaks', args, { cwd: root, stdio: ['ignore', 'ignore', 'ignore'], timeout: 120000 });
|
|
354
|
+
ran = true;
|
|
355
|
+
break;
|
|
356
|
+
} catch (err) {
|
|
357
|
+
// Older/newer CLI syntax mismatch → try the next form. A written
|
|
358
|
+
// report despite non-zero exit also counts as a successful run.
|
|
359
|
+
if (fs.existsSync(reportPath)) { ran = true; break; }
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
if (!ran) return;
|
|
363
|
+
const leaks = JSON.parse(fs.readFileSync(reportPath, 'utf8'));
|
|
364
|
+
const already = new Set(findings.map((f) => `${f.file}:${f.line || ''}`));
|
|
365
|
+
let added = 0;
|
|
366
|
+
for (const leak of leaks) {
|
|
367
|
+
const rel = path.relative(root, path.isAbsolute(leak.File) ? leak.File : path.join(root, leak.File))
|
|
368
|
+
.split(path.sep).join('/');
|
|
369
|
+
if (ignored(rel) || already.has(`${rel}:${leak.StartLine}`)) continue;
|
|
370
|
+
findings.push({
|
|
371
|
+
id: 'secret-in-code',
|
|
372
|
+
severity: 'critical',
|
|
373
|
+
file: rel,
|
|
374
|
+
line: leak.StartLine,
|
|
375
|
+
detail: `${leak.RuleID} (found by gitleaks)`,
|
|
376
|
+
});
|
|
377
|
+
added++;
|
|
378
|
+
}
|
|
379
|
+
notes.push(`gitleaks also ran (${added === 0 ? 'no additional findings' : `${added} additional finding${added === 1 ? '' : 's'}`}).`);
|
|
380
|
+
} catch {
|
|
381
|
+
// gitleaks present but unusable — the built-in scan already ran.
|
|
382
|
+
} finally {
|
|
383
|
+
fs.rmSync(reportPath, { force: true });
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
// When osv-scanner is installed it covers every ecosystem (Python, Go, Rust,
|
|
388
|
+
// …), so it takes precedence over the npm-only audit path.
|
|
389
|
+
function runOsvScanner(root, findings, notes) {
|
|
390
|
+
try {
|
|
391
|
+
let out;
|
|
392
|
+
try {
|
|
393
|
+
out = execFileSync('osv-scanner', ['--format', 'json', '-r', root], {
|
|
394
|
+
cwd: root, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], timeout: 120000,
|
|
395
|
+
});
|
|
396
|
+
} catch (err) {
|
|
397
|
+
out = err.stdout; // non-zero exit when vulnerabilities exist
|
|
398
|
+
}
|
|
399
|
+
const results = JSON.parse(out)?.results || [];
|
|
400
|
+
let vulns = 0;
|
|
401
|
+
for (const r of results) for (const p of r.packages || []) vulns += (p.vulnerabilities || []).length;
|
|
402
|
+
if (vulns > 0) {
|
|
403
|
+
findings.push({
|
|
404
|
+
id: 'dependency-vulns',
|
|
405
|
+
severity: 'high',
|
|
406
|
+
detail: `${vulns} known ${vulns === 1 ? 'vulnerability' : 'vulnerabilities'} across your dependencies (osv-scanner).`,
|
|
407
|
+
});
|
|
408
|
+
}
|
|
409
|
+
notes.push('Dependency check ran via osv-scanner (covers all ecosystems).');
|
|
410
|
+
return true;
|
|
411
|
+
} catch {
|
|
412
|
+
return false;
|
|
413
|
+
}
|
|
414
|
+
}
|