shipclear 0.2.1 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -4
- package/adapters/claude-code/.claude-plugin/plugin.json +1 -1
- package/adapters/claude-code/commands/safe-to-ship.md +7 -1
- package/adapters/generic/INSTRUCTIONS.md +7 -1
- package/cli/bin/shipclear.js +111 -3
- package/cli/lib/badge.js +216 -0
- package/cli/lib/checks.js +13 -2
- package/cli/lib/git.js +21 -4
- package/cli/lib/infra.js +536 -0
- package/cli/lib/patterns.js +220 -5
- package/cli/lib/probe.js +175 -0
- package/cli/lib/report.js +77 -8
- package/core/gate-checklist.md +13 -1
- package/docs/quickstart.md +23 -0
- package/docs/troubleshooting.md +37 -0
- package/package.json +1 -1
package/cli/lib/patterns.js
CHANGED
|
@@ -29,10 +29,25 @@ function isServiceRoleJwt(match) {
|
|
|
29
29
|
return base64UrlDecode(match.split('.')[1]).includes('"role":"service_role"');
|
|
30
30
|
}
|
|
31
31
|
|
|
32
|
+
// A PEM header on its own is documentation — a line telling the reader where
|
|
33
|
+
// to paste their certificate, which is how our own docs describe setting one
|
|
34
|
+
// up. Key *material* is the header followed by its base64 body. Requiring the
|
|
35
|
+
// body is what keeps a shell comment from being reported as a leaked key, and
|
|
36
|
+
// (because history findings run through the same matcher) keeps a comment from
|
|
37
|
+
// ever recommending a force-push of rewritten history.
|
|
38
|
+
function privateKeyHasBody(match, content, index) {
|
|
39
|
+
const after = (content || '').slice(index + match.length, index + match.length + 400);
|
|
40
|
+
// Covers both real PEM files and one-line JSON/env embeddings, where the
|
|
41
|
+
// body follows a literal \n escape rather than a newline.
|
|
42
|
+
const body = after.replace(/\\n/g, '\n').match(/[A-Za-z0-9+/=]{40,}/);
|
|
43
|
+
return !!body && entropy(body[0]) > 3.5;
|
|
44
|
+
}
|
|
45
|
+
|
|
32
46
|
// Provider-specific formats. These are high-confidence: a match is treated as
|
|
33
47
|
// a real secret even if it "looks fake", because a real-shaped key must be
|
|
34
48
|
// rotated anyway once it has touched a tracked file. A pattern may carry a
|
|
35
|
-
// `validate(match)` to veto matches that are provably
|
|
49
|
+
// `validate(match, content, index)` to veto matches that are provably
|
|
50
|
+
// templates.
|
|
36
51
|
export const SECRET_PATTERNS = [
|
|
37
52
|
{ id: 'aws-access-key', name: 'AWS access key', regex: /\bAKIA[0-9A-Z]{16}\b/g },
|
|
38
53
|
{ id: 'anthropic-api-key', name: 'Anthropic API key', regex: /\bsk-ant-[A-Za-z0-9_-]{20,}\b/g },
|
|
@@ -51,7 +66,7 @@ export const SECRET_PATTERNS = [
|
|
|
51
66
|
{ id: 'xai-api-key', name: 'xAI API key', regex: /\bxai-[A-Za-z0-9]{20,}\b/g },
|
|
52
67
|
{ id: 'azure-account-key', name: 'Azure storage account key', regex: /AccountKey=[A-Za-z0-9+/=]{40,}/g },
|
|
53
68
|
{ id: 'supabase-service-role', name: 'Supabase service-role key (JWT)', regex: /\beyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/g, validate: isServiceRoleJwt },
|
|
54
|
-
{ id: 'private-key-block', name: 'Private key material', regex: /-----BEGIN (?:RSA |EC |OPENSSH |DSA |PGP )?PRIVATE KEY(?: BLOCK)?-----/g },
|
|
69
|
+
{ id: 'private-key-block', name: 'Private key material', regex: /-----BEGIN (?:RSA |EC |OPENSSH |DSA |PGP )?PRIVATE KEY(?: BLOCK)?-----/g, validate: privateKeyHasBody },
|
|
55
70
|
{ id: 'connection-string', name: 'Database URL with embedded password', regex: /\b(?:postgres(?:ql)?|mysql|mongodb(?:\+srv)?|redis|amqps?):\/\/[^:/\s'"]+:[^@/\s'"]+@/g, validate: connectionStringHasRealPassword },
|
|
56
71
|
];
|
|
57
72
|
|
|
@@ -101,13 +116,22 @@ export function lineOfIndex(content, index) {
|
|
|
101
116
|
* by position. Overlapping matches are deduplicated in favor of the more
|
|
102
117
|
* specific pattern (SECRET_PATTERNS order, generics last).
|
|
103
118
|
*/
|
|
104
|
-
|
|
119
|
+
// In a documentation file, a key block wrapped in instructions is a worked
|
|
120
|
+
// example, not a leak — "paste your key here", "this file is .gitignored".
|
|
121
|
+
// Outside docs this never applies: a real key pasted into code is a real key.
|
|
122
|
+
const INSTRUCTIONAL = /\.gitignore|paste|example|sample|placeholder|replace|your own|looks like/i;
|
|
123
|
+
|
|
124
|
+
function isWorkedExample(content, index) {
|
|
125
|
+
return INSTRUCTIONAL.test(content.slice(Math.max(0, index - 300), index + 300));
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
export function findSecrets(content, { includeGeneric = true, docFile = false } = {}) {
|
|
105
129
|
const raw = [];
|
|
106
130
|
for (const p of SECRET_PATTERNS) {
|
|
107
131
|
p.regex.lastIndex = 0;
|
|
108
132
|
let m;
|
|
109
133
|
while ((m = p.regex.exec(content))) {
|
|
110
|
-
if (p.validate && !p.validate(m[0])) continue;
|
|
134
|
+
if (p.validate && !p.validate(m[0], content, m.index)) continue;
|
|
111
135
|
raw.push({
|
|
112
136
|
patternId: p.id,
|
|
113
137
|
name: p.name,
|
|
@@ -135,8 +159,11 @@ export function findSecrets(content, { includeGeneric = true } = {}) {
|
|
|
135
159
|
});
|
|
136
160
|
}
|
|
137
161
|
}
|
|
162
|
+
const candidates = docFile
|
|
163
|
+
? raw.filter((h) => !(h.patternId === 'private-key-block' && isWorkedExample(content, h.index)))
|
|
164
|
+
: raw;
|
|
138
165
|
const kept = [];
|
|
139
|
-
for (const cand of
|
|
166
|
+
for (const cand of candidates) {
|
|
140
167
|
const overlaps = kept.some(
|
|
141
168
|
(k) => cand.index < k.index + k.match.length && k.index < cand.index + cand.match.length
|
|
142
169
|
);
|
|
@@ -224,3 +251,191 @@ export function personalPaths(content) {
|
|
|
224
251
|
}
|
|
225
252
|
return hits;
|
|
226
253
|
}
|
|
254
|
+
|
|
255
|
+
// ===========================================================================
|
|
256
|
+
// Deployment configuration patterns.
|
|
257
|
+
//
|
|
258
|
+
// Everything below answers a different question from the patterns above. Those
|
|
259
|
+
// ask "is there a secret written here that shouldn't be?"; these ask "is the
|
|
260
|
+
// value this app actually runs on one that anybody can look up?" — the live
|
|
261
|
+
// config, the fallback it silently uses when a variable is missing, and the
|
|
262
|
+
// places a deployment hands a backing service straight to the internet.
|
|
263
|
+
// Used by lib/infra.js.
|
|
264
|
+
// ===========================================================================
|
|
265
|
+
|
|
266
|
+
// Keys whose *value* is a credential. Compound forms only, with one
|
|
267
|
+
// exception: a bare all-caps `KEY`. A lowercase `key:` is ordinary YAML
|
|
268
|
+
// (`- key: tls.crt`), where testing the value would misfire.
|
|
269
|
+
const SECRET_KEY_NAME = /pass(?:word|wd)?|pword|pwd|secret|token|credential|private|_key$|^api[_-]?key$|^access[_-]?key$/i;
|
|
270
|
+
// Names that match the above but are public by design, or hold something
|
|
271
|
+
// that isn't the credential itself (a path, a length, an algorithm).
|
|
272
|
+
const NOT_SECRET_KEY_NAME = /^(?:NEXT_PUBLIC|VITE|REACT_APP|NUXT_PUBLIC|EXPO_PUBLIC|PUBLIC)_|(?:PUBLIC|PUBLISHABLE|ANON|SITE)_KEY$|_KEY_(?:FILE|PATH|ID)$|_KEYFILE$|_(?:URL|URI|HOST|PORT|ID|NAME|USER|USERNAME|ALGORITHM|ALGO|LENGTH|SIZE|EXPIRE|EXPIRES|EXPIRY|TTL|DAYS|HOURS|MINUTES|SECONDS|ROUNDS|ENABLED|REQUIRED|PROVIDER|TYPE|REGION|BUCKET|ISSUER|AUDIENCE|HEADER|COOKIE|PREFIX|VERSION)$/i;
|
|
273
|
+
|
|
274
|
+
export function isSecretishKey(key) {
|
|
275
|
+
if (NOT_SECRET_KEY_NAME.test(key)) return false;
|
|
276
|
+
return key === 'KEY' || SECRET_KEY_NAME.test(key);
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
// Credentials that ship as the documented default of a service, or that
|
|
280
|
+
// every tutorial for it uses. An attacker's first guess, by definition.
|
|
281
|
+
const KNOWN_DEFAULT_CREDENTIALS = new Set([
|
|
282
|
+
'minioadmin', 'postgres', 'mysql', 'mariadb', 'mongo', 'mongodb', 'redis',
|
|
283
|
+
'elastic', 'elasticsearch', 'rabbitmq', 'neo4j', 'keycloak', 'grafana',
|
|
284
|
+
'root', 'toor', 'admin', 'administrator', 'guest', 'user', 'test', 'dev', 'docker',
|
|
285
|
+
'password', 'passwd', 'pass', 'secret', 'changeme', 'change-me', 'change_me',
|
|
286
|
+
'letmein', 'supersecret', 'mysecret', 'mysecretpassword',
|
|
287
|
+
'1234', '12345', '123456', 'password123', 'admin123',
|
|
288
|
+
]);
|
|
289
|
+
// Not a service's default — a stand-in the project itself shipped and nobody
|
|
290
|
+
// replaced. Just as guessable, but it reads differently in a finding.
|
|
291
|
+
const LEFTOVER_STAND_IN = /^(?:change[-_]?me|changethis|django-insecure-|your[-_]|insecure|dev[-_]?only|test[-_]?pass|replace[-_]?me)/i;
|
|
292
|
+
|
|
293
|
+
export function isKnownDefaultCredential(value) {
|
|
294
|
+
return KNOWN_DEFAULT_CREDENTIALS.has(value.toLowerCase());
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
export function isLeftoverStandIn(value) {
|
|
298
|
+
return LEFTOVER_STAND_IN.test(value);
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
// `${FOO}` / `$FOO` is indirection, not a value — whatever it resolves to
|
|
302
|
+
// isn't visible here, so it can't be judged (check #2 handles the form that
|
|
303
|
+
// carries a default).
|
|
304
|
+
export function isVariableReference(value) {
|
|
305
|
+
return /^\$\{[^}]*\}$/.test(value) || /^\$[A-Za-z_][A-Za-z0-9_]*$/.test(value);
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
const PLACEHOLDER_VALUE = /change|example|sample|dev[-_]?only|todo|xxx|placeholder|dummy|fake|replace|your[-_]|[-_]here$/i;
|
|
309
|
+
|
|
310
|
+
/** A value that is visibly a stand-in rather than a generated credential. */
|
|
311
|
+
export function placeholderShapedCredential(key, value) {
|
|
312
|
+
if (/^<.*>$/.test(value) || /^__.*__$/.test(value)) return true;
|
|
313
|
+
if (PLACEHOLDER_VALUE.test(value)) return true;
|
|
314
|
+
// A key, token or signing secret is generated and long. A short one was
|
|
315
|
+
// typed by a human, which means it is guessable. (Passwords are excluded:
|
|
316
|
+
// plenty of real ones are short, and the lists above already cover the
|
|
317
|
+
// guessable ones.)
|
|
318
|
+
return /secret|token|_key$|^key$|api[_-]?key/i.test(key) && value.length < 12 && !/^[\d.]+$/.test(value);
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
// Signals that this config describes something the public can reach, which is
|
|
322
|
+
// what turns a known credential from untidy into urgent.
|
|
323
|
+
const PRODUCTION_FLAG = /\b(?:ENVIRONMENT|APP_ENV|NODE_ENV|FLASK_ENV|RAILS_ENV|DJANGO_ENV|ENV)\s*[:=]\s*["']?prod(?:uction)?\b|\b(?:SESSION_)?COOKIE_SECURE\s*[:=]\s*["']?[Tt]rue\b/;
|
|
324
|
+
const DOMAIN_SETTING = /(?:BASE_URL|SITE_URL|PUBLIC_URL|FRONTEND_URL|BACKEND_URL|API_URL|ALLOWED_HOSTS|ALLOWED_ORIGINS|CORS_ORIGINS|SERVER_NAME|host)\s*[:=]\s*["']?(?:https?:\/\/)?([a-z0-9][a-z0-9.-]*\.[a-z]{2,})/gi;
|
|
325
|
+
const NON_PUBLIC_HOST = /^(?:localhost|127\.0\.0\.1|0\.0\.0\.0|example\.com|.+\.(?:local|localhost|test|internal|example\.com|docker\.internal|svc\.cluster\.local))$/i;
|
|
326
|
+
|
|
327
|
+
/** The production signal this text carries, as a quotable phrase, or null. */
|
|
328
|
+
export function productionSignal(text) {
|
|
329
|
+
const flag = text.match(PRODUCTION_FLAG);
|
|
330
|
+
if (flag) return `\`${flag[0].trim()}\``;
|
|
331
|
+
DOMAIN_SETTING.lastIndex = 0;
|
|
332
|
+
let m;
|
|
333
|
+
while ((m = DOMAIN_SETTING.exec(text))) {
|
|
334
|
+
if (!NON_PUBLIC_HOST.test(m[1])) return `the public address \`${m[1]}\``;
|
|
335
|
+
}
|
|
336
|
+
return null;
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
// --- #2: a missing secret that silently becomes a known one ---
|
|
340
|
+
|
|
341
|
+
// Shell/compose default expansion: ${FOO:-default}, ${FOO-default}.
|
|
342
|
+
export const SHELL_DEFAULT = /\$\{([A-Za-z_][A-Za-z0-9_]*)\s*:?-\s*([^}\n]*)\}/g;
|
|
343
|
+
|
|
344
|
+
// Code-level "work even if it isn't configured" defaults. Each returns the
|
|
345
|
+
// variable name in group 1 and the default value in group 2.
|
|
346
|
+
export const CODE_DEFAULTS = [
|
|
347
|
+
// Python: os.getenv("X", "default") / os.environ.get("X", "default")
|
|
348
|
+
/(?:os\.)?(?:getenv|environ\.get)\(\s*['"]([A-Za-z_][A-Za-z0-9_]*)['"]\s*,\s*['"]([^'"\n]+)['"]/g,
|
|
349
|
+
// JS/TS: process.env.X || "default" / process.env["X"] ?? "default"
|
|
350
|
+
/process\.env(?:\.([A-Za-z_][A-Za-z0-9_]*)|\[['"]([A-Za-z_][A-Za-z0-9_]*)['"]\])\s*(?:\|\||\?\?)\s*['"]([^'"\n]+)['"]/g,
|
|
351
|
+
// Ruby: ENV.fetch("X", "default") / ENV["X"] || "default"
|
|
352
|
+
/ENV(?:\.fetch\(|\[)\s*['"]([A-Za-z_][A-Za-z0-9_]*)['"]\s*(?:,|\]\s*\|\|)\s*['"]([^'"\n]+)['"]/g,
|
|
353
|
+
// Go: flag.String("x", "default", ...) / os.Getenv fallbacks
|
|
354
|
+
/flag\.String\(\s*"([A-Za-z_][A-Za-z0-9_.-]*)"\s*,\s*"([^"\n]+)"/g,
|
|
355
|
+
// Settings classes (pydantic, dataclasses): secret_key: str = "default"
|
|
356
|
+
/^[ \t]+([a-z_][a-z0-9_]*)\s*:\s*[A-Za-z_][A-Za-z0-9_.\[\]]*\s*=\s*['"]([^'"\n]+)['"]/gm,
|
|
357
|
+
];
|
|
358
|
+
|
|
359
|
+
// --- #3: storage anyone can read, or list ---
|
|
360
|
+
|
|
361
|
+
// `mc anonymous set download` is the trap: despite the name it is not
|
|
362
|
+
// read-only-per-object, it grants s3:ListBucket too.
|
|
363
|
+
// The trailing `alias/bucket` is required, not decoration: a real invocation
|
|
364
|
+
// always names a target, and demanding one is what keeps prose *about* this
|
|
365
|
+
// command — our own fix text included — from being read as the command.
|
|
366
|
+
export const MC_ANONYMOUS = /\bmc\s+(?:--\S+\s+)*(?:anonymous|policy)\s+set(?:-json)?\s+(download|upload|public|write|none)\s+(\S+\/\S+)/g;
|
|
367
|
+
export const AWS_PUBLIC_ACL = /--acl[= ]\s*(public-read-write|public-read)\b/g;
|
|
368
|
+
export const TF_PUBLIC_ACL = /\bacl\s*=\s*"(public-read-write|public-read)"/g;
|
|
369
|
+
export const TF_PUBLIC_ACCESS_BLOCK_OFF = /\b(block_public_acls|block_public_policy|ignore_public_acls|restrict_public_buckets)\s*=\s*false\b/g;
|
|
370
|
+
export const WILDCARD_PRINCIPAL = /"Principal"\s*:\s*(?:"\*"|\{[^}]*"AWS"\s*:\s*(?:\[\s*)?"\*")/g;
|
|
371
|
+
export const S3_ACTION = /"(s3:[A-Za-z*]+)"/g;
|
|
372
|
+
export const GCS_ALL_USERS = /["'](allUsers|allAuthenticatedUsers)["']/g;
|
|
373
|
+
export const AZURE_PUBLIC_CONTAINER = /(?:public_access|container_access_type)\s*=\s*"(container|blob)"/gi;
|
|
374
|
+
export const FIREBASE_OPEN_RULE = /allow\s+([a-z, ]+)\s*:\s*if\s+true\b/g;
|
|
375
|
+
|
|
376
|
+
// --- #4: a public route that forwards into a backing service ---
|
|
377
|
+
|
|
378
|
+
export const PROXY_UPSTREAM_LINE = /reverse_proxy|proxy_pass|ProxyPass|^\s*server\s+\S+:\d+|loadBalancer|load_balancer|serviceName|service_name|^\s*(?:name|number|port):/;
|
|
379
|
+
export const PROXY_ROUTE_LINE = /^\s*(?:handle_path|handle|route|location|path|ProxyPass)\s+([^\s{]+)|^\s*-\s*path:\s*(\S+)/;
|
|
380
|
+
export const HOST_PORT = /\b([a-z0-9][a-z0-9_.-]*):(\d{2,5})\b/gi;
|
|
381
|
+
|
|
382
|
+
// Backing services that should never answer the internet wholesale. Value is
|
|
383
|
+
// the severity: a database or admin UI is worse than an object store, which
|
|
384
|
+
// at least is partly meant to serve files.
|
|
385
|
+
export const BACKING_SERVICES = {
|
|
386
|
+
postgres: 'critical', postgresql: 'critical', mysql: 'critical', mariadb: 'critical',
|
|
387
|
+
mongo: 'critical', mongodb: 'critical', redis: 'critical', valkey: 'critical',
|
|
388
|
+
elasticsearch: 'critical', opensearch: 'critical', clickhouse: 'critical',
|
|
389
|
+
couchdb: 'critical', neo4j: 'critical', cassandra: 'critical', memcached: 'critical',
|
|
390
|
+
rabbitmq: 'critical', kafka: 'critical', nats: 'critical',
|
|
391
|
+
consul: 'critical', vault: 'critical', etcd: 'critical',
|
|
392
|
+
adminer: 'critical', pgadmin: 'critical', phpmyadmin: 'critical', mailhog: 'critical',
|
|
393
|
+
kibana: 'critical', grafana: 'critical', prometheus: 'critical',
|
|
394
|
+
minio: 'high', s3: 'high', seaweedfs: 'high', garage: 'high',
|
|
395
|
+
};
|
|
396
|
+
|
|
397
|
+
export const BACKING_PORTS = {
|
|
398
|
+
5432: 'postgres', 3306: 'mysql', 27017: 'mongodb', 6379: 'redis',
|
|
399
|
+
9200: 'elasticsearch', 9300: 'elasticsearch', 5601: 'kibana', 8123: 'clickhouse',
|
|
400
|
+
5984: 'couchdb', 7474: 'neo4j', 7687: 'neo4j', 11211: 'memcached',
|
|
401
|
+
15672: 'rabbitmq', 5672: 'rabbitmq', 9092: 'kafka', 8500: 'consul', 8200: 'vault',
|
|
402
|
+
9090: 'prometheus', 3000: null, 9000: null, 9001: null,
|
|
403
|
+
};
|
|
404
|
+
|
|
405
|
+
// --- #5: debug and monitoring pages open to the public ---
|
|
406
|
+
|
|
407
|
+
// Paths that answer with the internals of a running app. The value is the
|
|
408
|
+
// severity: most are an information leak, a few hand over the environment.
|
|
409
|
+
export const SENSITIVE_PATHS = {
|
|
410
|
+
'/metrics': 'medium', '/debug': 'medium', '/debug/pprof': 'critical',
|
|
411
|
+
'/actuator': 'medium', '/actuator/env': 'critical', '/actuator/heapdump': 'critical',
|
|
412
|
+
'/server-status': 'medium', '/swagger': 'medium', '/swagger-ui': 'medium',
|
|
413
|
+
'/api/docs': 'medium', '/api/redoc': 'medium', '/openapi.json': 'medium',
|
|
414
|
+
'/graphql': 'medium', '/adminer': 'critical', '/pgadmin': 'critical',
|
|
415
|
+
'/.env': 'critical', '/phpinfo.php': 'critical',
|
|
416
|
+
};
|
|
417
|
+
|
|
418
|
+
export const ROUTE_DEFINITION = /@(?:app|router|bp|blueprint)\.(?:get|post|route)\(\s*['"]([^'"]+)['"]|\b(?:app|router|server|api)\.(?:get|use|all)\(\s*['"]([^'"]+)['"]|HandleFunc\(\s*"([^"]+)"|(?:docs_url|redoc_url|openapi_url|metrics_path)\s*=\s*['"]([^'"]+)['"]/g;
|
|
419
|
+
|
|
420
|
+
// Signals that a route is already protected, or only exists outside
|
|
421
|
+
// production. The second kind matters as much as the first: a docs route
|
|
422
|
+
// written as `docs_url="/api/docs" if not settings.is_production else None`
|
|
423
|
+
// is the *correct* pattern and must not be reported.
|
|
424
|
+
export const AUTH_NEARBY = /Depends\(|Security\(|login_required|require(?:s)?_(?:auth|login|admin)|authenticate|authorized|IsAuthenticated|permission_classes|basic_?auth|api_?key|@auth|ensureAuth|requireUser|verify_token|HTTPBasic|guard/i;
|
|
425
|
+
export const NON_PRODUCTION_GUARD = /if\s+not\s+[A-Za-z_.]*(?:is_)?prod|if\s+[A-Za-z_.]*(?:DEBUG|debug)\b|NODE_ENV\s*[!=]==?\s*['"]production|!==\s*['"]production|ENVIRONMENT\s*[!=]=\s*['"]production|\bdev(?:elopment)?_only\b/;
|
|
426
|
+
export const PROXY_DENY = /deny\s+all|return\s+403|respond\s+403|abort\s*\(\s*404/;
|
|
427
|
+
|
|
428
|
+
// --- #6: security headers that only cover part of the site ---
|
|
429
|
+
|
|
430
|
+
export const CSP_HEADER = /Content-Security-Policy/i;
|
|
431
|
+
export const CSP_MENTION = /content[- ]security[- ]policy|\bCSP\b/i;
|
|
432
|
+
|
|
433
|
+
// --- #7: the server fetching any address a user hands it ---
|
|
434
|
+
|
|
435
|
+
// A URL that arrives in a request. Pydantic/FastAPI bodies, Express, Flask,
|
|
436
|
+
// Rails — the shapes an "import from URL" feature actually gets written in.
|
|
437
|
+
export const REQUEST_URL_SOURCE = /\b(?:HttpUrl|AnyHttpUrl|AnyUrl)\b|(?:req|request)\.(?:body|query|params|json|args|form|GET|POST)(?:\.|\[\s*['"])\s*[A-Za-z_]*(?:url|uri|link|src|href|endpoint|webhook|target|image)/i;
|
|
438
|
+
export const OUTBOUND_FETCH = /\b(?:httpx|requests|aiohttp|urllib3|session|client)\.(?:get|post|put|patch|request|stream)\s*\(|\burllib\.request\.urlopen\s*\(|\bAsyncClient\(\)[^\n]*\.(?:get|post)\s*\(|\bfetch\s*\(|\baxios(?:\.(?:get|post|put|request))?\s*\(|\bgot(?:\.(?:get|post))?\s*\(|Net::HTTP|HTTParty|\bcurl\s+["'$]/;
|
|
439
|
+
// Anything that resolves the address and refuses the private ones.
|
|
440
|
+
export const SSRF_GUARD = /ipaddress|ip_address|is_private|is_global|is_loopback|is_link_local|getaddrinfo|gethostbyname|allow(?:ed)?[_-]?(?:list|hosts|domains)|whitelist|blocklist|blacklist|denied_hosts|advocate|ssrf|validate_url|is_safe_url|safe_url|private_ip|169\.254\.169\.254/i;
|
|
441
|
+
export const FOLLOWS_REDIRECTS = /follow_redirects\s*=\s*True|allow_redirects\s*=\s*True|maxRedirects\s*:\s*[1-9]|redirect\s*:\s*['"]follow['"]|followRedirect\s*:\s*true/;
|
package/cli/lib/probe.js
ADDED
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
// The live probe: ask the deployed app what is actually reachable.
|
|
2
|
+
//
|
|
3
|
+
// Three of the findings on the real app that prompted these checks were
|
|
4
|
+
// invisible in source — a bucket policy applied at runtime by a one-shot
|
|
5
|
+
// container, a proxy route's real behaviour, a response header that isn't
|
|
6
|
+
// there. Five were confirmable with a single GET each. This is the difference
|
|
7
|
+
// between "your config looks risky" and "anyone can list your users' uploads
|
|
8
|
+
// right now, here is the URL".
|
|
9
|
+
//
|
|
10
|
+
// Hard rules, enforced here and not negotiable elsewhere in the codebase:
|
|
11
|
+
// • GET and HEAD only. Nothing is ever written, posted, or deleted.
|
|
12
|
+
// • No credentials, cookies, or auth headers are sent, and no login or
|
|
13
|
+
// auth-bypass is ever attempted. A 401 or 403 is a pass, not a puzzle.
|
|
14
|
+
// • Only the URL the user typed, and paths beneath it.
|
|
15
|
+
// • Opt-in per run (`ship --probe <url>`), after an explicit confirmation.
|
|
16
|
+
// The HTTP layer is injected so the whole thing is testable without a network.
|
|
17
|
+
import { SENSITIVE_PATHS } from './patterns.js';
|
|
18
|
+
|
|
19
|
+
const HEADERS_THAT_PROTECT_PAGES = [
|
|
20
|
+
['content-security-policy', 'Content-Security-Policy (stops injected scripts running)'],
|
|
21
|
+
['strict-transport-security', 'Strict-Transport-Security (forces https)'],
|
|
22
|
+
['x-frame-options', 'X-Frame-Options or a frame-ancestors policy (stops your site being framed)'],
|
|
23
|
+
['x-content-type-options', 'X-Content-Type-Options (stops content-type guessing)'],
|
|
24
|
+
];
|
|
25
|
+
|
|
26
|
+
// A response body that is an index of a bucket's contents, in the shapes S3,
|
|
27
|
+
// MinIO and ordinary directory listings produce.
|
|
28
|
+
const LOOKS_LIKE_A_LISTING = /<ListBucketResult|<Key>|"Contents"\s*:|<title>Index of|Directory listing for/i;
|
|
29
|
+
|
|
30
|
+
/** Normalize what a user typed into an origin we can append paths to. */
|
|
31
|
+
export function normalizeBase(input) {
|
|
32
|
+
const scheme = String(input).match(/^([a-z][a-z0-9+.-]*):/i);
|
|
33
|
+
if (scheme && !/^https?$/i.test(scheme[1])) {
|
|
34
|
+
return { error: `Only http:// and https:// addresses can be probed (\`${scheme[1]}:\` is not one).` };
|
|
35
|
+
}
|
|
36
|
+
const withScheme = /^https?:\/\//i.test(input) ? input : `https://${input}`;
|
|
37
|
+
let url;
|
|
38
|
+
try {
|
|
39
|
+
url = new URL(withScheme);
|
|
40
|
+
} catch {
|
|
41
|
+
return { error: `\`${input}\` is not a web address ShipClear can read.` };
|
|
42
|
+
}
|
|
43
|
+
if (url.protocol !== 'http:' && url.protocol !== 'https:') {
|
|
44
|
+
return { error: 'Only http:// and https:// addresses can be probed.' };
|
|
45
|
+
}
|
|
46
|
+
return { base: url.origin + url.pathname.replace(/\/+$/, '') };
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** One request, built so it cannot be anything but a read. */
|
|
50
|
+
async function httpRead(url, method) {
|
|
51
|
+
const controller = new AbortController();
|
|
52
|
+
const timer = setTimeout(() => controller.abort(), 8000);
|
|
53
|
+
try {
|
|
54
|
+
const res = await fetch(url, {
|
|
55
|
+
method,
|
|
56
|
+
redirect: 'manual',
|
|
57
|
+
headers: { 'user-agent': 'ShipClear probe (read-only)' },
|
|
58
|
+
signal: controller.signal,
|
|
59
|
+
});
|
|
60
|
+
const headers = {};
|
|
61
|
+
res.headers.forEach((v, k) => { headers[k.toLowerCase()] = v; });
|
|
62
|
+
let body = '';
|
|
63
|
+
if (method === 'GET') body = (await res.text()).slice(0, 4096);
|
|
64
|
+
return { status: res.status, headers, body };
|
|
65
|
+
} catch (err) {
|
|
66
|
+
return { error: err.name === 'AbortError' ? 'no answer within 8 seconds' : err.message };
|
|
67
|
+
} finally {
|
|
68
|
+
clearTimeout(timer);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
const reachable = (res) => !res.error && res.status >= 200 && res.status < 300;
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Probe a deployed app. `staticFindings` is the deterministic layer's output:
|
|
76
|
+
* findings it already made are *confirmed* in place (escalated and reworded)
|
|
77
|
+
* rather than reported twice, so the report says "proven", not "and also".
|
|
78
|
+
* Returns { findings, notes, requests }.
|
|
79
|
+
*/
|
|
80
|
+
export async function runProbe(baseUrl, staticFindings = [], { request = httpRead } = {}) {
|
|
81
|
+
const { base, error } = normalizeBase(baseUrl);
|
|
82
|
+
if (error) return { findings: [], notes: [`Probe skipped: ${error}`], requests: 0 };
|
|
83
|
+
|
|
84
|
+
const findings = [];
|
|
85
|
+
const notes = [];
|
|
86
|
+
let requests = 0;
|
|
87
|
+
const get = async (path) => { requests++; return request(base + path, 'GET'); };
|
|
88
|
+
|
|
89
|
+
// Some apps answer 200 with the same page for every path (SPA catch-alls).
|
|
90
|
+
// Without this, every probe below would "find" everything.
|
|
91
|
+
const sentinel = await get(`/shipclear-probe-${Math.random().toString(36).slice(2, 10)}`);
|
|
92
|
+
const answersEverything = reachable(sentinel);
|
|
93
|
+
if (sentinel.error) {
|
|
94
|
+
return {
|
|
95
|
+
findings: [],
|
|
96
|
+
notes: [`Probe could not reach ${base} (${sentinel.error}) — nothing was tested. Check the address, or that the site is up.`],
|
|
97
|
+
requests,
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
const differsFromSentinel = (res) =>
|
|
101
|
+
!answersEverything
|
|
102
|
+
|| res.headers['content-type'] !== sentinel.headers['content-type']
|
|
103
|
+
|| Math.abs(res.body.length - sentinel.body.length) > 64;
|
|
104
|
+
if (answersEverything) {
|
|
105
|
+
notes.push('This site answers 200 for addresses that do not exist, so the probe only reported pages whose response actually differed from that catch-all.');
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
// --- the headers on the response that is an actual page ---
|
|
109
|
+
const root = await get('/');
|
|
110
|
+
if (reachable(root)) {
|
|
111
|
+
const missing = HEADERS_THAT_PROTECT_PAGES.filter(([h]) => !root.headers[h]);
|
|
112
|
+
// frame-ancestors in a CSP does the job of X-Frame-Options.
|
|
113
|
+
const csp = root.headers['content-security-policy'] || '';
|
|
114
|
+
const stillMissing = missing.filter(([h]) => !(h === 'x-frame-options' && /frame-ancestors/i.test(csp)));
|
|
115
|
+
if (stillMissing.length) {
|
|
116
|
+
const existing = staticFindings.find((f) => f.id === 'headers-html-uncovered');
|
|
117
|
+
const detail = `your live home page is served without ${stillMissing.length} of these headers: `
|
|
118
|
+
+ `${stillMissing.map(([, label]) => label).join('; ')}. Checked just now with a single request to ${base}/.`;
|
|
119
|
+
if (existing) {
|
|
120
|
+
existing.detail = detail;
|
|
121
|
+
existing.probeConfirmed = true;
|
|
122
|
+
} else {
|
|
123
|
+
findings.push({ id: 'headers-html-uncovered', severity: 'medium', file: `${base}/`, detail, probeConfirmed: true });
|
|
124
|
+
}
|
|
125
|
+
} else {
|
|
126
|
+
notes.push(`Probe: your home page does send the headers that protect pages (checked ${base}/).`);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
// --- debug and monitoring pages, as the internet sees them ---
|
|
131
|
+
for (const [path, staticSeverity] of Object.entries(SENSITIVE_PATHS)) {
|
|
132
|
+
const res = await get(path);
|
|
133
|
+
if (!reachable(res) || !differsFromSentinel(res)) continue;
|
|
134
|
+
// A suspected-open route that is *proven* open is worse than suspected.
|
|
135
|
+
const severity = staticSeverity === 'critical' ? 'critical' : 'high';
|
|
136
|
+
// Deliberately not "from the internet": the address might be a staging
|
|
137
|
+
// host or localhost, and the claim has to stay true for whatever was typed.
|
|
138
|
+
const detail = `\`GET ${path}\` answered ${res.status} to a request carrying no login of any kind. Verified just now against ${base}.`;
|
|
139
|
+
const existing = staticFindings.find((f) => f.id === 'debug-route-public' && (f.detail || '').includes(path));
|
|
140
|
+
if (existing) {
|
|
141
|
+
existing.severity = severity;
|
|
142
|
+
existing.detail = `${existing.detail} Confirmed live: ${detail}`;
|
|
143
|
+
existing.probeConfirmed = true;
|
|
144
|
+
} else {
|
|
145
|
+
findings.push({ id: 'debug-route-public', severity, file: `${base}${path}`, detail, probeConfirmed: true });
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
// --- the storage prefixes the proxy config pointed at ---
|
|
150
|
+
for (const f of staticFindings.filter((x) => x.id === 'proxy-exposes-service')) {
|
|
151
|
+
const prefix = (f.route || '/').replace(/\*+$/, '').replace(/\/+$/, '');
|
|
152
|
+
const listing = await get(`${prefix}/`);
|
|
153
|
+
if (reachable(listing) && LOOKS_LIKE_A_LISTING.test(listing.body)) {
|
|
154
|
+
findings.push({
|
|
155
|
+
id: 'public-bucket',
|
|
156
|
+
severity: 'critical',
|
|
157
|
+
file: `${base}${prefix}/`,
|
|
158
|
+
listing: true,
|
|
159
|
+
probeConfirmed: true,
|
|
160
|
+
detail: `\`GET ${prefix}/\` returned a list of the files in your storage, to a request with no login. Anyone with this address can read that list and download what is in it. This is not a reading of your config — it is what your live site answered just now.`,
|
|
161
|
+
});
|
|
162
|
+
}
|
|
163
|
+
// The service's own API answering through the route is the proof that the
|
|
164
|
+
// route forwards more than the files it was meant to serve.
|
|
165
|
+
const health = await get(`${prefix}/minio/health/live`);
|
|
166
|
+
if (reachable(health)) {
|
|
167
|
+
f.severity = f.severity === 'critical' ? 'critical' : 'high';
|
|
168
|
+
f.probeConfirmed = true;
|
|
169
|
+
f.detail = `${f.detail} Confirmed live: \`GET ${prefix}/minio/health/live\` answered ${health.status}, so the service's own API — not just your files — is reachable from the internet.`;
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
notes.push(`Probe: ${requests} read-only requests to ${base} (GET only; nothing was written, no logins attempted).`);
|
|
174
|
+
return { findings, notes, requests };
|
|
175
|
+
}
|
package/cli/lib/report.js
CHANGED
|
@@ -11,11 +11,20 @@ const untrackThenCommit = (file, what) =>
|
|
|
11
11
|
|
|
12
12
|
const dbUrlNote = ' (For a database URL, "the service" is wherever your database is hosted — change its password there.)';
|
|
13
13
|
|
|
14
|
+
// "Change its password there" is incomplete — and silently so — for a
|
|
15
|
+
// database running in a container: POSTGRES_PASSWORD and friends only apply
|
|
16
|
+
// the first time the data volume is created. Change the variable on a
|
|
17
|
+
// database that already has data and the old password keeps working, while
|
|
18
|
+
// the user believes they have rotated it.
|
|
19
|
+
const containerDbNote = (f) => (f.dbInContainer
|
|
20
|
+
? ' One more step, and it is easy to miss: changing this variable alone will NOT work if your database already has data — that variable only applies the first time the database is created. Do both: run `ALTER USER <your db user> WITH PASSWORD \'<new password>\';` against the database itself (`docker compose exec db psql -U <your db user>` gets you a prompt), and update the variable so a future rebuild matches. Then restart the services that connect to it.'
|
|
21
|
+
: '');
|
|
22
|
+
|
|
14
23
|
const EXPLAIN = {
|
|
15
24
|
'secret-in-code': {
|
|
16
25
|
title: 'A secret key is written directly in your code',
|
|
17
26
|
why: 'Anyone who sees this code — on GitHub, in a screenshot, in your deployed app — can use this key as you, and you get the bill.',
|
|
18
|
-
fix: 'Move the value into your .env file and read it from an environment variable (`process.env.YOUR_KEY_NAME` in JavaScript, `os.environ["YOUR_KEY_NAME"]` in Python). Most frameworks (Next.js, Create React App, Django, Rails, and others) load .env automatically; plain Node.js does not — add `require(\'dotenv\').config()` at the top of your entry file (`npm install dotenv` first), or on Node 20+ run your app with `node --env-file=.env yourapp.js`. Then rotate the key: log in to the provider and generate a new one, because this one is burned.' + dbUrlNote,
|
|
27
|
+
fix: (f) => 'Move the value into your .env file and read it from an environment variable (`process.env.YOUR_KEY_NAME` in JavaScript, `os.environ["YOUR_KEY_NAME"]` in Python). Most frameworks (Next.js, Create React App, Django, Rails, and others) load .env automatically; plain Node.js does not — add `require(\'dotenv\').config()` at the top of your entry file (`npm install dotenv` first), or on Node 20+ run your app with `node --env-file=.env yourapp.js`. Then rotate the key: log in to the provider and generate a new one, because this one is burned.' + dbUrlNote + containerDbNote(f),
|
|
19
28
|
},
|
|
20
29
|
'secret-in-history': {
|
|
21
30
|
title: 'A secret exists in your git history',
|
|
@@ -132,6 +141,53 @@ const EXPLAIN = {
|
|
|
132
141
|
why: 'Your AI kept the logins it made for testing in .shipclear/test-accounts.json (git-ignored, so not in the repo). If any of them were ever added to a real database, they are working logins to your live app.',
|
|
133
142
|
fix: 'Before launch: make sure none of these accounts exist in your production database, then delete .shipclear/test-accounts.json. Nothing to do in git — this is about your live app, not the code.',
|
|
134
143
|
},
|
|
144
|
+
'credential-in-use-is-public': {
|
|
145
|
+
title: 'Your app is running on a credential anyone can look up',
|
|
146
|
+
why: 'This is not a secret written in the wrong place — it is a secret that was never secret. A value copied from your own example file is published in your repo; a service\'s default password is in its documentation. Being "just the default" is the problem: defaults are the first thing anyone tries, by automated scanners, within hours of a domain going live.',
|
|
147
|
+
fix: (f) => 'Generate a real value and use it in both places. To make one: `openssl rand -base64 36` (Mac/Linux) or, on any machine with Node, `node -e "console.log(require(\'crypto\').randomBytes(27).toString(\'base64\'))"`. Put it in `.env` only — never in the example file — and change it on the service itself too, or they will no longer match. Then stop it coming back: make the app refuse to start when the value is still a placeholder (a one-line check at startup), because that is what turns this from something to remember into something that cannot happen.'
|
|
148
|
+
+ containerDbNote(f),
|
|
149
|
+
},
|
|
150
|
+
'secret-default-fallback': {
|
|
151
|
+
title: 'A missing secret quietly becomes a known one',
|
|
152
|
+
why: 'If this variable is ever missing — a new server, a typo, a fresh deploy — your app will not fail. It will start up, use this fallback value instead, and look perfectly healthy. That value is written in your code for anyone to read, so the app ends up running on a password that is public, with nothing anywhere saying so.',
|
|
153
|
+
fix: (f) => {
|
|
154
|
+
// The reader's own variable name, not a placeholder — same rule as the
|
|
155
|
+
// file-naming fix texts above.
|
|
156
|
+
const k = f.key ? f.key.toUpperCase() : 'YOUR_VARIABLE';
|
|
157
|
+
const lower = f.key || 'your_variable';
|
|
158
|
+
return `Remove the fallback so a missing value fails loudly instead of quietly. In docker-compose, write \`\${${k}:?set ${k} in .env}\` — the \`:?\` makes the container refuse to start and print that message instead of booting on a known password. In code, read the variable with no default and raise if it is missing: \`os.environ["${k}"]\` in Python rather than \`os.getenv("${k}", "something")\`, or for a settings class \`${lower}: str\` with no \`= "..."\` after it; in JavaScript, \`if (!process.env.${k}) throw new Error("${k} is not set")\`. A crash on the first deploy is a good outcome here; a silent public password is not.`
|
|
159
|
+
+ containerDbNote(f);
|
|
160
|
+
},
|
|
161
|
+
},
|
|
162
|
+
'public-bucket': {
|
|
163
|
+
title: 'Anyone can read your uploaded files — and list them',
|
|
164
|
+
why: 'This does not just let someone view one file if they already have the link. It lets anyone ask your storage for a list of every file in it, and then download them all. If your users upload anything personal — photos, screenshots, documents, invoices — it is public right now, and a listing is what turns "nobody knows the URL" into "here is the index".',
|
|
165
|
+
fix: (f) => (f.listing
|
|
166
|
+
? 'Grant permission to read individual objects only, never to list the bucket. For MinIO, replace `mc anonymous set download` with an explicit policy that allows just `s3:GetObject` on `arn:aws:s3:::YOUR-BUCKET/*` and nothing on the bucket itself. For AWS S3, drop `s3:ListBucket` from the policy and turn the public-access block back on. Then check it rather than trusting the config: open your storage path in a browser (for example `https://yourapp.com/media/your-bucket/`) and confirm you do NOT get a list of files. Better still, serve user files through your own app so it can check who is asking.'
|
|
167
|
+
: 'Confirm you meant this bucket to be world-readable. If it only holds things like public logos, that is fine — but add nothing else to it. If it holds anything a user uploaded, make it private and serve those files through your app, which can check who is asking. Verify by opening the bucket path in a browser window where you are not logged in.'),
|
|
168
|
+
},
|
|
169
|
+
'proxy-exposes-service': {
|
|
170
|
+
title: 'A public web address forwards straight into a backing service',
|
|
171
|
+
why: 'Everything that service answers is now reachable from the internet, not only the part you meant to share — its own status, admin and API endpoints come along with the route. If that service also uses a default password (very common for databases and storage in a docker-compose setup), it is fully exposed.',
|
|
172
|
+
fix: (f) => `Narrow the route to the exact paths you need and reject everything else. For \`${f.route || 'that route'}\`, that means allowing only the specific path prefix your pages actually request, blocking the service's own API and admin paths, and allowing only \`GET\` and \`HEAD\` if it is just serving files. Check your work from outside: \`curl -i https://yourapp.com${f.route && f.route !== '/' ? f.route.replace(/[*]$/, '') : '/'}\` for something you expect to work, and a request for the service's own health or admin path to confirm it does not. If nothing in the browser needs this service directly, remove the public route altogether and have your app fetch from it internally.`,
|
|
173
|
+
},
|
|
174
|
+
'debug-route-public': {
|
|
175
|
+
title: 'A debug or monitoring page is open to the public',
|
|
176
|
+
why: 'These pages exist to tell you what your app is doing — request counts, internal paths, configuration, sometimes environment variables or a memory dump. Handed to a stranger, that is a map of your app and, at worst, your credentials.',
|
|
177
|
+
fix: 'Either require authentication for this route, or make it exist only outside production. The pattern to copy is a conditional: define the route only when a production flag is off (in FastAPI, `docs_url="/api/docs" if not settings.is_production else None`; in Express, wrap the route in `if (process.env.NODE_ENV !== "production")`). If you need metrics in production, put them behind a login or restrict them to your own network at the proxy. Then confirm with `curl -i https://yourapp.com/<the path above>` — a 401, 403 or 404 is what you want to see.',
|
|
178
|
+
},
|
|
179
|
+
'headers-html-uncovered': {
|
|
180
|
+
// Covers both shapes this finding takes: headers mounted on the API only,
|
|
181
|
+
// and (when the probe checked the live root) none on the pages at all.
|
|
182
|
+
title: 'Your safety headers do not cover the pages people load',
|
|
183
|
+
why: 'These headers protect the pages people load in a browser — they tell the browser what the page is allowed to run. Setting them on your API, which does not run scripts, protects nothing. This one is easy to believe you have handled, because the configuration genuinely exists.',
|
|
184
|
+
fix: 'Set the headers on the responses that serve your pages, not only on the API: the frontend service, the catch-all proxy route, or your framework\'s middleware for all paths. Then verify from outside rather than from the config: run `curl -I https://yourapp.com/` and check that `content-security-policy` is actually listed in the output (also worth seeing there: `strict-transport-security`, `x-frame-options`, `x-content-type-options`).',
|
|
185
|
+
},
|
|
186
|
+
'ssrf-unchecked-fetch': {
|
|
187
|
+
title: 'Your server will fetch any web address a user gives it',
|
|
188
|
+
why: 'Your server sits inside your private network, so "fetch this address" also means your database, your file storage, your admin pages — and on a cloud server, the internal address that hands out your cloud credentials. Whatever comes back can end up in whatever the feature shows the user. This is the usual cost of an import-from-URL, link-preview, avatar-from-URL or webhook feature.',
|
|
189
|
+
fix: 'Before fetching, resolve the address and refuse anything that is not a public internet address. In Python: `ipaddress.ip_address(socket.gethostbyname(host)).is_global` must be true. In Node: resolve with `dns.promises.lookup(host)` and reject private ranges (`10.`, `172.16–31.`, `192.168.`, `127.`, `169.254.`) plus `::1` and `fc00::/7`. Then do the same check again after every redirect, and cap how many you follow (`follow_redirects=False` and handling them yourself is the simplest safe version) — a URL check on its own is stepped around by a redirect, which is the part people miss. Allowing only a short list of domains you expect is even better when the feature can live with it.',
|
|
190
|
+
},
|
|
135
191
|
'env-example-missing': {
|
|
136
192
|
title: 'No .env.example template exists',
|
|
137
193
|
why: 'Not a leak — just a courtesy: a template tells collaborators (and AI agents) which settings exist without revealing any values.',
|
|
@@ -170,8 +226,12 @@ const QUICK_VERDICT_TEXT = {
|
|
|
170
226
|
CLEARED: '✅ No secrets found in this quick check — but this is NOT the full gate (it looks at your current files only, not git history or dependencies).',
|
|
171
227
|
};
|
|
172
228
|
|
|
173
|
-
|
|
174
|
-
|
|
229
|
+
// Deliberately not a badge image. A static green "cleared to ship" picture
|
|
230
|
+
// keeps making that claim on every later commit, checked or not — a false
|
|
231
|
+
// GO, handed out by the tool whose whole job is preventing them. The real
|
|
232
|
+
// badge is a live GitHub Actions status installed by `shipclear badge`,
|
|
233
|
+
// which re-runs this gate on GitHub for every push (see lib/badge.js).
|
|
234
|
+
export const BADGE_NEXT_STEP = 'shipclear badge';
|
|
175
235
|
|
|
176
236
|
function sorted(findings) {
|
|
177
237
|
return [...findings].sort(
|
|
@@ -229,8 +289,10 @@ export function renderTerminal({ findings, notes, verdict, fixes = [], version,
|
|
|
229
289
|
// that claim. Never offer it off the back of a quick, partial check.
|
|
230
290
|
if (verdict === 'CLEARED' && !quick) {
|
|
231
291
|
lines.push('');
|
|
232
|
-
lines.push(' Show it off —
|
|
233
|
-
lines.push(
|
|
292
|
+
lines.push(' Show it off — put a badge in your README that stays honest:');
|
|
293
|
+
lines.push(` ${BADGE_NEXT_STEP}`);
|
|
294
|
+
lines.push(' (It re-runs this gate on GitHub for every push, so it turns red if a');
|
|
295
|
+
lines.push(' commit lands that ShipClear would not clear.)');
|
|
234
296
|
} else if (open.length) {
|
|
235
297
|
// A stuck first-timer needs the pointer exactly here, at the verdict.
|
|
236
298
|
lines.push(' Stuck on any of the above? Plain-English help: https://github.com/adamtimmerberg/ShipClear/blob/main/docs/troubleshooting.md');
|
|
@@ -251,12 +313,14 @@ export function renderMarkdown({ findings, notes, verdict, fixes = [], version,
|
|
|
251
313
|
out.push(`## Verdict: ${VERDICT_TEXT[verdict]}`);
|
|
252
314
|
out.push('');
|
|
253
315
|
if (verdict === 'CLEARED') {
|
|
254
|
-
out.push('Cleared!
|
|
316
|
+
out.push('Cleared! To show this in your README, run:');
|
|
255
317
|
out.push('');
|
|
256
|
-
out.push('```
|
|
257
|
-
out.push(
|
|
318
|
+
out.push('```bash');
|
|
319
|
+
out.push(BADGE_NEXT_STEP);
|
|
258
320
|
out.push('```');
|
|
259
321
|
out.push('');
|
|
322
|
+
out.push('That installs a badge backed by a GitHub Actions run of this same gate, so it reflects what is actually on GitHub — green while every pushed commit clears, red as soon as one does not, grey if nothing was checked.');
|
|
323
|
+
out.push('');
|
|
260
324
|
}
|
|
261
325
|
if (open.length === 0) {
|
|
262
326
|
out.push('No findings from the deterministic scan.');
|
|
@@ -305,6 +369,11 @@ export function renderMarkdown({ findings, notes, verdict, fixes = [], version,
|
|
|
305
369
|
out.push('- [ ] **Session and abuse surface:** tokens in httpOnly cookies (not localStorage); rate limiting on login/signup/reset.');
|
|
306
370
|
out.push('- [ ] **Debug surface:** no debug endpoints, stack traces shown to users, or `*` CORS on authenticated APIs.');
|
|
307
371
|
out.push('- [ ] **AI surface** (if the app calls LLMs/agents): prompt inputs treated as untrusted, model output validated before executing/rendering/storing, agent and MCP tools least-privilege.');
|
|
372
|
+
out.push('- [ ] **Credentials actually in use:** every password, key and token in the live config differs from the value in the example/sample config and from the service\'s documented default; the app refuses to start in production if a placeholder remains.');
|
|
373
|
+
out.push('- [ ] **File and object storage:** no bucket or container is publicly readable, and nothing anonymous can **list** its contents. Confirm with a request to the bucket root, not just by reading config.');
|
|
374
|
+
out.push('- [ ] **Proxy and ingress:** no public route forwards wholesale into a backing service (database, object store, queue, admin UI); routes are narrowed to the paths actually needed.');
|
|
375
|
+
out.push('- [ ] **Outbound fetches:** if the server fetches URLs supplied by users (import-from-URL, link previews, webhooks, avatars), internal and cloud-metadata addresses are refused — and re-checked after every redirect.');
|
|
376
|
+
out.push('- [ ] **Header coverage:** security headers apply to the responses that serve HTML, not only to the API.');
|
|
308
377
|
out.push('- [ ] **Verdict update:** append any semantic findings here and restate the verdict (a critical one means DO NOT SHIP).');
|
|
309
378
|
out.push('');
|
|
310
379
|
out.push("Running ShipClear without an AI coding tool? Paste this checklist and your code into any AI chat (ChatGPT, Claude.ai, Gemini — even a free one) and ask it to work through the list — or ask a developer friend. These checks matter, but they need something that can read your code to answer them.");
|
package/core/gate-checklist.md
CHANGED
|
@@ -15,7 +15,14 @@ This is the full specification of what `shipclear ship` verifies (deterministic
|
|
|
15
15
|
| `pii-data-file` | Tracked data files (CSV/TSV/SQL/JSONL) containing real-looking emails or SSNs | high |
|
|
16
16
|
| `test-account` | Hardcoded credential pairs (email + password literals) in source | high |
|
|
17
17
|
| `dependency-vulns` | Known CVEs in dependencies (`npm audit`, high and critical) | high |
|
|
18
|
+
| `credential-in-use-is-public` | The live config (`.env`, compose, k8s, tfvars, Helm values) runs on a value copied from the project's own `.env.example`, a service's documented default, or a leftover placeholder | critical with a production signal, else medium |
|
|
19
|
+
| `public-bucket` | Object storage readable — or **listable** — by anyone (`mc anonymous set download`, `--acl public-read`, wildcard bucket policy, `allUsers`, Azure `container`, Firebase `if true`) | critical when listing or writing is granted, else medium |
|
|
20
|
+
| `proxy-exposes-service` | A public proxy/ingress route forwards wholesale into a backing service (database, queue, object store, admin UI) | critical for databases/queues/admin UIs, high for object storage |
|
|
21
|
+
| `debug-route-public` | A debug or monitoring path (`/metrics`, `/actuator/env`, `/debug/pprof`, `/api/docs`, `/adminer`…) served with no auth check and no production switch | critical when it exposes environment or memory, else medium |
|
|
22
|
+
| `ssrf-unchecked-fetch` | A URL arriving in a request is fetched by the server with no public-address check — or redirects are followed, which steps around one | high |
|
|
18
23
|
| `generic-secret` | Suspicious hardcoded values assigned to names like `apiKey`, `password`, `token` | medium |
|
|
24
|
+
| `secret-default-fallback` | A secret-ish variable has a working default, so a missing value silently becomes a known one (`${FOO_PASSWORD:-…}`, `os.getenv("X", "…")`, `process.env.X \|\| "…"`, settings-class defaults) | medium, critical when that default is the value actually in use |
|
|
25
|
+
| `headers-html-uncovered` | Security headers (CSP and friends) configured for the API only, or described in a proxy config's comments while it sets none — the HTML responses get nothing | medium |
|
|
19
26
|
| `tls-verification-off` | Certificate checking disabled (`rejectUnauthorized: false`, `NODE_TLS_REJECT_UNAUTHORIZED=0`, `verify=False`) | medium |
|
|
20
27
|
| `debug-mode-on` | Framework debug mode left on in shipped code (`app.run(debug=True)`, `FLASK_DEBUG=1`) | medium |
|
|
21
28
|
| `personal-path` | Absolute paths exposing local usernames (`/home/adam/...`, `C:\Users\...`) | low |
|
|
@@ -33,7 +40,12 @@ This is the full specification of what `shipclear ship` verifies (deterministic
|
|
|
33
40
|
7. **Session and abuse surface:** session tokens in httpOnly cookies rather than localStorage; rate limiting (or provider-equivalent) on login, signup, and password-reset endpoints.
|
|
34
41
|
8. **Debug surface:** look for debug endpoints, verbose stack traces returned to users, `console.log` of sensitive values, and permissive CORS on authenticated APIs.
|
|
35
42
|
9. **AI surface** (when the app itself calls LLMs or runs agents/MCP tools): treat user input and any external content entering prompts as untrusted; never execute, render as HTML, or write to the database unvalidated model output; give agent/MCP tools the narrowest permissions that work, and keep agents away from production resources during testing.
|
|
36
|
-
10. **
|
|
43
|
+
10. **Credentials actually in use:** every password, key and token in the live config differs from the value in the example/sample config and from the service's documented default. The app refuses to start in production if a placeholder remains.
|
|
44
|
+
11. **File and object storage:** no bucket or container is publicly readable; nothing anonymous can **list** its contents. Confirm with a request to the bucket root, not just by reading config.
|
|
45
|
+
12. **Proxy and ingress:** no public route forwards wholesale into a backing service (database, object store, queue, admin UI); routes are narrowed to the paths actually needed.
|
|
46
|
+
13. **Outbound fetches:** if the server fetches URLs supplied by users (import-from-URL, link previews, webhooks, avatars), internal and cloud-metadata addresses are refused — and re-checked after every redirect.
|
|
47
|
+
14. **Header coverage:** security headers apply to the responses that serve HTML, not only to the API.
|
|
48
|
+
15. **Report merge:** append semantic findings to `SHIP-REPORT.md` in the same *What / Why / Fix* format, then restate the final verdict — a semantic critical downgrades the overall verdict to DO NOT SHIP.
|
|
37
49
|
|
|
38
50
|
## Verdict rules
|
|
39
51
|
|