specshield 3.5.2 → 3.5.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,176 @@
1
+ 'use strict';
2
+
3
+ /**
4
+ * Passive checks on captured traffic. Reads a HAR file and sends nothing.
5
+ * A capture is full of real secrets, so a finding says where one was seen and
6
+ * never what it was.
7
+ */
8
+
9
+ const fs = require('fs');
10
+ const { finding } = require('./rules');
11
+ const { sensitivity } = require('./diff');
12
+ const { isLocalHost } = require('../prove/http');
13
+ const { templatePath } = require('../har/pathTemplate');
14
+
15
+ const MAX_BODY = 2 * 1024 * 1024;
16
+ const MAX_NODES = 20000;
17
+ const API_MIME = /json|xml|graphql|text\/plain/i;
18
+ const SESSION_COOKIE = /sess|sid|auth|token|jwt/i;
19
+ const MASKED = /^[*xX•.\-\s]*$|^\*+|\*{3,}|x{4,}|redacted|^null$|^undefined$/i;
20
+
21
+ const SECRET_VALUES = [
22
+ ['AWS access key', /\bAKIA[0-9A-Z]{16}\b/],
23
+ ['private key', /-----BEGIN (?:[A-Z]+ )?PRIVATE KEY-----/],
24
+ ['GitHub token', /\b(?:gh[pousr]_[A-Za-z0-9]{36}|github_pat_[A-Za-z0-9_]{40,})\b/],
25
+ ['Stripe live key', /\b[sr]k_live_[A-Za-z0-9]{16,}\b/],
26
+ ['Slack token', /\bxox[abprs]-[A-Za-z0-9-]{10,}\b/],
27
+ ['Google API key', /\bAIza[0-9A-Za-z_-]{35}\b/],
28
+ ];
29
+ const STACK = /Traceback \(most recent call last\)|Exception in thread "|\n\s+at [\w.$<>]+\([\w.]+\.(?:java|kt|scala):\d+\)|\n\s+at .+ \(.+\.[jt]s:\d+:\d+\)|\.php on line \d+/;
30
+
31
+ function auditHar(files, { hosts = null, inventory = null } = {}) {
32
+ const found = new Map();
33
+ const stats = { entries: 0, api: 0, otherHosts: 0, hosts: new Set() };
34
+ const add = (rule, severity, opts) => {
35
+ const key = [rule, opts.endpoint || '', opts.field || ''].join('|');
36
+ if (!found.has(key)) found.set(key, finding(rule, severity, { ...opts, source: 'har' }));
37
+ };
38
+ for (const file of files) {
39
+ for (const e of entriesOf(file)) {
40
+ stats.entries++;
41
+ if (inventory) inventory.add(recordOf(e));
42
+ checkEntry(e, { add, hosts, stats });
43
+ }
44
+ }
45
+ return { findings: [...found.values()], stats: { ...stats, hosts: stats.hosts.size } };
46
+ }
47
+
48
+ function entriesOf(file) {
49
+ let doc;
50
+ try { doc = JSON.parse(fs.readFileSync(file, 'utf8')); } catch (e) { throw new Error(`${file} is not valid HAR JSON (${e.message})`); }
51
+ if (!doc || !doc.log || !Array.isArray(doc.log.entries)) throw new Error(`${file} is not a HAR file (no log.entries)`);
52
+ return doc.log.entries.filter((e) => e && e.request && e.request.url && e.response);
53
+ }
54
+
55
+ /** The same shape the access-log parsers produce, plus the response type. */
56
+ function recordOf(e) {
57
+ let url;
58
+ try { url = new URL(e.request.url); } catch { return null; }
59
+ const time = Date.parse(e.startedDateTime);
60
+ return {
61
+ method: String(e.request.method || 'GET').toUpperCase(),
62
+ path: url.pathname,
63
+ query: url.search.replace(/^\?/, ''),
64
+ host: url.hostname.toLowerCase(),
65
+ status: e.response.status > 0 ? e.response.status : null,
66
+ time: Number.isNaN(time) ? null : time,
67
+ userAgent: ((e.request.headers || []).find((h) => h && /^user-agent$/i.test(h.name)) || {}).value || null,
68
+ mime: (e.response.content && e.response.content.mimeType) || null,
69
+ bytes: e.response.content && e.response.content.size >= 0 ? e.response.content.size : null,
70
+ };
71
+ }
72
+
73
+ function headerMap(list) {
74
+ const map = new Map();
75
+ for (const h of list || []) {
76
+ if (!h || typeof h.name !== 'string') continue;
77
+ const k = h.name.toLowerCase();
78
+ map.set(k, [...(map.get(k) || []), String(h.value)]);
79
+ }
80
+ return map;
81
+ }
82
+
83
+ function bodyOf(res) {
84
+ const c = res.content;
85
+ if (!c || typeof c.text !== 'string') return null;
86
+ const text = c.encoding === 'base64' ? Buffer.from(c.text.slice(0, MAX_BODY * 2), 'base64').toString('utf8') : c.text;
87
+ return text.slice(0, MAX_BODY);
88
+ }
89
+
90
+ function checkEntry(e, { add, hosts, stats }) {
91
+ let url;
92
+ try { url = new URL(e.request.url); } catch { return; }
93
+ if (!/^https?:$/.test(url.protocol)) return;
94
+ const host = url.hostname.toLowerCase();
95
+ if (hosts && !hosts.has(host)) { stats.otherHosts++; return; }
96
+ const res = headerMap(e.response.headers);
97
+ const mime = String((e.response.content && e.response.content.mimeType) || (res.get('content-type') || [''])[0]);
98
+ // A browser capture is mostly pages and assets; these checks are about the API.
99
+ if (!API_MIME.test(mime)) return;
100
+ stats.api++;
101
+ stats.hosts.add(host);
102
+
103
+ const endpoint = `${String(e.request.method || 'GET').toUpperCase()} ${templatePath(url.pathname).templated}`;
104
+ const local = isLocalHost(host);
105
+ const https = url.protocol === 'https:';
106
+
107
+ if (!https && !local) add('cleartext-traffic', 'high', { field: host });
108
+ if (https && !local) {
109
+ if (!res.has('strict-transport-security')) add('missing-hsts', 'medium', { field: host });
110
+ }
111
+ if (!res.has('x-content-type-options')) add('missing-nosniff', 'low', { field: host });
112
+
113
+ for (const [name, value] of url.searchParams) {
114
+ const tier = sensitivity(name, { type: 'string' });
115
+ if (value && (tier === 'credential' || tier === 'regulated' || tier === 'token')) add('secret-in-url', 'high', { endpoint, field: name });
116
+ const hit = SECRET_VALUES.find(([, re]) => re.test(value));
117
+ if (hit) add('secret-in-url', 'high', { endpoint, field: name, detail: hit[0] });
118
+ }
119
+
120
+ checkCors(res, { add, host });
121
+ checkCookies(res, { add, host, https, local });
122
+ for (const h of ['server', 'x-powered-by', 'x-aspnet-version']) {
123
+ if ((res.get(h) || []).some((v) => /\d+\.\d+/.test(v))) add('version-disclosure', 'low', { field: host, detail: h });
124
+ }
125
+
126
+ const body = bodyOf(e.response);
127
+ if (!body) return;
128
+ if (e.response.status >= 400 && STACK.test(body)) add('stack-trace', 'medium', { endpoint });
129
+ for (const [label, re] of SECRET_VALUES) if (re.test(body)) add('secret-in-response', 'high', { endpoint, field: label });
130
+ if (/^\s*[[{]/.test(body)) {
131
+ try { scanJson(JSON.parse(body), { add, endpoint }); } catch { /* truncated or not JSON after all */ }
132
+ }
133
+ }
134
+
135
+ function checkCors(res, { add, host }) {
136
+ const origin = (res.get('access-control-allow-origin') || [])[0];
137
+ if (!origin) return;
138
+ const credentials = (res.get('access-control-allow-credentials') || [])[0] === 'true';
139
+ if (origin === '*' && credentials) add('cors-wildcard-credentials', 'high', { field: host });
140
+ else if (origin === '*') add('cors-any-origin', 'low', { field: host });
141
+ else if (origin === 'null') add('cors-null-origin', 'medium', { field: host });
142
+ }
143
+
144
+ function checkCookies(res, { add, host, https, local }) {
145
+ for (const raw of res.get('set-cookie') || []) {
146
+ const [pair, ...attrs] = raw.split(';');
147
+ const name = pair.split('=')[0].trim();
148
+ if (!SESSION_COOKIE.test(name)) continue;
149
+ const has = (a) => attrs.some((x) => x.trim().toLowerCase().startsWith(a));
150
+ const missing = [
151
+ https && !local && !has('secure') ? 'Secure' : null,
152
+ !has('httponly') ? 'HttpOnly' : null,
153
+ !has('samesite') ? 'SameSite' : null,
154
+ ].filter(Boolean);
155
+ if (missing.length) add('insecure-cookie', 'medium', { field: name, detail: `${host}: no ${missing.join(', ')}` });
156
+ }
157
+ }
158
+
159
+ function scanJson(root, { add, endpoint }) {
160
+ let nodes = 0;
161
+ const visit = (v, key) => {
162
+ if (++nodes > MAX_NODES || v === null || v === undefined) return;
163
+ if (Array.isArray(v)) { for (const x of v) visit(x, key); return; }
164
+ if (typeof v === 'object') { for (const [k, x] of Object.entries(v)) visit(x, k); return; }
165
+ if (typeof v !== 'string' && typeof v !== 'number') return;
166
+ const text = String(v);
167
+ if (!key || !text || MASKED.test(text)) return;
168
+ const tier = sensitivity(key, { type: 'string' });
169
+ if (tier === 'credential') add('secret-in-response', 'high', { endpoint, field: key });
170
+ else if (tier === 'regulated') add('regulated-data-in-response', 'high', { endpoint, field: key });
171
+ else if (tier === 'personal') add('personal-data-in-response', 'low', { endpoint, field: key });
172
+ };
173
+ visit(root, null);
174
+ }
175
+
176
+ module.exports = { auditHar };
@@ -41,7 +41,7 @@ function operations(doc) {
41
41
  for (const method of METHODS) {
42
42
  if (!item[method] || typeof item[method] !== 'object') continue;
43
43
  // A renamed path parameter is the same operation.
44
- ops.set(`${method} ${path.replace(/\{[^}]+\}/g, '{}')}`, { path, method, op: item[method] });
44
+ ops.set(`${method} ${path.replace(/\{[^}]+\}/g, '{}')}`, { path, method, op: item[method], item });
45
45
  }
46
46
  }
47
47
  return ops;
@@ -152,6 +152,7 @@ function addedEndpoint(h, req, doc) {
152
152
 
153
153
  /** Specs that declare no auth anywhere say nothing about which endpoints are public. */
154
154
  function declaresAuth(doc) {
155
+ if ((doc.security || []).some((a) => a && Object.keys(a).length)) return true;
155
156
  return [...operations(doc).values()].some((o) => !allowsAnonymous(requirement(doc, o.op)));
156
157
  }
157
158
 
@@ -220,22 +221,23 @@ const leaf = (p) => p.split('.').pop().replace(/\[\]$/, '');
220
221
  /** Sensitive fields a 2xx response can carry, with their paths. */
221
222
  function sensitiveResponseFields(doc, op) {
222
223
  const out = [];
223
- const memo = memoFor(doc);
224
224
  for (const [status, raw] of Object.entries(op.responses || {})) {
225
225
  if (!/^2/.test(status)) continue;
226
226
  const res = deref(doc, raw);
227
227
  if (!res) continue;
228
228
  const schemas = res.schema ? [res.schema] : Object.entries(res.content || {})
229
229
  .filter(([type]) => /json/i.test(type)).map(([, m]) => m && m.schema).filter(Boolean);
230
- for (const s of schemas) out.push(...walk(doc, s, memo, new Set(), 0));
230
+ for (const s of schemas) out.push(...fields(doc, s, sensitivity, 'writeOnly'));
231
231
  }
232
232
  return out;
233
233
  }
234
234
 
235
235
  const memos = new WeakMap();
236
- function memoFor(doc) {
236
+ function memoFor(doc, classify) {
237
237
  if (!memos.has(doc)) memos.set(doc, new Map());
238
- return memos.get(doc);
238
+ const byClassifier = memos.get(doc);
239
+ if (!byClassifier.has(classify)) byClassifier.set(classify, new Map());
240
+ return byClassifier.get(classify);
239
241
  }
240
242
 
241
243
  function deref(doc, node) {
@@ -247,32 +249,38 @@ function deref(doc, node) {
247
249
  return n && typeof n === 'object' ? n : null;
248
250
  }
249
251
 
250
- /** [{path, tier}] relative to this schema. Named schemas are memoized; cycles are cut. */
251
- function walk(doc, node, memo, stack, depth) {
252
+ /** [{path, tier}] for each property `classify(name, schema)` tags; `skip` names a flag that hides a property. */
253
+ function fields(doc, schema, classify, skip) {
254
+ return walk({ doc, classify, skip, memo: memoFor(doc, classify), stack: new Set() }, schema, 0);
255
+ }
256
+
257
+ /** Named schemas are memoized; cycles are cut. */
258
+ function walk(ctx, node, depth) {
252
259
  if (!node || depth > MAX_DEPTH) return [];
260
+ const { doc, memo, stack } = ctx;
253
261
  if (node.$ref) {
254
262
  if (memo.has(node.$ref)) return memo.get(node.$ref);
255
263
  if (stack.has(node.$ref)) return [];
256
264
  stack.add(node.$ref);
257
- const r = walk(doc, deref(doc, node), memo, stack, depth + 1);
265
+ const r = walk(ctx, deref(doc, node), depth + 1);
258
266
  stack.delete(node.$ref);
259
267
  memo.set(node.$ref, r);
260
268
  return r;
261
269
  }
262
270
  const out = [];
263
271
  for (const key of ['allOf', 'oneOf', 'anyOf']) {
264
- if (Array.isArray(node[key])) for (const s of node[key]) out.push(...walk(doc, s, memo, stack, depth + 1));
272
+ if (Array.isArray(node[key])) for (const s of node[key]) out.push(...walk(ctx, s, depth + 1));
265
273
  }
266
- if (node.items) out.push(...walk(doc, node.items, memo, stack, depth + 1).map((f) => ({ ...f, path: `[].${f.path}` })));
274
+ if (node.items) out.push(...walk(ctx, node.items, depth + 1).map((f) => ({ ...f, path: `[].${f.path}` })));
267
275
  if (node.additionalProperties && typeof node.additionalProperties === 'object') {
268
- out.push(...walk(doc, node.additionalProperties, memo, stack, depth + 1));
276
+ out.push(...walk(ctx, node.additionalProperties, depth + 1));
269
277
  }
270
278
  for (const [name, raw] of Object.entries(node.properties || {})) {
271
279
  const prop = deref(doc, raw);
272
- if (!prop || prop.writeOnly) continue;
273
- const tier = sensitivity(name, prop);
280
+ if (!prop || prop[ctx.skip]) continue;
281
+ const tier = ctx.classify(name, prop);
274
282
  if (tier) out.push({ path: name, tier });
275
- out.push(...walk(doc, raw, memo, stack, depth + 1).map((f) => ({ ...f, path: `${name}.${f.path}` })));
283
+ out.push(...walk(ctx, raw, depth + 1).map((f) => ({ ...f, path: `${name}.${f.path}` })));
276
284
  }
277
285
  return out.map((f) => ({ ...f, path: f.path.replace(/^\[\]\./, '') }));
278
286
  }
@@ -301,4 +309,7 @@ function merge(found) {
301
309
  return out;
302
310
  }
303
311
 
304
- module.exports = { securityDiff, requirement, sensitivity };
312
+ module.exports = {
313
+ securityDiff, requirement, sensitivity, operations, allowsAnonymous, declaresAuth,
314
+ schemeOf, deref, fields, sensitiveResponseFields, leaf, words,
315
+ };
@@ -0,0 +1,367 @@
1
+ 'use strict';
2
+
3
+ /**
4
+ * Improper inventory (OWASP API9) from real traffic: what the API serves that
5
+ * the spec doesn't say.
6
+ *
7
+ * shadow-endpoint a path the spec doesn't have, answering requests
8
+ * undocumented-method a documented path, answering a method the spec doesn't list
9
+ * zombie-version /v1/... still answering, where the spec documents /v2/...
10
+ * sunset-passed an operation past its x-sunset date, still answering
11
+ * exposed-sensitive-path /.env, /.git/config, /actuator/env... answering 200
12
+ *
13
+ * The response status decides how sure a finding is: a 2xx (or 304) means the
14
+ * endpoint served the request; 401/403/400 mean it exists; 404/405 and
15
+ * redirects say nothing.
16
+ * A host that answers 200 to scanner probes is most likely a catch-all (a
17
+ * single-page app's index.html), so its findings are downgraded, not trusted.
18
+ */
19
+
20
+ const { createRouteMatcher } = require('../usage/routeMatcher');
21
+ const { templatePath } = require('../har/pathTemplate');
22
+ const { finding } = require('./rules');
23
+
24
+ const MAX_ENTRIES = 2000;
25
+ const MAX_CLIENTS = 100;
26
+ const SKIP_METHODS = new Set(['HEAD', 'OPTIONS']);
27
+ const WRITE = new Set(['POST', 'PUT', 'PATCH', 'DELETE']);
28
+ const STATIC = /\.(m?js|css|map|png|jpe?g|gif|svg|ico|webp|avif|woff2?|ttf|eot|html?|txt|webmanifest)$/i;
29
+ const PRIVILEGED_PATH = /(^|\/)(admin|internal|debug|actuator|management|private|_[a-z]+)(\/|$)/i;
30
+ const VERSION = /^v\d+(\.\d+)?$/i;
31
+ // Every API has these, and few specs list them: worth knowing about, not alarming.
32
+ const CONVENTIONAL = /^\/(health|healthz|livez|readyz|ping|status|version|openapi|swagger|api-docs|v\d+\/api-docs|\.well-known)(\/|$|\.json$|\.yaml$)/i;
33
+ // A position with this many distinct values is a parameter, whatever the values look like.
34
+ const DISTINCT = 4;
35
+
36
+ // Paths no production server should answer with content.
37
+ const PROBES = [
38
+ ['environment file', /^\/(.*\/)?\.env(\.[\w-]+)?$/i],
39
+ ['git repository', /^\/(.*\/)?\.git\/(config|HEAD|index)$/i],
40
+ ['cloud credentials', /^\/(.*\/)?\.aws\/credentials$/i],
41
+ ['Spring Boot actuator', /^\/(.*\/)?actuator\/(env|heapdump|threaddump|configprops|beans|mappings|loggers|jolokia|shutdown)$/i],
42
+ ['heap dump', /^\/(.*\/)?heapdump$/i],
43
+ ['Go debug endpoint', /^\/debug\/(pprof|vars)(\/.*)?$/i],
44
+ ['server status page', /^\/server-(status|info)$/i],
45
+ ['PHP info page', /^\/(.*\/)?phpinfo\.php$/i],
46
+ ['database backup', /^\/(.*\/)?[\w-]*(backup|dump|db)[\w-]*\.(sql|sql\.gz|bak)$/i],
47
+ ];
48
+ // This many probe paths answered with the same page means the host answers everything with it.
49
+ const CATCH_ALL = 3;
50
+ const MAX_SIZES = 50;
51
+
52
+ function createInventory(spec, { basePath, hosts = null, now = Date.now() } = {}) {
53
+ const matcher = spec ? createRouteMatcher(spec, { basePath }) : null;
54
+ const versions = spec ? knownVersions(spec, matcher) : [];
55
+ const entries = new Map();
56
+ const probes = new Map();
57
+ const probeHits = new Map(); // site → probe path → probe answered 2xx
58
+ const home = new Map(); // site → size buckets of its home page
59
+ const documented = new Map();
60
+ const stats = { requests: 0, considered: 0, outside: 0, static: 0, otherHosts: 0, overflow: 0 };
61
+
62
+ function add(rec) {
63
+ if (!rec || !rec.method || typeof rec.path !== 'string') return;
64
+ stats.requests++;
65
+ const method = String(rec.method).toUpperCase();
66
+ const host = rec.host ? String(rec.host).toLowerCase() : null;
67
+ if (hosts && host && !hosts.has(host)) { stats.otherHosts++; return; }
68
+ // Per-site logs record no host; the file they came from stands in, so one
69
+ // site's catch-all page doesn't cast doubt on another's findings.
70
+ const site = host || (rec.site ? String(rec.site) : null);
71
+ if (SKIP_METHODS.has(method)) return;
72
+ let path = rec.path.replace(/\/{2,}/g, '/');
73
+ if (path.length > 1) path = path.replace(/\/$/, '');
74
+ // A capture tells us what came back; an HTML page is the app, not the API.
75
+ const html = rec.mime ? /html/i.test(rec.mime) : false;
76
+
77
+ // Single-page apps and CMS rewrites answer unknown paths with the home page.
78
+ if (path === '/' && method === 'GET' && ok(rec.status) && typeof rec.bytes === 'number') {
79
+ if (!home.has(site)) home.set(site, new Set());
80
+ const sizes = home.get(site);
81
+ const b = bucket(rec.bytes);
82
+ if (sizes.size < MAX_SIZES) [b - 1, b, b + 1].forEach((x) => sizes.add(x));
83
+ }
84
+
85
+ const probe = PROBES.find(([, re]) => re.test(path));
86
+ if (probe) {
87
+ // "/.env/" is a directory request: whatever answered it, it wasn't the file.
88
+ const asDirectory = (rec.trailingSlash || (rec.path.length > 1 && rec.path.endsWith('/'))) && !/^\/debug\//.test(path);
89
+ if (ok(rec.status) && !html && !asDirectory) {
90
+ const key = `${site || ''} ${path}`;
91
+ if (!probes.has(key)) probes.set(key, { label: probe[0], path, host: site, calls: 0, last: null, sizes: [] });
92
+ const p = probes.get(key);
93
+ p.calls++;
94
+ p.last = latest(p.last, rec.time);
95
+ if (typeof rec.bytes === 'number' && p.sizes.length < MAX_SIZES) p.sizes.push(rec.bytes);
96
+ if (!probeHits.has(site)) probeHits.set(site, new Map());
97
+ probeHits.get(site).set(path, p);
98
+ }
99
+ return;
100
+ }
101
+ if (!matcher) return;
102
+ if (path === '/' || STATIC.test(path)) { stats.static++; return; }
103
+ if (html) return;
104
+ stats.considered++;
105
+
106
+ const m = matcher.match(method, path) || greedy(method, path);
107
+ if (m && m.operation) { record(documented, `${method} ${m.template}`, { method, template: m.template }, rec, site); return; }
108
+ if (m) { remember('undocumented-method', method, m.template, rec, site); return; }
109
+ const older = versions.length ? otherVersion(method, path) : null;
110
+ if (older) { remember('zombie-version', method, templatePath(path).templated, rec, site, older); return; }
111
+ // Named like the spec names its paths, so a waiver for it reads the same.
112
+ const base = matcher.basePaths.find((b) => path === b || path.startsWith(`${b}/`));
113
+ if (matcher.basePaths.length && !base) { stats.outside++; return; }
114
+ remember('shadow-endpoint', method, templatePath(base ? path.slice(base.length) || '/' : path).templated, rec, site);
115
+ }
116
+
117
+ /**
118
+ * Kubernetes and others document "/proxy/{path}" where {path} spans several
119
+ * segments. OpenAPI doesn't say a parameter can't hold a slash, so try the
120
+ * path with its tail folded into the last segment.
121
+ */
122
+ function greedy(method, path) {
123
+ const segs = path.split('/');
124
+ for (let n = segs.length - 1; n > 1; n--) {
125
+ const m = matcher.match(method, [...segs.slice(0, n), segs.slice(n).join('%2F')].join('/'));
126
+ if (m && /\/\{[^}/]+\}$/.test(m.template)) return m;
127
+ }
128
+ return null;
129
+ }
130
+
131
+ /** /v1/orders when the spec serves /v2/orders: the same operation under a version the spec dropped. */
132
+ function otherVersion(method, path) {
133
+ const segs = path.split('/');
134
+ for (let i = 0; i < segs.length; i++) {
135
+ if (!VERSION.test(segs[i]) || versions.includes(segs[i].toLowerCase())) continue;
136
+ for (const v of versions) {
137
+ const swapped = [...segs.slice(0, i), v, ...segs.slice(i + 1)].join('/');
138
+ const m = matcher.match(method, swapped);
139
+ if (m && m.operation) return `${method} ${m.template} (${v})`;
140
+ }
141
+ }
142
+ return null;
143
+ }
144
+
145
+ function remember(kind, method, template, rec, host, current = null) {
146
+ const key = `${kind}|${method} ${template}`;
147
+ if (!entries.has(key) && entries.size >= MAX_ENTRIES) { stats.overflow++; return; }
148
+ record(entries, key, { kind, method, template, current, fixed: kind === 'shadow-endpoint' && CONVENTIONAL.test(template) }, rec, host);
149
+ }
150
+
151
+ /**
152
+ * Whether a probe's 2xx was the site's catch-all page rather than the file.
153
+ * With sizes: other probe paths got a response of the same size, so it isn't
154
+ * this file's content. Comparing per probe copes with a site that has several
155
+ * such pages, or whose page changed. Only probes count: an ordinary API
156
+ * response of a similar size says nothing. Without sizes: the site answered
157
+ * 2xx to several probes.
158
+ */
159
+ function catchAllAnswer(p) {
160
+ const size = median(p.sizes);
161
+ const siblings = [...probeHits.get(p.host).values()].filter((q) => q !== p);
162
+ if (size === null) {
163
+ const n = siblings.length + 1;
164
+ return n >= CATCH_ALL ? `this host answers 200 to ${n} probe paths, so it is probably a catch-all page` : null;
165
+ }
166
+ const b = bucket(size);
167
+ if ((home.get(p.host) || new Set()).has(b)) return `${size} bytes, the size of this host's home page; probably a rewrite to it`;
168
+ const same = siblings.filter((q) => { const m = median(q.sizes); return m !== null && Math.abs(bucket(m) - b) <= 1; });
169
+ if (same.length + 1 < CATCH_ALL) return null;
170
+ return `${size} bytes, the same as ${same.length} other probe paths got (e.g. ${same[0].path}); probably a catch-all page`;
171
+ }
172
+
173
+ function findings() {
174
+ const verdicts = new Map([...probes.values()].map((p) => [p, catchAllAnswer(p)]));
175
+ const catchAll = new Set([...verdicts].filter(([, v]) => v).map(([p]) => p.host));
176
+ const out = [];
177
+ for (const e of collapse([...entries.values()])) {
178
+ const f = judge(e, catchAll);
179
+ if (f) out.push(f);
180
+ }
181
+ for (const op of sunsetPassed(spec, documented, now)) out.push(op);
182
+ for (const [p, why] of verdicts) {
183
+ const size = median(p.sizes);
184
+ const detail = why ? `${p.label}; ${why}` : `${p.label}, answered 2xx ${times(p.calls)}${size !== null ? ` with ${size} bytes no other path got` : ''}`;
185
+ out.push(finding('exposed-sensitive-path', why ? 'low' : 'high', { endpoint: `GET ${p.path}`, field: p.host, detail, source: 'traffic' }));
186
+ }
187
+ return out;
188
+ }
189
+
190
+ return {
191
+ add,
192
+ findings,
193
+ get stats() {
194
+ const real = [...entries.values()].filter((e) => count(e, (st) => st !== 404 && st !== 405) > 0).length;
195
+ return { ...stats, documentedSeen: documented.size, documentedTotal: matcher ? matcher.operations.length : 0, endpoints: real };
196
+ },
197
+ };
198
+ }
199
+
200
+ /**
201
+ * Folds routes that differ only where a value varies: /things/noxus/x and
202
+ * /things/fooab/x become /things/{thingId}/x once DISTINCT siblings share both
203
+ * a parent and the shape of what follows them. Object names that don't look
204
+ * like IDs would otherwise each be a finding, and fixed words (/readyz,
205
+ * /metrics, /openapi) differ in what follows, so they stay apart. The first
206
+ * segment is never folded.
207
+ */
208
+ function collapse(list) {
209
+ let routes = list.map((e) => ({ ...e, segs: e.template.split('/'), variants: e.variants || 1, examples: e.examples || [] }));
210
+ const depth = Math.max(0, ...routes.map((r) => r.segs.length));
211
+ const isParam = (v) => v.startsWith('{');
212
+ for (let i = 2; i < depth; i++) {
213
+ const familyOf = (r) => [r.kind, r.method, r.segs.slice(0, i).join('/')].join('|');
214
+ const shapes = new Map(); // family → value → shapes of what follows
215
+ for (const r of routes) {
216
+ if (r.segs.length <= i || isParam(r.segs[i])) continue;
217
+ const fam = familyOf(r);
218
+ if (!shapes.has(fam)) shapes.set(fam, new Map());
219
+ const byValue = shapes.get(fam);
220
+ if (!byValue.has(r.segs[i])) byValue.set(r.segs[i], new Set());
221
+ byValue.get(r.segs[i]).add(r.segs.slice(i + 1).map((x) => (isParam(x) ? '{}' : '*')).join('/'));
222
+ }
223
+ const fold = new Set();
224
+ for (const [fam, byValue] of shapes) {
225
+ const bySignature = new Map();
226
+ for (const [value, set] of byValue) {
227
+ const sig = [...set].sort().join('|');
228
+ if (!bySignature.has(sig)) bySignature.set(sig, []);
229
+ bySignature.get(sig).push(value);
230
+ }
231
+ for (const values of bySignature.values()) {
232
+ if (values.length >= DISTINCT) values.forEach((v) => fold.add(`${fam}|${v}`));
233
+ }
234
+ }
235
+ if (!fold.size) continue;
236
+ const merged = new Map();
237
+ for (const r of routes) {
238
+ const folded = r.segs.length > i && fold.has(`${familyOf(r)}|${r.segs[i]}`);
239
+ const segs = folded ? [...r.segs.slice(0, i), `{${paramName(r.segs[i - 1])}}`, ...r.segs.slice(i + 1)] : r.segs;
240
+ const key = [r.kind, r.method, segs.join('/')].join('|');
241
+ const into = merged.get(key);
242
+ const examples = folded && !r.examples.length ? [r.template] : r.examples;
243
+ if (!into) {
244
+ merged.set(key, { ...r, segs, examples: [...examples], statuses: new Map(r.statuses), clients: new Set(r.clients), hosts: new Set(r.hosts) });
245
+ continue;
246
+ }
247
+ into.calls += r.calls;
248
+ into.variants += r.variants;
249
+ for (const x of examples) if (into.examples.length < 3) into.examples.push(x);
250
+ for (const [st, c] of r.statuses) into.statuses.set(st, (into.statuses.get(st) || 0) + c);
251
+ for (const c of r.clients) if (into.clients.size < MAX_CLIENTS) into.clients.add(c);
252
+ for (const h of r.hosts) into.hosts.add(h);
253
+ into.last = latest(into.last, r.last);
254
+ into.lastOk = latest(into.lastOk, r.lastOk) ?? into.lastOk ?? r.lastOk;
255
+ }
256
+ routes = [...merged.values()];
257
+ }
258
+ return routes.map((r) => ({ ...r, template: unique(r.segs).join('/') }));
259
+ }
260
+
261
+ /** {id}/{id} → {id}/{id2}: a template can't repeat a parameter name. */
262
+ function unique(segs) {
263
+ const seen = new Map();
264
+ return segs.map((s) => {
265
+ const m = /^\{(.+)\}$/.exec(s);
266
+ if (!m) return s;
267
+ const base = m[1].replace(/\d+$/, '');
268
+ const n = (seen.get(base) || 0) + 1;
269
+ seen.set(base, n);
270
+ return n === 1 ? `{${base}}` : `{${base}${n}}`;
271
+ });
272
+ }
273
+
274
+ function paramName(prev) {
275
+ const m = /^([a-z][a-z0-9]*?)(e?s)?$/i.exec(prev || '');
276
+ return m && m[1].length > 1 && !prev.startsWith('{') ? `${m[1]}Id` : 'id';
277
+ }
278
+
279
+ const median = (xs) => (xs.length ? [...xs].sort((a, b) => a - b)[Math.floor(xs.length / 2)] : null);
280
+ // Buckets about 3% wide: a catch-all page that echoes the path varies by a few bytes.
281
+ const bucket = (bytes) => Math.round(Math.log(bytes + 64) / Math.log(1.03));
282
+
283
+ function record(map, key, base, rec, host) {
284
+ if (!map.has(key)) map.set(key, { ...base, calls: 0, statuses: new Map(), clients: new Set(), hosts: new Set(), last: null, lastOk: null });
285
+ const e = map.get(key);
286
+ e.calls++;
287
+ const s = rec.status || null;
288
+ e.statuses.set(s, (e.statuses.get(s) || 0) + 1);
289
+ if (rec.userAgent && e.clients.size < MAX_CLIENTS) e.clients.add(rec.userAgent);
290
+ if (host) e.hosts.add(host);
291
+ e.last = latest(e.last, rec.time);
292
+ if (ok(s)) e.lastOk = latest(e.lastOk, rec.time) ?? e.lastOk ?? true;
293
+ }
294
+
295
+ // 304 means the client's copy of an existing resource is current. Other 3xx
296
+ // prove nothing: an HTTP→HTTPS redirect answers every path, real or not.
297
+ const ok = (s) => (s >= 200 && s < 300) || s === 304;
298
+ const latest = (a, t) => (typeof t === 'number' && (typeof a !== 'number' || t > a) ? t : a);
299
+ const times = (n) => (n === 1 ? 'once' : `${n} times`);
300
+
301
+ function count(e, test) {
302
+ let n = 0;
303
+ for (const [s, c] of e.statuses) if (test(s)) n += c;
304
+ return n;
305
+ }
306
+
307
+ function judge(e, catchAll) {
308
+ const served = count(e, ok);
309
+ const exists = count(e, (s) => s === 401 || s === 403 || s === 400 || s === 422 || s === 429 || (s >= 500 && s < 600));
310
+ const unknown = count(e, (s) => s === null);
311
+ if (!served && !exists && !unknown) return null; // only 404/405: it isn't there
312
+ const write = WRITE.has(e.method);
313
+ const doubtful = [...e.hosts].some((h) => catchAll.has(h));
314
+ const conventional = Boolean(e.fixed);
315
+ let severity = 'low';
316
+ if (served && !doubtful && !conventional) {
317
+ if (e.kind === 'shadow-endpoint') severity = write || PRIVILEGED_PATH.test(e.template) ? 'high' : 'medium';
318
+ else severity = write ? 'high' : 'medium';
319
+ }
320
+ const notes = [describe(e)];
321
+ if (e.variants > 1) notes.push(`${e.variants} routes folded into this one, e.g. ${e.examples.join(', ')}`);
322
+ if (e.current) notes.push(`now ${e.current}`);
323
+ if (conventional) notes.push('a conventional health or discovery endpoint');
324
+ if (!served && unknown === e.calls) notes.push('the logs have no status, so it may not exist');
325
+ else if (!served) notes.push('never answered 2xx, but it exists');
326
+ if (doubtful) notes.push('this host looks like a catch-all');
327
+ return finding(e.kind, severity, { endpoint: `${e.method} ${e.template}`, detail: notes.join('; '), source: 'traffic' });
328
+ }
329
+
330
+ function describe(e) {
331
+ const statuses = [...e.statuses].sort((a, b) => b[1] - a[1]).slice(0, 4).map(([s, n]) => `${s ?? '?'}×${n}`).join(' ');
332
+ const last = typeof e.last === 'number' ? `, last ${new Date(e.last).toISOString().slice(0, 10)}` : '';
333
+ return `${e.calls} ${e.calls === 1 ? 'call' : 'calls'} from ${e.clients.size || '?'} ${e.clients.size === 1 ? 'client' : 'clients'} (${statuses})${last}`;
334
+ }
335
+
336
+ /** Operations whose x-sunset date has passed and that still answered 2xx after it. */
337
+ function sunsetPassed(spec, documented, now) {
338
+ const out = [];
339
+ if (!spec) return out;
340
+ for (const [path, item] of Object.entries(spec.paths || {})) {
341
+ if (!item || typeof item !== 'object') continue;
342
+ for (const [method, op] of Object.entries(item)) {
343
+ if (!op || typeof op !== 'object' || !op['x-sunset']) continue;
344
+ const v = op['x-sunset'];
345
+ const sunset = (v instanceof Date ? v : new Date(String(v))).getTime();
346
+ if (Number.isNaN(sunset) || sunset > now) continue;
347
+ const e = documented.get(`${method.toUpperCase()} ${path}`);
348
+ if (!e || !e.lastOk || (typeof e.lastOk === 'number' && e.lastOk < sunset)) continue;
349
+ out.push(finding('sunset-passed', 'medium', {
350
+ endpoint: `${method.toUpperCase()} ${path}`,
351
+ detail: `sunset ${new Date(sunset).toISOString().slice(0, 10)}; ${describe(e)}`,
352
+ source: 'traffic',
353
+ }));
354
+ }
355
+ }
356
+ return out;
357
+ }
358
+
359
+ function knownVersions(spec, matcher) {
360
+ const found = new Set();
361
+ const scan = (p) => p.split('/').forEach((s) => { if (VERSION.test(s)) found.add(s.toLowerCase()); });
362
+ for (const p of Object.keys(spec.paths || {})) scan(p);
363
+ matcher.basePaths.forEach(scan);
364
+ return [...found];
365
+ }
366
+
367
+ module.exports = { createInventory, PROBES };