specshield 3.4.10 → 3.4.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,127 @@
1
+ 'use strict';
2
+
3
+ /**
4
+ * Maps a concrete request path from a log ("/v1/orders/123/receipt") onto the
5
+ * OpenAPI path template that serves it ("/orders/{orderId}/receipt").
6
+ *
7
+ * Handles the two things that make naive string matching fail:
8
+ * - base paths: spec paths are relative to servers[].url (OpenAPI 3) or
9
+ * basePath (Swagger 2), while logs record the full path;
10
+ * - overlapping templates: /users/me and /users/{id} both match "/users/me",
11
+ * and the one with more literal segments wins, as in most routers.
12
+ *
13
+ * Paths that are already templated (usage.json written without --spec stores
14
+ * "/orders/{orderId}") also match: a "{…}" segment matches any template
15
+ * parameter, never a literal.
16
+ */
17
+
18
+ const HTTP_METHODS = ['get', 'put', 'post', 'delete', 'options', 'head', 'patch', 'trace'];
19
+
20
+ function createRouteMatcher(spec, opts = {}) {
21
+ const routes = [];
22
+ for (const [template, item] of Object.entries((spec && spec.paths) || {})) {
23
+ if (!item || typeof item !== 'object') continue;
24
+ const methods = new Set(HTTP_METHODS.filter((m) => item[m]).map((m) => m.toUpperCase()));
25
+ if (methods.size === 0) continue;
26
+ const segs = splitPath(template);
27
+ routes.push({
28
+ template,
29
+ methods,
30
+ matchers: segs.map(segmentMatcher),
31
+ literals: segs.filter((s) => !s.includes('{')).length,
32
+ });
33
+ }
34
+
35
+ const bySegCount = new Map();
36
+ for (const r of routes) {
37
+ const n = r.matchers.length;
38
+ if (!bySegCount.has(n)) bySegCount.set(n, []);
39
+ bySegCount.get(n).push(r);
40
+ }
41
+ for (const list of bySegCount.values()) list.sort((a, b) => b.literals - a.literals);
42
+
43
+ const basePaths = resolveBasePaths(spec, opts.basePath);
44
+
45
+ /**
46
+ * @returns {{ template, method, operation: boolean } | null}
47
+ * operation=false when the path exists but the spec doesn't define that method.
48
+ */
49
+ function match(method, path) {
50
+ const upper = String(method).toUpperCase();
51
+ for (const prefix of basePaths) {
52
+ let rel;
53
+ if (prefix === '') rel = path;
54
+ else if (path === prefix) rel = '/';
55
+ else if (path.startsWith(prefix + '/')) rel = path.slice(prefix.length);
56
+ else continue;
57
+
58
+ const segs = splitPath(rel);
59
+ const candidates = bySegCount.get(segs.length) || [];
60
+ let pathOnly = null;
61
+ for (const r of candidates) {
62
+ if (!r.matchers.every((test, i) => test(segs[i]))) continue;
63
+ if (r.methods.has(upper)) return { template: r.template, method: upper, operation: true };
64
+ if (!pathOnly) pathOnly = { template: r.template, method: upper, operation: false };
65
+ }
66
+ if (pathOnly) return pathOnly;
67
+ }
68
+ return null;
69
+ }
70
+
71
+ return {
72
+ match,
73
+ basePaths: basePaths.filter(Boolean),
74
+ operations: routes.flatMap((r) => [...r.methods].map((m) => ({ method: m, template: r.template }))),
75
+ };
76
+ }
77
+
78
+ /** Longest base path first, always ending with '' (no prefix). */
79
+ function resolveBasePaths(spec, explicit) {
80
+ const found = new Set();
81
+ const add = (p) => {
82
+ if (typeof p !== 'string') return;
83
+ let clean = p.trim().replace(/\/{2,}/g, '/');
84
+ if (clean.endsWith('/')) clean = clean.slice(0, -1);
85
+ if (clean && clean.startsWith('/')) found.add(clean);
86
+ };
87
+
88
+ if (explicit) add(explicit);
89
+ if (spec && typeof spec.basePath === 'string') add(spec.basePath);
90
+ for (const server of (spec && Array.isArray(spec.servers) ? spec.servers : [])) {
91
+ if (!server || typeof server.url !== 'string') continue;
92
+ let url = server.url;
93
+ for (const [name, v] of Object.entries(server.variables || {})) {
94
+ if (v && v.default !== undefined) url = url.split(`{${name}}`).join(String(v.default));
95
+ }
96
+ if (url.includes('{')) continue; // unresolved variable — can't use it
97
+ if (/^[a-z][a-z0-9+.-]*:\/\//i.test(url)) {
98
+ try { add(new URL(url).pathname); } catch { /* malformed server url */ }
99
+ } else {
100
+ add(url.startsWith('/') ? url : null);
101
+ }
102
+ }
103
+ return [...found].sort((a, b) => b.length - a.length).concat('');
104
+ }
105
+
106
+ function splitPath(p) {
107
+ return String(p).split('/').filter(Boolean);
108
+ }
109
+
110
+ /** One template segment → predicate. "{id}" any value; "v{n}" / "{id}.json" partial. */
111
+ function segmentMatcher(seg) {
112
+ if (!seg.includes('{')) {
113
+ return (s) => s === seg;
114
+ }
115
+ if (/^\{[^}]+\}$/.test(seg)) {
116
+ return (s) => typeof s === 'string' && s.length > 0;
117
+ }
118
+ const re = new RegExp('^' + seg.split(/\{[^}]+\}/).map(escapeRe).join('[^/]+') + '$');
119
+ // An already-templated segment ("{orderId}") counts as matching any parameter.
120
+ return (s) => typeof s === 'string' && (re.test(s) || /^\{[^}]+\}$/.test(s));
121
+ }
122
+
123
+ function escapeRe(s) {
124
+ return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
125
+ }
126
+
127
+ module.exports = { createRouteMatcher, resolveBasePaths };
@@ -0,0 +1,213 @@
1
+ 'use strict';
2
+
3
+ const fsExtra = require('fs-extra');
4
+ const { templatePath } = require('../har/pathTemplate');
5
+ const { queryParamNames } = require('./logParsers');
6
+ const { clientIdOf } = require('./clientIdentity');
7
+
8
+ /**
9
+ * Aggregates request records into the usage summary written to usage.json.
10
+ *
11
+ * Only aggregates are kept: per (method, path template, client) a call count,
12
+ * first/last seen, and how often each query-parameter NAME was sent. Concrete
13
+ * paths (which carry IDs), query values, IPs and bodies are never stored — the
14
+ * file is safe to commit or upload.
15
+ */
16
+
17
+ const FORMAT = 'specshield-usage';
18
+ const VERSION = 1;
19
+ const MAX_ENDPOINTS = 5000;
20
+ const MAX_UNMATCHED = 500;
21
+ const MAX_PARAMS = 50;
22
+
23
+ class UsageAggregator {
24
+ /**
25
+ * @param {object} opts
26
+ * @param {object} opts.clientSpec from parseClientSpec
27
+ * @param {object} [opts.matcher] from createRouteMatcher (import --spec)
28
+ * @param {number} [opts.sinceMs] drop records older than this epoch ms
29
+ */
30
+ constructor({ clientSpec, matcher = null, sinceMs = null }) {
31
+ this.clientSpec = clientSpec;
32
+ this.matcher = matcher;
33
+ this.sinceMs = sinceMs;
34
+ this.endpoints = new Map();
35
+ this.unmatchedCount = 0;
36
+ this.stats = { requests: 0, matched: 0, unmatched: 0, notFound: 0, tooOld: 0, overflow: 0 };
37
+ this.from = null;
38
+ this.to = null;
39
+ }
40
+
41
+ add(record) {
42
+ if (this.sinceMs !== null && record.time !== null && record.time < this.sinceMs) {
43
+ this.stats.tooOld++;
44
+ return;
45
+ }
46
+
47
+ let path;
48
+ let matched = null;
49
+ if (this.matcher) {
50
+ const m = this.matcher.match(record.method, record.path);
51
+ if (m && m.operation) {
52
+ path = m.template;
53
+ matched = true;
54
+ } else {
55
+ // 404/405 on a path the spec doesn't know is almost always a scanner
56
+ // probing /wp-login.php and friends — not an undocumented endpoint.
57
+ if (record.status === 404 || record.status === 405) { this.stats.notFound++; return; }
58
+ path = templatePath(record.path).templated;
59
+ matched = false;
60
+ }
61
+ } else {
62
+ path = templatePath(record.path).templated;
63
+ }
64
+
65
+ const key = `${record.method} ${path}`;
66
+ let ep = this.endpoints.get(key);
67
+ if (!ep) {
68
+ if (this.endpoints.size >= MAX_ENDPOINTS || (matched === false && this.unmatchedCount >= MAX_UNMATCHED)) {
69
+ this.stats.overflow++;
70
+ return;
71
+ }
72
+ ep = { method: record.method, path, matched, clients: new Map() };
73
+ this.endpoints.set(key, ep);
74
+ if (matched === false) this.unmatchedCount++;
75
+ }
76
+
77
+ const clientId = clientIdOf(record, this.clientSpec);
78
+ let c = ep.clients.get(clientId);
79
+ if (!c) {
80
+ c = { calls: 0, firstSeen: null, lastSeen: null, params: new Map(), paramsTruncated: false };
81
+ ep.clients.set(clientId, c);
82
+ }
83
+ c.calls++;
84
+ if (record.time !== null) {
85
+ if (c.firstSeen === null || record.time < c.firstSeen) c.firstSeen = record.time;
86
+ if (c.lastSeen === null || record.time > c.lastSeen) c.lastSeen = record.time;
87
+ if (this.from === null || record.time < this.from) this.from = record.time;
88
+ if (this.to === null || record.time > this.to) this.to = record.time;
89
+ }
90
+ for (const name of queryParamNames(record.query)) {
91
+ if (c.params.has(name)) c.params.set(name, c.params.get(name) + 1);
92
+ else if (c.params.size < MAX_PARAMS) c.params.set(name, 1);
93
+ else c.paramsTruncated = true;
94
+ }
95
+
96
+ this.stats.requests++;
97
+ if (matched === true) this.stats.matched++;
98
+ else if (matched === false) this.stats.unmatched++;
99
+ }
100
+
101
+ /** Fold a previously written usage.json into this aggregate (import --append). */
102
+ absorb(usage) {
103
+ for (const e of usage.endpoints) {
104
+ const key = `${e.method} ${e.path}`;
105
+ let ep = this.endpoints.get(key);
106
+ if (!ep) {
107
+ ep = { method: e.method, path: e.path, matched: e.matched ?? null, clients: new Map() };
108
+ this.endpoints.set(key, ep);
109
+ if (ep.matched === false) this.unmatchedCount++;
110
+ }
111
+ for (const ec of e.clients) {
112
+ let c = ep.clients.get(ec.id);
113
+ if (!c) {
114
+ c = { calls: 0, firstSeen: null, lastSeen: null, params: new Map(), paramsTruncated: false };
115
+ ep.clients.set(ec.id, c);
116
+ }
117
+ c.calls += ec.calls;
118
+ c.firstSeen = minTime(c.firstSeen, toMs(ec.firstSeen));
119
+ c.lastSeen = maxTime(c.lastSeen, toMs(ec.lastSeen));
120
+ for (const [name, n] of Object.entries(ec.params || {})) {
121
+ if (c.params.has(name)) c.params.set(name, c.params.get(name) + n);
122
+ else if (c.params.size < MAX_PARAMS) c.params.set(name, n);
123
+ else c.paramsTruncated = true;
124
+ }
125
+ if (ec.paramsTruncated) c.paramsTruncated = true;
126
+ }
127
+ }
128
+ this.from = minTime(this.from, toMs(usage.window && usage.window.from));
129
+ this.to = maxTime(this.to, toMs(usage.window && usage.window.to));
130
+ for (const k of Object.keys(this.stats)) this.stats[k] += (usage.stats && usage.stats[k]) || 0;
131
+ }
132
+
133
+ toJSON({ generator, client, spec, basePath, sources }) {
134
+ const endpoints = [...this.endpoints.values()].map((ep) => {
135
+ const clients = [...ep.clients.entries()]
136
+ .map(([id, c]) => {
137
+ const out = { id, calls: c.calls, firstSeen: toIso(c.firstSeen), lastSeen: toIso(c.lastSeen) };
138
+ if (c.params.size > 0) out.params = Object.fromEntries([...c.params.entries()].sort((a, b) => b[1] - a[1]));
139
+ if (c.paramsTruncated) out.paramsTruncated = true;
140
+ return out;
141
+ })
142
+ .sort((a, b) => b.calls - a.calls || a.id.localeCompare(b.id));
143
+ const calls = clients.reduce((n, c) => n + c.calls, 0);
144
+ const lastSeen = clients.reduce((t, c) => maxTime(t, toMs(c.lastSeen)), null);
145
+ return { method: ep.method, path: ep.path, matched: ep.matched, calls, lastSeen: toIso(lastSeen), clients };
146
+ }).sort((a, b) => b.calls - a.calls || `${a.path} ${a.method}`.localeCompare(`${b.path} ${b.method}`));
147
+
148
+ return {
149
+ format: FORMAT,
150
+ version: VERSION,
151
+ generatedAt: new Date().toISOString(),
152
+ generator,
153
+ client,
154
+ spec: spec || null,
155
+ basePath: basePath || null,
156
+ window: { from: toIso(this.from), to: toIso(this.to) },
157
+ stats: { ...this.stats },
158
+ sources,
159
+ endpoints,
160
+ };
161
+ }
162
+ }
163
+
164
+ /** Reads and validates a usage.json. Throws with a readable message. */
165
+ async function loadUsage(file) {
166
+ if (!(await fsExtra.pathExists(file))) throw new Error(`Usage file not found: ${file}`);
167
+ let data;
168
+ try {
169
+ data = JSON.parse(await fsExtra.readFile(file, 'utf8'));
170
+ } catch (err) {
171
+ throw new Error(`Usage file "${file}" is not valid JSON: ${err.message}`);
172
+ }
173
+ return validateUsage(data, file);
174
+ }
175
+
176
+ function validateUsage(data, file = 'usage.json') {
177
+ if (!data || data.format !== FORMAT || !Array.isArray(data.endpoints)) {
178
+ throw new Error(`"${file}" is not a SpecShield usage file. Create one with: specshield usage import <access.log>`);
179
+ }
180
+ if (data.version !== VERSION) {
181
+ throw new Error(`"${file}" is usage format v${data.version}; this CLI reads v${VERSION}. Re-run specshield usage import.`);
182
+ }
183
+ for (const e of data.endpoints) {
184
+ if (typeof e.method !== 'string' || typeof e.path !== 'string' || !Array.isArray(e.clients)) {
185
+ throw new Error(`"${file}" has a malformed endpoint entry`);
186
+ }
187
+ for (const c of e.clients) {
188
+ if (typeof c.id !== 'string' || !Number.isFinite(c.calls) || c.calls < 0) {
189
+ throw new Error(`"${file}" has a malformed client entry under ${e.method} ${e.path}`);
190
+ }
191
+ }
192
+ }
193
+ return data;
194
+ }
195
+
196
+ /** "30d" / "12h" / "2w" / "45m" → milliseconds. */
197
+ function parseDuration(s) {
198
+ const m = /^(\d+)\s*(m|h|d|w)$/i.exec(String(s || '').trim());
199
+ if (!m) throw new Error(`Invalid duration "${s}". Use e.g. 30d, 12h, 2w`);
200
+ const unit = { m: 60e3, h: 3600e3, d: 86400e3, w: 7 * 86400e3 }[m[2].toLowerCase()];
201
+ return Number(m[1]) * unit;
202
+ }
203
+
204
+ function toMs(v) {
205
+ if (v === null || v === undefined) return null;
206
+ const t = typeof v === 'number' ? v : Date.parse(v);
207
+ return Number.isNaN(t) ? null : t;
208
+ }
209
+ function toIso(ms) { return ms === null || ms === undefined ? null : new Date(ms).toISOString(); }
210
+ function minTime(a, b) { return a === null ? b : b === null ? a : Math.min(a, b); }
211
+ function maxTime(a, b) { return a === null ? b : b === null ? a : Math.max(a, b); }
212
+
213
+ module.exports = { UsageAggregator, loadUsage, validateUsage, parseDuration, toMs, FORMAT, VERSION };