specshield 3.4.11 → 3.4.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,285 @@
1
+ 'use strict';
2
+
3
+ /**
4
+ * Access-log line parsers for `specshield usage import`.
5
+ *
6
+ * Every parser turns one line into a request record, or null when the line
7
+ * isn't a request it understands (blank, comment, health-check noise in an
8
+ * unknown shape):
9
+ *
10
+ * { method, path, query, status, time, userAgent, extras, json }
11
+ *
12
+ * method upper-case HTTP method
13
+ * path URL path only — no scheme/host, no query string
14
+ * host lower-case host without port, when the log records it (ALB,
15
+ * JSON with a host field, absolute request URLs), else null
16
+ * query raw query string without the leading '?' ('' when absent)
17
+ * status numeric status code, or null
18
+ * time epoch ms, or null when the line carries no parseable timestamp
19
+ * userAgent raw User-Agent, or null
20
+ * extras quoted fields after the User-Agent (combined format only) —
21
+ * where nginx `log_format` additions such as $http_x_api_key land
22
+ * json the parsed object (json format only), for --client field:<path>
23
+ *
24
+ * Supported formats:
25
+ * combined nginx / Apache "combined" and "common" log formats
26
+ * alb AWS Application Load Balancer access logs
27
+ * json one JSON object per line — nginx JSON log_format, Kong file-log,
28
+ * API Gateway JSON access logs, Envoy/Traefik JSON, ECS-style logs
29
+ */
30
+
31
+ const HTTP_METHODS = new Set(['GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS', 'TRACE', 'CONNECT']);
32
+
33
+ // ─── combined / common ──────────────────────────────────────────────────────
34
+
35
+ // host ident authuser [time] "request" status bytes ["referer" "user-agent"] [extras...]
36
+ const COMBINED_RE = /^(\S+) \S+ \S+ \[([^\]]+)\] "((?:[^"\\]|\\.)*)" (\d{3}|-) \S+(?: "((?:[^"\\]|\\.)*)" "((?:[^"\\]|\\.)*)")?(.*)$/;
37
+
38
+ function parseCombined(line) {
39
+ const m = COMBINED_RE.exec(line);
40
+ if (!m) return null;
41
+ const req = splitRequestLine(unescape(m[3]));
42
+ if (!req) return null;
43
+ return {
44
+ ...req,
45
+ status: m[4] === '-' ? null : Number(m[4]),
46
+ time: parseClfTime(m[2]),
47
+ userAgent: m[6] !== undefined ? nullIfDash(unescape(m[6])) : null,
48
+ extras: m[7] ? tokenize(m[7]).map(nullIfDash) : [],
49
+ json: null,
50
+ };
51
+ }
52
+
53
+ const MONTHS = { Jan: 0, Feb: 1, Mar: 2, Apr: 3, May: 4, Jun: 5, Jul: 6, Aug: 7, Sep: 8, Oct: 9, Nov: 10, Dec: 11 };
54
+
55
+ /** "10/Oct/2026:13:55:36 +0530" → epoch ms. */
56
+ function parseClfTime(s) {
57
+ const m = /^(\d{2})\/(\w{3})\/(\d{4}):(\d{2}):(\d{2}):(\d{2})(?: ([+-])(\d{2})(\d{2}))?$/.exec(s || '');
58
+ if (!m || MONTHS[m[2]] === undefined) return null;
59
+ const utc = Date.UTC(+m[3], MONTHS[m[2]], +m[1], +m[4], +m[5], +m[6]);
60
+ const offsetMin = m[7] ? (m[7] === '-' ? -1 : 1) * (+m[8] * 60 + +m[9]) : 0;
61
+ return utc - offsetMin * 60000;
62
+ }
63
+
64
+ // ─── AWS ALB ────────────────────────────────────────────────────────────────
65
+
66
+ // type time elb client:port target:port req_t target_t resp_t elb_status
67
+ // target_status recv sent "request" "user_agent" ...
68
+ const ALB_TYPES = new Set(['http', 'https', 'h2', 'grpcs', 'ws', 'wss']);
69
+
70
+ function parseAlb(line) {
71
+ const f = tokenize(line);
72
+ if (f.length < 14 || !ALB_TYPES.has(f[0])) return null;
73
+ const req = splitRequestLine(f[12]);
74
+ if (!req) return null;
75
+ const time = Date.parse(f[1]);
76
+ return {
77
+ ...req,
78
+ status: /^\d{3}$/.test(f[8]) ? Number(f[8]) : null,
79
+ time: Number.isNaN(time) ? null : time,
80
+ userAgent: nullIfDash(f[13]),
81
+ extras: [],
82
+ json: null,
83
+ };
84
+ }
85
+
86
+ // ─── JSON lines ─────────────────────────────────────────────────────────────
87
+
88
+ // Field names used by common producers: nginx/Apache JSON log_format, Kong
89
+ // file-log, Caddy, Traefik, Envoy, Google Cloud HTTP(S) LB, AWS API Gateway,
90
+ // ECS/OpenTelemetry, Elastic examples. Order matters — first non-empty wins,
91
+ // so nested keys come before same-named containers ("response.status" before
92
+ // "response").
93
+ const JSON_KEYS = {
94
+ method: ['method', 'request_method', 'httpMethod', 'http_method', 'verb', 'RequestMethod',
95
+ 'request.method', 'req.method', 'http.method', 'http.request.method', 'httpRequest.requestMethod'],
96
+ path: ['path', 'uri', 'request_uri', 'url', 'RequestPath', 'request.uri', 'request.url', 'req.url',
97
+ 'req.path', 'http.url', 'http.target', 'url.original', 'url.path', 'httpRequest.requestUrl'],
98
+ query: ['query', 'query_string', 'args', 'url.query'],
99
+ host: ['host', 'http_host', 'RequestHost', 'request.host', 'request.headers.host', 'request.headers.Host',
100
+ 'req.headers.host', 'headers.host', 'url.domain', 'http.host', 'domainName'],
101
+ status: ['status', 'status_code', 'statusCode', 'response_code', 'DownstreamStatus', 'OriginStatus',
102
+ 'response.status', 'res.statusCode', 'http.status_code', 'http.response.status_code',
103
+ 'httpRequest.status', 'response'],
104
+ time: ['time', 'timestamp', '@timestamp', 'time_iso8601', 'time_local', 'requestTime',
105
+ 'requestTimeEpoch', 'started_at', 'ts', 'start_time', 'StartUTC'],
106
+ userAgent: ['user_agent', 'http_user_agent', 'userAgent', 'agent', 'user_agent.original',
107
+ 'request.headers.user-agent', 'request.headers.User-Agent', 'req.headers.user-agent',
108
+ 'headers.user-agent', 'http.user_agent', 'request_User-Agent', 'httpRequest.userAgent'],
109
+ };
110
+
111
+ function parseJsonLine(line) {
112
+ const trimmed = line.trim();
113
+ if (!trimmed.startsWith('{')) return null;
114
+ let obj;
115
+ try { obj = JSON.parse(trimmed); } catch { return null; }
116
+ if (!obj || typeof obj !== 'object') return null;
117
+
118
+ let method = pick(obj, JSON_KEYS.method);
119
+ let target = pick(obj, JSON_KEYS.path);
120
+
121
+ // nginx JSON formats often log "$request" ("GET /x?y HTTP/1.1") instead of
122
+ // separate method and uri fields.
123
+ if ((!method || !target) && typeof obj.request === 'string') {
124
+ const req = splitRequestLine(obj.request);
125
+ if (req) { method = method || req.method; target = target || req.path + (req.query ? `?${req.query}` : ''); }
126
+ }
127
+ if (typeof method !== 'string' || typeof target !== 'string') return null;
128
+ method = method.toUpperCase();
129
+ if (!HTTP_METHODS.has(method)) return null;
130
+
131
+ const { path, query, host } = splitTarget(target);
132
+ if (!path) return null;
133
+ const explicitQuery = pick(obj, JSON_KEYS.query);
134
+ const status = Number(pick(obj, JSON_KEYS.status));
135
+
136
+ return {
137
+ method,
138
+ path,
139
+ query: query || (typeof explicitQuery === 'string' ? explicitQuery.replace(/^\?/, '') : ''),
140
+ host: host || normalizeHost(asString(pick(obj, JSON_KEYS.host))),
141
+ status: Number.isInteger(status) && status >= 100 && status <= 599 ? status : null,
142
+ time: parseAnyTime(pick(obj, JSON_KEYS.time)),
143
+ userAgent: nullIfDash(asString(pick(obj, JSON_KEYS.userAgent))),
144
+ extras: [],
145
+ json: obj,
146
+ };
147
+ }
148
+
149
+ /** Value at a dotted path. Tries the literal key first ("user_agent.original" as one key). */
150
+ function lookup(obj, dotted) {
151
+ if (obj == null) return undefined;
152
+ if (Object.prototype.hasOwnProperty.call(obj, dotted)) return obj[dotted];
153
+ let cur = obj;
154
+ for (const part of dotted.split('.')) {
155
+ if (cur == null || typeof cur !== 'object') return undefined;
156
+ cur = cur[part];
157
+ }
158
+ return cur;
159
+ }
160
+
161
+ function pick(obj, keys) {
162
+ for (const k of keys) {
163
+ const v = lookup(obj, k);
164
+ if (v !== undefined && v !== null && v !== '') return v;
165
+ }
166
+ return undefined;
167
+ }
168
+
169
+ function parseAnyTime(v) {
170
+ if (v === undefined || v === null || v === '') return null;
171
+ if (typeof v === 'number' || /^\d+(\.\d+)?$/.test(String(v))) {
172
+ const n = Number(v);
173
+ return n > 1e12 ? Math.round(n) : Math.round(n * 1000); // ms vs seconds
174
+ }
175
+ const s = String(v);
176
+ const clf = parseClfTime(s);
177
+ if (clf !== null) return clf;
178
+ const t = Date.parse(s);
179
+ return Number.isNaN(t) ? null : t;
180
+ }
181
+
182
+ // ─── Format detection ───────────────────────────────────────────────────────
183
+
184
+ const PARSERS = { combined: parseCombined, alb: parseAlb, json: parseJsonLine };
185
+
186
+ /** Picks the format that parses the most of the sample lines. */
187
+ function detectFormat(sampleLines) {
188
+ let best = null;
189
+ let bestHits = 0;
190
+ for (const [name, parse] of Object.entries(PARSERS)) {
191
+ const hits = sampleLines.filter((l) => parse(l)).length;
192
+ if (hits > bestHits) { best = name; bestHits = hits; }
193
+ }
194
+ return best;
195
+ }
196
+
197
+ // ─── Helpers ────────────────────────────────────────────────────────────────
198
+
199
+ /** "GET /a/b?x=1 HTTP/1.1" (or an absolute URL as ALB logs it) → { method, path, query }. */
200
+ function splitRequestLine(s) {
201
+ if (typeof s !== 'string') return null;
202
+ const parts = s.trim().split(/\s+/);
203
+ if (parts.length < 2) return null;
204
+ const method = parts[0].toUpperCase();
205
+ if (!HTTP_METHODS.has(method)) return null;
206
+ const { path, query, host } = splitTarget(parts[1]);
207
+ if (!path) return null;
208
+ return { method, path, query, host };
209
+ }
210
+
211
+ function splitTarget(target) {
212
+ let t = target;
213
+ let host = null;
214
+ // Absolute URL (ALB, some proxies) → keep only the path + query, note the host.
215
+ const abs = /^[a-z][a-z0-9+.-]*:\/\/([^/?#]*)/i.exec(t);
216
+ if (abs) {
217
+ host = normalizeHost(abs[1]);
218
+ t = t.slice(abs[0].length) || '/';
219
+ }
220
+ if (!t.startsWith('/')) return { path: null, query: '', host };
221
+ const hash = t.indexOf('#');
222
+ if (hash >= 0) t = t.slice(0, hash);
223
+ const q = t.indexOf('?');
224
+ let path = q >= 0 ? t.slice(0, q) : t;
225
+ const query = q >= 0 ? t.slice(q + 1) : '';
226
+ path = path.replace(/\/{2,}/g, '/');
227
+ if (path.length > 1 && path.endsWith('/')) path = path.slice(0, -1);
228
+ return { path, query, host };
229
+ }
230
+
231
+ /** "API.Example.com:443" → "api.example.com"; "[::1]:80" → "[::1]". */
232
+ function normalizeHost(h) {
233
+ if (!h || h === '-') return null;
234
+ const s = String(h).trim().toLowerCase().replace(/^.*@/, '');
235
+ const m = /^(\[[^\]]+\]|[^:]+)(?::\d+)?$/.exec(s);
236
+ return m ? m[1] : null;
237
+ }
238
+
239
+ /** Query-string parameter NAMES only — values are never kept. */
240
+ function queryParamNames(query) {
241
+ if (!query) return [];
242
+ const names = new Set();
243
+ for (const pair of query.split('&')) {
244
+ if (!pair) continue;
245
+ const raw = pair.split('=')[0];
246
+ let name;
247
+ try { name = decodeURIComponent(raw.replace(/\+/g, ' ')); } catch { name = raw; }
248
+ name = name.replace(/\[\]$/, ''); // tags[]=a → tags
249
+ if (name && name.length <= 100) names.add(name);
250
+ }
251
+ return [...names];
252
+ }
253
+
254
+ /** Space-separated tokens; "double quoted" tokens may contain spaces and \" escapes. */
255
+ function tokenize(s) {
256
+ const out = [];
257
+ const re = /"((?:[^"\\]|\\.)*)"|(\S+)/g;
258
+ let m;
259
+ while ((m = re.exec(s)) !== null) out.push(m[1] !== undefined ? unescape(m[1]) : m[2]);
260
+ return out;
261
+ }
262
+
263
+ function unescape(s) {
264
+ return s.replace(/\\(["\\])/g, '$1');
265
+ }
266
+
267
+ function nullIfDash(s) {
268
+ return s === undefined || s === null || s === '' || s === '-' ? null : s;
269
+ }
270
+
271
+ function asString(v) {
272
+ if (v === undefined || v === null) return null;
273
+ return Array.isArray(v) ? String(v[0]) : String(v);
274
+ }
275
+
276
+ module.exports = {
277
+ PARSERS,
278
+ parseCombined,
279
+ parseAlb,
280
+ parseJsonLine,
281
+ detectFormat,
282
+ queryParamNames,
283
+ lookup,
284
+ splitTarget,
285
+ };
@@ -0,0 +1,127 @@
1
+ 'use strict';
2
+
3
+ /**
4
+ * Maps a concrete request path from a log ("/v1/orders/123/receipt") onto the
5
+ * OpenAPI path template that serves it ("/orders/{orderId}/receipt").
6
+ *
7
+ * Handles the two things that make naive string matching fail:
8
+ * - base paths: spec paths are relative to servers[].url (OpenAPI 3) or
9
+ * basePath (Swagger 2), while logs record the full path;
10
+ * - overlapping templates: /users/me and /users/{id} both match "/users/me",
11
+ * and the one with more literal segments wins, as in most routers.
12
+ *
13
+ * Paths that are already templated (usage.json written without --spec stores
14
+ * "/orders/{orderId}") also match: a "{…}" segment matches any template
15
+ * parameter, never a literal.
16
+ */
17
+
18
+ const HTTP_METHODS = ['get', 'put', 'post', 'delete', 'options', 'head', 'patch', 'trace'];
19
+
20
+ function createRouteMatcher(spec, opts = {}) {
21
+ const routes = [];
22
+ for (const [template, item] of Object.entries((spec && spec.paths) || {})) {
23
+ if (!item || typeof item !== 'object') continue;
24
+ const methods = new Set(HTTP_METHODS.filter((m) => item[m]).map((m) => m.toUpperCase()));
25
+ if (methods.size === 0) continue;
26
+ const segs = splitPath(template);
27
+ routes.push({
28
+ template,
29
+ methods,
30
+ matchers: segs.map(segmentMatcher),
31
+ literals: segs.filter((s) => !s.includes('{')).length,
32
+ });
33
+ }
34
+
35
+ const bySegCount = new Map();
36
+ for (const r of routes) {
37
+ const n = r.matchers.length;
38
+ if (!bySegCount.has(n)) bySegCount.set(n, []);
39
+ bySegCount.get(n).push(r);
40
+ }
41
+ for (const list of bySegCount.values()) list.sort((a, b) => b.literals - a.literals);
42
+
43
+ const basePaths = resolveBasePaths(spec, opts.basePath);
44
+
45
+ /**
46
+ * @returns {{ template, method, operation: boolean } | null}
47
+ * operation=false when the path exists but the spec doesn't define that method.
48
+ */
49
+ function match(method, path) {
50
+ const upper = String(method).toUpperCase();
51
+ for (const prefix of basePaths) {
52
+ let rel;
53
+ if (prefix === '') rel = path;
54
+ else if (path === prefix) rel = '/';
55
+ else if (path.startsWith(prefix + '/')) rel = path.slice(prefix.length);
56
+ else continue;
57
+
58
+ const segs = splitPath(rel);
59
+ const candidates = bySegCount.get(segs.length) || [];
60
+ let pathOnly = null;
61
+ for (const r of candidates) {
62
+ if (!r.matchers.every((test, i) => test(segs[i]))) continue;
63
+ if (r.methods.has(upper)) return { template: r.template, method: upper, operation: true };
64
+ if (!pathOnly) pathOnly = { template: r.template, method: upper, operation: false };
65
+ }
66
+ if (pathOnly) return pathOnly;
67
+ }
68
+ return null;
69
+ }
70
+
71
+ return {
72
+ match,
73
+ basePaths: basePaths.filter(Boolean),
74
+ operations: routes.flatMap((r) => [...r.methods].map((m) => ({ method: m, template: r.template }))),
75
+ };
76
+ }
77
+
78
+ /** Longest base path first, always ending with '' (no prefix). */
79
+ function resolveBasePaths(spec, explicit) {
80
+ const found = new Set();
81
+ const add = (p) => {
82
+ if (typeof p !== 'string') return;
83
+ let clean = p.trim().replace(/\/{2,}/g, '/');
84
+ if (clean.endsWith('/')) clean = clean.slice(0, -1);
85
+ if (clean && clean.startsWith('/')) found.add(clean);
86
+ };
87
+
88
+ if (explicit) add(explicit);
89
+ if (spec && typeof spec.basePath === 'string') add(spec.basePath);
90
+ for (const server of (spec && Array.isArray(spec.servers) ? spec.servers : [])) {
91
+ if (!server || typeof server.url !== 'string') continue;
92
+ let url = server.url;
93
+ for (const [name, v] of Object.entries(server.variables || {})) {
94
+ if (v && v.default !== undefined) url = url.split(`{${name}}`).join(String(v.default));
95
+ }
96
+ if (url.includes('{')) continue; // unresolved variable — can't use it
97
+ if (/^[a-z][a-z0-9+.-]*:\/\//i.test(url)) {
98
+ try { add(new URL(url).pathname); } catch { /* malformed server url */ }
99
+ } else {
100
+ add(url.startsWith('/') ? url : null);
101
+ }
102
+ }
103
+ return [...found].sort((a, b) => b.length - a.length).concat('');
104
+ }
105
+
106
+ function splitPath(p) {
107
+ return String(p).split('/').filter(Boolean);
108
+ }
109
+
110
+ /** One template segment → predicate. "{id}" any value; "v{n}" / "{id}.json" partial. */
111
+ function segmentMatcher(seg) {
112
+ if (!seg.includes('{')) {
113
+ return (s) => s === seg;
114
+ }
115
+ if (/^\{[^}]+\}$/.test(seg)) {
116
+ return (s) => typeof s === 'string' && s.length > 0;
117
+ }
118
+ const re = new RegExp('^' + seg.split(/\{[^}]+\}/).map(escapeRe).join('[^/]+') + '$');
119
+ // An already-templated segment ("{orderId}") counts as matching any parameter.
120
+ return (s) => typeof s === 'string' && (re.test(s) || /^\{[^}]+\}$/.test(s));
121
+ }
122
+
123
+ function escapeRe(s) {
124
+ return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
125
+ }
126
+
127
+ module.exports = { createRouteMatcher, resolveBasePaths };
@@ -0,0 +1,235 @@
1
+ 'use strict';
2
+
3
+ const fsExtra = require('fs-extra');
4
+ const { templatePath } = require('../har/pathTemplate');
5
+ const { queryParamNames } = require('./logParsers');
6
+ const { clientIdOf } = require('./clientIdentity');
7
+
8
+ /**
9
+ * Aggregates request records into the usage summary written to usage.json.
10
+ *
11
+ * Only aggregates are kept: per (method, path template, client) a call count,
12
+ * first/last seen, and how often each query-parameter NAME was sent. Concrete
13
+ * paths (which carry IDs), query values, IPs and bodies are never stored — the
14
+ * file is safe to commit or upload.
15
+ */
16
+
17
+ const FORMAT = 'specshield-usage';
18
+ const VERSION = 1;
19
+ const MAX_ENDPOINTS = 5000;
20
+ const MAX_UNMATCHED = 500;
21
+ const MAX_PARAMS = 50;
22
+
23
+ class UsageAggregator {
24
+ /**
25
+ * @param {object} opts
26
+ * @param {object} opts.clientSpec from parseClientSpec
27
+ * @param {object} [opts.matcher] from createRouteMatcher (import --spec)
28
+ * @param {number} [opts.sinceMs] drop records older than this epoch ms
29
+ */
30
+ constructor({ clientSpec, matcher = null, sinceMs = null, hosts = [] }) {
31
+ this.clientSpec = clientSpec;
32
+ this.matcher = matcher;
33
+ this.sinceMs = sinceMs;
34
+ this.hostTest = compileHostFilter(hosts);
35
+ this.hostsSeen = new Map();
36
+ this.endpoints = new Map();
37
+ this.unmatchedCount = 0;
38
+ this.stats = { requests: 0, matched: 0, unmatched: 0, notFound: 0, tooOld: 0, otherHost: 0, overflow: 0 };
39
+ this.from = null;
40
+ this.to = null;
41
+ }
42
+
43
+ add(record) {
44
+ // One load balancer often fronts several APIs by host name. Requests for
45
+ // another host must not count as usage of this API.
46
+ if (record.host) {
47
+ if (this.hostTest && !this.hostTest(record.host)) { this.stats.otherHost++; return; }
48
+ if (this.hostsSeen.size < 50 || this.hostsSeen.has(record.host)) {
49
+ this.hostsSeen.set(record.host, (this.hostsSeen.get(record.host) || 0) + 1);
50
+ }
51
+ }
52
+ if (this.sinceMs !== null && record.time !== null && record.time < this.sinceMs) {
53
+ this.stats.tooOld++;
54
+ return;
55
+ }
56
+
57
+ let path;
58
+ let matched = null;
59
+ if (this.matcher) {
60
+ const m = this.matcher.match(record.method, record.path);
61
+ if (m && m.operation) {
62
+ path = m.template;
63
+ matched = true;
64
+ } else {
65
+ // 404/405 on a path the spec doesn't know is almost always a scanner
66
+ // probing /wp-login.php and friends — not an undocumented endpoint.
67
+ if (record.status === 404 || record.status === 405) { this.stats.notFound++; return; }
68
+ path = templatePath(record.path).templated;
69
+ matched = false;
70
+ }
71
+ } else {
72
+ path = templatePath(record.path).templated;
73
+ }
74
+
75
+ const key = `${record.method} ${path}`;
76
+ let ep = this.endpoints.get(key);
77
+ if (!ep) {
78
+ if (this.endpoints.size >= MAX_ENDPOINTS || (matched === false && this.unmatchedCount >= MAX_UNMATCHED)) {
79
+ this.stats.overflow++;
80
+ return;
81
+ }
82
+ ep = { method: record.method, path, matched, clients: new Map() };
83
+ this.endpoints.set(key, ep);
84
+ if (matched === false) this.unmatchedCount++;
85
+ }
86
+
87
+ const clientId = clientIdOf(record, this.clientSpec);
88
+ let c = ep.clients.get(clientId);
89
+ if (!c) {
90
+ c = { calls: 0, firstSeen: null, lastSeen: null, params: new Map(), paramsTruncated: false };
91
+ ep.clients.set(clientId, c);
92
+ }
93
+ c.calls++;
94
+ if (record.time !== null) {
95
+ if (c.firstSeen === null || record.time < c.firstSeen) c.firstSeen = record.time;
96
+ if (c.lastSeen === null || record.time > c.lastSeen) c.lastSeen = record.time;
97
+ if (this.from === null || record.time < this.from) this.from = record.time;
98
+ if (this.to === null || record.time > this.to) this.to = record.time;
99
+ }
100
+ for (const name of queryParamNames(record.query)) {
101
+ if (c.params.has(name)) c.params.set(name, c.params.get(name) + 1);
102
+ else if (c.params.size < MAX_PARAMS) c.params.set(name, 1);
103
+ else c.paramsTruncated = true;
104
+ }
105
+
106
+ this.stats.requests++;
107
+ if (matched === true) this.stats.matched++;
108
+ else if (matched === false) this.stats.unmatched++;
109
+ }
110
+
111
+ /** Fold a previously written usage.json into this aggregate (import --append). */
112
+ absorb(usage) {
113
+ for (const e of usage.endpoints) {
114
+ const key = `${e.method} ${e.path}`;
115
+ let ep = this.endpoints.get(key);
116
+ if (!ep) {
117
+ ep = { method: e.method, path: e.path, matched: e.matched ?? null, clients: new Map() };
118
+ this.endpoints.set(key, ep);
119
+ if (ep.matched === false) this.unmatchedCount++;
120
+ }
121
+ for (const ec of e.clients) {
122
+ let c = ep.clients.get(ec.id);
123
+ if (!c) {
124
+ c = { calls: 0, firstSeen: null, lastSeen: null, params: new Map(), paramsTruncated: false };
125
+ ep.clients.set(ec.id, c);
126
+ }
127
+ c.calls += ec.calls;
128
+ c.firstSeen = minTime(c.firstSeen, toMs(ec.firstSeen));
129
+ c.lastSeen = maxTime(c.lastSeen, toMs(ec.lastSeen));
130
+ for (const [name, n] of Object.entries(ec.params || {})) {
131
+ if (c.params.has(name)) c.params.set(name, c.params.get(name) + n);
132
+ else if (c.params.size < MAX_PARAMS) c.params.set(name, n);
133
+ else c.paramsTruncated = true;
134
+ }
135
+ if (ec.paramsTruncated) c.paramsTruncated = true;
136
+ }
137
+ }
138
+ this.from = minTime(this.from, toMs(usage.window && usage.window.from));
139
+ this.to = maxTime(this.to, toMs(usage.window && usage.window.to));
140
+ for (const k of Object.keys(this.stats)) this.stats[k] += (usage.stats && usage.stats[k]) || 0;
141
+ }
142
+
143
+ toJSON({ generator, client, spec, basePath, sources }) {
144
+ const endpoints = [...this.endpoints.values()].map((ep) => {
145
+ const clients = [...ep.clients.entries()]
146
+ .map(([id, c]) => {
147
+ const out = { id, calls: c.calls, firstSeen: toIso(c.firstSeen), lastSeen: toIso(c.lastSeen) };
148
+ if (c.params.size > 0) out.params = Object.fromEntries([...c.params.entries()].sort((a, b) => b[1] - a[1]));
149
+ if (c.paramsTruncated) out.paramsTruncated = true;
150
+ return out;
151
+ })
152
+ .sort((a, b) => b.calls - a.calls || a.id.localeCompare(b.id));
153
+ const calls = clients.reduce((n, c) => n + c.calls, 0);
154
+ const lastSeen = clients.reduce((t, c) => maxTime(t, toMs(c.lastSeen)), null);
155
+ return { method: ep.method, path: ep.path, matched: ep.matched, calls, lastSeen: toIso(lastSeen), clients };
156
+ }).sort((a, b) => b.calls - a.calls || `${a.path} ${a.method}`.localeCompare(`${b.path} ${b.method}`));
157
+
158
+ return {
159
+ format: FORMAT,
160
+ version: VERSION,
161
+ generatedAt: new Date().toISOString(),
162
+ generator,
163
+ client,
164
+ spec: spec || null,
165
+ basePath: basePath || null,
166
+ window: { from: toIso(this.from), to: toIso(this.to) },
167
+ stats: { ...this.stats },
168
+ sources,
169
+ endpoints,
170
+ };
171
+ }
172
+ }
173
+
174
+ /** --host values: exact host names, or globs like "*.example.com". Null = no filter. */
175
+ function compileHostFilter(hosts) {
176
+ const list = (hosts || []).filter(Boolean).map((h) => String(h).toLowerCase());
177
+ if (list.length === 0) return null;
178
+ const tests = list.map((h) => {
179
+ if (!h.includes('*')) return (x) => x === h;
180
+ const re = new RegExp('^' + h.split('*').map((p) => p.replace(/[.+?^${}()|[\]\\]/g, '\\$&')).join('.*') + '$');
181
+ return (x) => re.test(x);
182
+ });
183
+ return (host) => tests.some((t) => t(host));
184
+ }
185
+
186
+ /** Reads and validates a usage.json. Throws with a readable message. */
187
+ async function loadUsage(file) {
188
+ if (!(await fsExtra.pathExists(file))) throw new Error(`Usage file not found: ${file}`);
189
+ let data;
190
+ try {
191
+ data = JSON.parse(await fsExtra.readFile(file, 'utf8'));
192
+ } catch (err) {
193
+ throw new Error(`Usage file "${file}" is not valid JSON: ${err.message}`);
194
+ }
195
+ return validateUsage(data, file);
196
+ }
197
+
198
+ function validateUsage(data, file = 'usage.json') {
199
+ if (!data || data.format !== FORMAT || !Array.isArray(data.endpoints)) {
200
+ throw new Error(`"${file}" is not a SpecShield usage file. Create one with: specshield usage import <access.log>`);
201
+ }
202
+ if (data.version !== VERSION) {
203
+ throw new Error(`"${file}" is usage format v${data.version}; this CLI reads v${VERSION}. Re-run specshield usage import.`);
204
+ }
205
+ for (const e of data.endpoints) {
206
+ if (typeof e.method !== 'string' || typeof e.path !== 'string' || !Array.isArray(e.clients)) {
207
+ throw new Error(`"${file}" has a malformed endpoint entry`);
208
+ }
209
+ for (const c of e.clients) {
210
+ if (typeof c.id !== 'string' || !Number.isFinite(c.calls) || c.calls < 0) {
211
+ throw new Error(`"${file}" has a malformed client entry under ${e.method} ${e.path}`);
212
+ }
213
+ }
214
+ }
215
+ return data;
216
+ }
217
+
218
+ /** "30d" / "12h" / "2w" / "45m" → milliseconds. */
219
+ function parseDuration(s) {
220
+ const m = /^(\d+)\s*(m|h|d|w)$/i.exec(String(s || '').trim());
221
+ if (!m) throw new Error(`Invalid duration "${s}". Use e.g. 30d, 12h, 2w`);
222
+ const unit = { m: 60e3, h: 3600e3, d: 86400e3, w: 7 * 86400e3 }[m[2].toLowerCase()];
223
+ return Number(m[1]) * unit;
224
+ }
225
+
226
+ function toMs(v) {
227
+ if (v === null || v === undefined) return null;
228
+ const t = typeof v === 'number' ? v : Date.parse(v);
229
+ return Number.isNaN(t) ? null : t;
230
+ }
231
+ function toIso(ms) { return ms === null || ms === undefined ? null : new Date(ms).toISOString(); }
232
+ function minTime(a, b) { return a === null ? b : b === null ? a : Math.min(a, b); }
233
+ function maxTime(a, b) { return a === null ? b : b === null ? a : Math.max(a, b); }
234
+
235
+ module.exports = { UsageAggregator, loadUsage, validateUsage, parseDuration, toMs, FORMAT, VERSION };