specshield 3.4.11 → 3.4.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/README.md +94 -1
- package/package.json +1 -1
- package/src/cli.js +2 -0
- package/src/commands/compare.js +63 -0
- package/src/commands/usage.js +243 -0
- package/src/core/configLoader.js +32 -0
- package/src/core/outputFormatter.js +97 -3
- package/src/core/usage/clientIdentity.js +70 -0
- package/src/core/usage/format.js +31 -0
- package/src/core/usage/impact.js +243 -0
- package/src/core/usage/importLogs.js +85 -0
- package/src/core/usage/logParsers.js +285 -0
- package/src/core/usage/routeMatcher.js +127 -0
- package/src/core/usage/usageStore.js +235 -0
|
@@ -0,0 +1,285 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Access-log line parsers for `specshield usage import`.
|
|
5
|
+
*
|
|
6
|
+
* Every parser turns one line into a request record, or null when the line
|
|
7
|
+
* isn't a request it understands (blank, comment, health-check noise in an
|
|
8
|
+
* unknown shape):
|
|
9
|
+
*
|
|
10
|
+
* { method, path, query, status, time, userAgent, extras, json }
|
|
11
|
+
*
|
|
12
|
+
* method upper-case HTTP method
|
|
13
|
+
* path URL path only — no scheme/host, no query string
|
|
14
|
+
* host lower-case host without port, when the log records it (ALB,
|
|
15
|
+
* JSON with a host field, absolute request URLs), else null
|
|
16
|
+
* query raw query string without the leading '?' ('' when absent)
|
|
17
|
+
* status numeric status code, or null
|
|
18
|
+
* time epoch ms, or null when the line carries no parseable timestamp
|
|
19
|
+
* userAgent raw User-Agent, or null
|
|
20
|
+
* extras quoted fields after the User-Agent (combined format only) —
|
|
21
|
+
* where nginx `log_format` additions such as $http_x_api_key land
|
|
22
|
+
* json the parsed object (json format only), for --client field:<path>
|
|
23
|
+
*
|
|
24
|
+
* Supported formats:
|
|
25
|
+
* combined nginx / Apache "combined" and "common" log formats
|
|
26
|
+
* alb AWS Application Load Balancer access logs
|
|
27
|
+
* json one JSON object per line — nginx JSON log_format, Kong file-log,
|
|
28
|
+
* API Gateway JSON access logs, Envoy/Traefik JSON, ECS-style logs
|
|
29
|
+
*/
|
|
30
|
+
|
|
31
|
+
const HTTP_METHODS = new Set(['GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS', 'TRACE', 'CONNECT']);
|
|
32
|
+
|
|
33
|
+
// ─── combined / common ──────────────────────────────────────────────────────
|
|
34
|
+
|
|
35
|
+
// host ident authuser [time] "request" status bytes ["referer" "user-agent"] [extras...]
|
|
36
|
+
const COMBINED_RE = /^(\S+) \S+ \S+ \[([^\]]+)\] "((?:[^"\\]|\\.)*)" (\d{3}|-) \S+(?: "((?:[^"\\]|\\.)*)" "((?:[^"\\]|\\.)*)")?(.*)$/;
|
|
37
|
+
|
|
38
|
+
function parseCombined(line) {
|
|
39
|
+
const m = COMBINED_RE.exec(line);
|
|
40
|
+
if (!m) return null;
|
|
41
|
+
const req = splitRequestLine(unescape(m[3]));
|
|
42
|
+
if (!req) return null;
|
|
43
|
+
return {
|
|
44
|
+
...req,
|
|
45
|
+
status: m[4] === '-' ? null : Number(m[4]),
|
|
46
|
+
time: parseClfTime(m[2]),
|
|
47
|
+
userAgent: m[6] !== undefined ? nullIfDash(unescape(m[6])) : null,
|
|
48
|
+
extras: m[7] ? tokenize(m[7]).map(nullIfDash) : [],
|
|
49
|
+
json: null,
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
const MONTHS = { Jan: 0, Feb: 1, Mar: 2, Apr: 3, May: 4, Jun: 5, Jul: 6, Aug: 7, Sep: 8, Oct: 9, Nov: 10, Dec: 11 };
|
|
54
|
+
|
|
55
|
+
/** "10/Oct/2026:13:55:36 +0530" → epoch ms. */
|
|
56
|
+
function parseClfTime(s) {
|
|
57
|
+
const m = /^(\d{2})\/(\w{3})\/(\d{4}):(\d{2}):(\d{2}):(\d{2})(?: ([+-])(\d{2})(\d{2}))?$/.exec(s || '');
|
|
58
|
+
if (!m || MONTHS[m[2]] === undefined) return null;
|
|
59
|
+
const utc = Date.UTC(+m[3], MONTHS[m[2]], +m[1], +m[4], +m[5], +m[6]);
|
|
60
|
+
const offsetMin = m[7] ? (m[7] === '-' ? -1 : 1) * (+m[8] * 60 + +m[9]) : 0;
|
|
61
|
+
return utc - offsetMin * 60000;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
// ─── AWS ALB ────────────────────────────────────────────────────────────────
|
|
65
|
+
|
|
66
|
+
// type time elb client:port target:port req_t target_t resp_t elb_status
|
|
67
|
+
// target_status recv sent "request" "user_agent" ...
|
|
68
|
+
const ALB_TYPES = new Set(['http', 'https', 'h2', 'grpcs', 'ws', 'wss']);
|
|
69
|
+
|
|
70
|
+
function parseAlb(line) {
|
|
71
|
+
const f = tokenize(line);
|
|
72
|
+
if (f.length < 14 || !ALB_TYPES.has(f[0])) return null;
|
|
73
|
+
const req = splitRequestLine(f[12]);
|
|
74
|
+
if (!req) return null;
|
|
75
|
+
const time = Date.parse(f[1]);
|
|
76
|
+
return {
|
|
77
|
+
...req,
|
|
78
|
+
status: /^\d{3}$/.test(f[8]) ? Number(f[8]) : null,
|
|
79
|
+
time: Number.isNaN(time) ? null : time,
|
|
80
|
+
userAgent: nullIfDash(f[13]),
|
|
81
|
+
extras: [],
|
|
82
|
+
json: null,
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
// ─── JSON lines ─────────────────────────────────────────────────────────────
|
|
87
|
+
|
|
88
|
+
// Field names used by common producers: nginx/Apache JSON log_format, Kong
|
|
89
|
+
// file-log, Caddy, Traefik, Envoy, Google Cloud HTTP(S) LB, AWS API Gateway,
|
|
90
|
+
// ECS/OpenTelemetry, Elastic examples. Order matters — first non-empty wins,
|
|
91
|
+
// so nested keys come before same-named containers ("response.status" before
|
|
92
|
+
// "response").
|
|
93
|
+
const JSON_KEYS = {
|
|
94
|
+
method: ['method', 'request_method', 'httpMethod', 'http_method', 'verb', 'RequestMethod',
|
|
95
|
+
'request.method', 'req.method', 'http.method', 'http.request.method', 'httpRequest.requestMethod'],
|
|
96
|
+
path: ['path', 'uri', 'request_uri', 'url', 'RequestPath', 'request.uri', 'request.url', 'req.url',
|
|
97
|
+
'req.path', 'http.url', 'http.target', 'url.original', 'url.path', 'httpRequest.requestUrl'],
|
|
98
|
+
query: ['query', 'query_string', 'args', 'url.query'],
|
|
99
|
+
host: ['host', 'http_host', 'RequestHost', 'request.host', 'request.headers.host', 'request.headers.Host',
|
|
100
|
+
'req.headers.host', 'headers.host', 'url.domain', 'http.host', 'domainName'],
|
|
101
|
+
status: ['status', 'status_code', 'statusCode', 'response_code', 'DownstreamStatus', 'OriginStatus',
|
|
102
|
+
'response.status', 'res.statusCode', 'http.status_code', 'http.response.status_code',
|
|
103
|
+
'httpRequest.status', 'response'],
|
|
104
|
+
time: ['time', 'timestamp', '@timestamp', 'time_iso8601', 'time_local', 'requestTime',
|
|
105
|
+
'requestTimeEpoch', 'started_at', 'ts', 'start_time', 'StartUTC'],
|
|
106
|
+
userAgent: ['user_agent', 'http_user_agent', 'userAgent', 'agent', 'user_agent.original',
|
|
107
|
+
'request.headers.user-agent', 'request.headers.User-Agent', 'req.headers.user-agent',
|
|
108
|
+
'headers.user-agent', 'http.user_agent', 'request_User-Agent', 'httpRequest.userAgent'],
|
|
109
|
+
};
|
|
110
|
+
|
|
111
|
+
function parseJsonLine(line) {
|
|
112
|
+
const trimmed = line.trim();
|
|
113
|
+
if (!trimmed.startsWith('{')) return null;
|
|
114
|
+
let obj;
|
|
115
|
+
try { obj = JSON.parse(trimmed); } catch { return null; }
|
|
116
|
+
if (!obj || typeof obj !== 'object') return null;
|
|
117
|
+
|
|
118
|
+
let method = pick(obj, JSON_KEYS.method);
|
|
119
|
+
let target = pick(obj, JSON_KEYS.path);
|
|
120
|
+
|
|
121
|
+
// nginx JSON formats often log "$request" ("GET /x?y HTTP/1.1") instead of
|
|
122
|
+
// separate method and uri fields.
|
|
123
|
+
if ((!method || !target) && typeof obj.request === 'string') {
|
|
124
|
+
const req = splitRequestLine(obj.request);
|
|
125
|
+
if (req) { method = method || req.method; target = target || req.path + (req.query ? `?${req.query}` : ''); }
|
|
126
|
+
}
|
|
127
|
+
if (typeof method !== 'string' || typeof target !== 'string') return null;
|
|
128
|
+
method = method.toUpperCase();
|
|
129
|
+
if (!HTTP_METHODS.has(method)) return null;
|
|
130
|
+
|
|
131
|
+
const { path, query, host } = splitTarget(target);
|
|
132
|
+
if (!path) return null;
|
|
133
|
+
const explicitQuery = pick(obj, JSON_KEYS.query);
|
|
134
|
+
const status = Number(pick(obj, JSON_KEYS.status));
|
|
135
|
+
|
|
136
|
+
return {
|
|
137
|
+
method,
|
|
138
|
+
path,
|
|
139
|
+
query: query || (typeof explicitQuery === 'string' ? explicitQuery.replace(/^\?/, '') : ''),
|
|
140
|
+
host: host || normalizeHost(asString(pick(obj, JSON_KEYS.host))),
|
|
141
|
+
status: Number.isInteger(status) && status >= 100 && status <= 599 ? status : null,
|
|
142
|
+
time: parseAnyTime(pick(obj, JSON_KEYS.time)),
|
|
143
|
+
userAgent: nullIfDash(asString(pick(obj, JSON_KEYS.userAgent))),
|
|
144
|
+
extras: [],
|
|
145
|
+
json: obj,
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/** Value at a dotted path. Tries the literal key first ("user_agent.original" as one key). */
|
|
150
|
+
function lookup(obj, dotted) {
|
|
151
|
+
if (obj == null) return undefined;
|
|
152
|
+
if (Object.prototype.hasOwnProperty.call(obj, dotted)) return obj[dotted];
|
|
153
|
+
let cur = obj;
|
|
154
|
+
for (const part of dotted.split('.')) {
|
|
155
|
+
if (cur == null || typeof cur !== 'object') return undefined;
|
|
156
|
+
cur = cur[part];
|
|
157
|
+
}
|
|
158
|
+
return cur;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
function pick(obj, keys) {
|
|
162
|
+
for (const k of keys) {
|
|
163
|
+
const v = lookup(obj, k);
|
|
164
|
+
if (v !== undefined && v !== null && v !== '') return v;
|
|
165
|
+
}
|
|
166
|
+
return undefined;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
function parseAnyTime(v) {
|
|
170
|
+
if (v === undefined || v === null || v === '') return null;
|
|
171
|
+
if (typeof v === 'number' || /^\d+(\.\d+)?$/.test(String(v))) {
|
|
172
|
+
const n = Number(v);
|
|
173
|
+
return n > 1e12 ? Math.round(n) : Math.round(n * 1000); // ms vs seconds
|
|
174
|
+
}
|
|
175
|
+
const s = String(v);
|
|
176
|
+
const clf = parseClfTime(s);
|
|
177
|
+
if (clf !== null) return clf;
|
|
178
|
+
const t = Date.parse(s);
|
|
179
|
+
return Number.isNaN(t) ? null : t;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
// ─── Format detection ───────────────────────────────────────────────────────
|
|
183
|
+
|
|
184
|
+
const PARSERS = { combined: parseCombined, alb: parseAlb, json: parseJsonLine };
|
|
185
|
+
|
|
186
|
+
/** Picks the format that parses the most of the sample lines. */
|
|
187
|
+
function detectFormat(sampleLines) {
|
|
188
|
+
let best = null;
|
|
189
|
+
let bestHits = 0;
|
|
190
|
+
for (const [name, parse] of Object.entries(PARSERS)) {
|
|
191
|
+
const hits = sampleLines.filter((l) => parse(l)).length;
|
|
192
|
+
if (hits > bestHits) { best = name; bestHits = hits; }
|
|
193
|
+
}
|
|
194
|
+
return best;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
// ─── Helpers ────────────────────────────────────────────────────────────────
|
|
198
|
+
|
|
199
|
+
/** "GET /a/b?x=1 HTTP/1.1" (or an absolute URL as ALB logs it) → { method, path, query }. */
|
|
200
|
+
function splitRequestLine(s) {
|
|
201
|
+
if (typeof s !== 'string') return null;
|
|
202
|
+
const parts = s.trim().split(/\s+/);
|
|
203
|
+
if (parts.length < 2) return null;
|
|
204
|
+
const method = parts[0].toUpperCase();
|
|
205
|
+
if (!HTTP_METHODS.has(method)) return null;
|
|
206
|
+
const { path, query, host } = splitTarget(parts[1]);
|
|
207
|
+
if (!path) return null;
|
|
208
|
+
return { method, path, query, host };
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
function splitTarget(target) {
|
|
212
|
+
let t = target;
|
|
213
|
+
let host = null;
|
|
214
|
+
// Absolute URL (ALB, some proxies) → keep only the path + query, note the host.
|
|
215
|
+
const abs = /^[a-z][a-z0-9+.-]*:\/\/([^/?#]*)/i.exec(t);
|
|
216
|
+
if (abs) {
|
|
217
|
+
host = normalizeHost(abs[1]);
|
|
218
|
+
t = t.slice(abs[0].length) || '/';
|
|
219
|
+
}
|
|
220
|
+
if (!t.startsWith('/')) return { path: null, query: '', host };
|
|
221
|
+
const hash = t.indexOf('#');
|
|
222
|
+
if (hash >= 0) t = t.slice(0, hash);
|
|
223
|
+
const q = t.indexOf('?');
|
|
224
|
+
let path = q >= 0 ? t.slice(0, q) : t;
|
|
225
|
+
const query = q >= 0 ? t.slice(q + 1) : '';
|
|
226
|
+
path = path.replace(/\/{2,}/g, '/');
|
|
227
|
+
if (path.length > 1 && path.endsWith('/')) path = path.slice(0, -1);
|
|
228
|
+
return { path, query, host };
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
/** "API.Example.com:443" → "api.example.com"; "[::1]:80" → "[::1]". */
|
|
232
|
+
function normalizeHost(h) {
|
|
233
|
+
if (!h || h === '-') return null;
|
|
234
|
+
const s = String(h).trim().toLowerCase().replace(/^.*@/, '');
|
|
235
|
+
const m = /^(\[[^\]]+\]|[^:]+)(?::\d+)?$/.exec(s);
|
|
236
|
+
return m ? m[1] : null;
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
/** Query-string parameter NAMES only — values are never kept. */
|
|
240
|
+
function queryParamNames(query) {
|
|
241
|
+
if (!query) return [];
|
|
242
|
+
const names = new Set();
|
|
243
|
+
for (const pair of query.split('&')) {
|
|
244
|
+
if (!pair) continue;
|
|
245
|
+
const raw = pair.split('=')[0];
|
|
246
|
+
let name;
|
|
247
|
+
try { name = decodeURIComponent(raw.replace(/\+/g, ' ')); } catch { name = raw; }
|
|
248
|
+
name = name.replace(/\[\]$/, ''); // tags[]=a → tags
|
|
249
|
+
if (name && name.length <= 100) names.add(name);
|
|
250
|
+
}
|
|
251
|
+
return [...names];
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
/** Space-separated tokens; "double quoted" tokens may contain spaces and \" escapes. */
|
|
255
|
+
function tokenize(s) {
|
|
256
|
+
const out = [];
|
|
257
|
+
const re = /"((?:[^"\\]|\\.)*)"|(\S+)/g;
|
|
258
|
+
let m;
|
|
259
|
+
while ((m = re.exec(s)) !== null) out.push(m[1] !== undefined ? unescape(m[1]) : m[2]);
|
|
260
|
+
return out;
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
function unescape(s) {
|
|
264
|
+
return s.replace(/\\(["\\])/g, '$1');
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
function nullIfDash(s) {
|
|
268
|
+
return s === undefined || s === null || s === '' || s === '-' ? null : s;
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
function asString(v) {
|
|
272
|
+
if (v === undefined || v === null) return null;
|
|
273
|
+
return Array.isArray(v) ? String(v[0]) : String(v);
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
module.exports = {
|
|
277
|
+
PARSERS,
|
|
278
|
+
parseCombined,
|
|
279
|
+
parseAlb,
|
|
280
|
+
parseJsonLine,
|
|
281
|
+
detectFormat,
|
|
282
|
+
queryParamNames,
|
|
283
|
+
lookup,
|
|
284
|
+
splitTarget,
|
|
285
|
+
};
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Maps a concrete request path from a log ("/v1/orders/123/receipt") onto the
|
|
5
|
+
* OpenAPI path template that serves it ("/orders/{orderId}/receipt").
|
|
6
|
+
*
|
|
7
|
+
* Handles the two things that make naive string matching fail:
|
|
8
|
+
* - base paths: spec paths are relative to servers[].url (OpenAPI 3) or
|
|
9
|
+
* basePath (Swagger 2), while logs record the full path;
|
|
10
|
+
* - overlapping templates: /users/me and /users/{id} both match "/users/me",
|
|
11
|
+
* and the one with more literal segments wins, as in most routers.
|
|
12
|
+
*
|
|
13
|
+
* Paths that are already templated (usage.json written without --spec stores
|
|
14
|
+
* "/orders/{orderId}") also match: a "{…}" segment matches any template
|
|
15
|
+
* parameter, never a literal.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
const HTTP_METHODS = ['get', 'put', 'post', 'delete', 'options', 'head', 'patch', 'trace'];
|
|
19
|
+
|
|
20
|
+
function createRouteMatcher(spec, opts = {}) {
|
|
21
|
+
const routes = [];
|
|
22
|
+
for (const [template, item] of Object.entries((spec && spec.paths) || {})) {
|
|
23
|
+
if (!item || typeof item !== 'object') continue;
|
|
24
|
+
const methods = new Set(HTTP_METHODS.filter((m) => item[m]).map((m) => m.toUpperCase()));
|
|
25
|
+
if (methods.size === 0) continue;
|
|
26
|
+
const segs = splitPath(template);
|
|
27
|
+
routes.push({
|
|
28
|
+
template,
|
|
29
|
+
methods,
|
|
30
|
+
matchers: segs.map(segmentMatcher),
|
|
31
|
+
literals: segs.filter((s) => !s.includes('{')).length,
|
|
32
|
+
});
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const bySegCount = new Map();
|
|
36
|
+
for (const r of routes) {
|
|
37
|
+
const n = r.matchers.length;
|
|
38
|
+
if (!bySegCount.has(n)) bySegCount.set(n, []);
|
|
39
|
+
bySegCount.get(n).push(r);
|
|
40
|
+
}
|
|
41
|
+
for (const list of bySegCount.values()) list.sort((a, b) => b.literals - a.literals);
|
|
42
|
+
|
|
43
|
+
const basePaths = resolveBasePaths(spec, opts.basePath);
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* @returns {{ template, method, operation: boolean } | null}
|
|
47
|
+
* operation=false when the path exists but the spec doesn't define that method.
|
|
48
|
+
*/
|
|
49
|
+
function match(method, path) {
|
|
50
|
+
const upper = String(method).toUpperCase();
|
|
51
|
+
for (const prefix of basePaths) {
|
|
52
|
+
let rel;
|
|
53
|
+
if (prefix === '') rel = path;
|
|
54
|
+
else if (path === prefix) rel = '/';
|
|
55
|
+
else if (path.startsWith(prefix + '/')) rel = path.slice(prefix.length);
|
|
56
|
+
else continue;
|
|
57
|
+
|
|
58
|
+
const segs = splitPath(rel);
|
|
59
|
+
const candidates = bySegCount.get(segs.length) || [];
|
|
60
|
+
let pathOnly = null;
|
|
61
|
+
for (const r of candidates) {
|
|
62
|
+
if (!r.matchers.every((test, i) => test(segs[i]))) continue;
|
|
63
|
+
if (r.methods.has(upper)) return { template: r.template, method: upper, operation: true };
|
|
64
|
+
if (!pathOnly) pathOnly = { template: r.template, method: upper, operation: false };
|
|
65
|
+
}
|
|
66
|
+
if (pathOnly) return pathOnly;
|
|
67
|
+
}
|
|
68
|
+
return null;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
return {
|
|
72
|
+
match,
|
|
73
|
+
basePaths: basePaths.filter(Boolean),
|
|
74
|
+
operations: routes.flatMap((r) => [...r.methods].map((m) => ({ method: m, template: r.template }))),
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/** Longest base path first, always ending with '' (no prefix). */
|
|
79
|
+
function resolveBasePaths(spec, explicit) {
|
|
80
|
+
const found = new Set();
|
|
81
|
+
const add = (p) => {
|
|
82
|
+
if (typeof p !== 'string') return;
|
|
83
|
+
let clean = p.trim().replace(/\/{2,}/g, '/');
|
|
84
|
+
if (clean.endsWith('/')) clean = clean.slice(0, -1);
|
|
85
|
+
if (clean && clean.startsWith('/')) found.add(clean);
|
|
86
|
+
};
|
|
87
|
+
|
|
88
|
+
if (explicit) add(explicit);
|
|
89
|
+
if (spec && typeof spec.basePath === 'string') add(spec.basePath);
|
|
90
|
+
for (const server of (spec && Array.isArray(spec.servers) ? spec.servers : [])) {
|
|
91
|
+
if (!server || typeof server.url !== 'string') continue;
|
|
92
|
+
let url = server.url;
|
|
93
|
+
for (const [name, v] of Object.entries(server.variables || {})) {
|
|
94
|
+
if (v && v.default !== undefined) url = url.split(`{${name}}`).join(String(v.default));
|
|
95
|
+
}
|
|
96
|
+
if (url.includes('{')) continue; // unresolved variable — can't use it
|
|
97
|
+
if (/^[a-z][a-z0-9+.-]*:\/\//i.test(url)) {
|
|
98
|
+
try { add(new URL(url).pathname); } catch { /* malformed server url */ }
|
|
99
|
+
} else {
|
|
100
|
+
add(url.startsWith('/') ? url : null);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
return [...found].sort((a, b) => b.length - a.length).concat('');
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function splitPath(p) {
|
|
107
|
+
return String(p).split('/').filter(Boolean);
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/** One template segment → predicate. "{id}" any value; "v{n}" / "{id}.json" partial. */
|
|
111
|
+
function segmentMatcher(seg) {
|
|
112
|
+
if (!seg.includes('{')) {
|
|
113
|
+
return (s) => s === seg;
|
|
114
|
+
}
|
|
115
|
+
if (/^\{[^}]+\}$/.test(seg)) {
|
|
116
|
+
return (s) => typeof s === 'string' && s.length > 0;
|
|
117
|
+
}
|
|
118
|
+
const re = new RegExp('^' + seg.split(/\{[^}]+\}/).map(escapeRe).join('[^/]+') + '$');
|
|
119
|
+
// An already-templated segment ("{orderId}") counts as matching any parameter.
|
|
120
|
+
return (s) => typeof s === 'string' && (re.test(s) || /^\{[^}]+\}$/.test(s));
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
function escapeRe(s) {
|
|
124
|
+
return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
module.exports = { createRouteMatcher, resolveBasePaths };
|
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const fsExtra = require('fs-extra');
|
|
4
|
+
const { templatePath } = require('../har/pathTemplate');
|
|
5
|
+
const { queryParamNames } = require('./logParsers');
|
|
6
|
+
const { clientIdOf } = require('./clientIdentity');
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Aggregates request records into the usage summary written to usage.json.
|
|
10
|
+
*
|
|
11
|
+
* Only aggregates are kept: per (method, path template, client) a call count,
|
|
12
|
+
* first/last seen, and how often each query-parameter NAME was sent. Concrete
|
|
13
|
+
* paths (which carry IDs), query values, IPs and bodies are never stored — the
|
|
14
|
+
* file is safe to commit or upload.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
const FORMAT = 'specshield-usage';
|
|
18
|
+
const VERSION = 1;
|
|
19
|
+
const MAX_ENDPOINTS = 5000;
|
|
20
|
+
const MAX_UNMATCHED = 500;
|
|
21
|
+
const MAX_PARAMS = 50;
|
|
22
|
+
|
|
23
|
+
class UsageAggregator {
|
|
24
|
+
/**
|
|
25
|
+
* @param {object} opts
|
|
26
|
+
* @param {object} opts.clientSpec from parseClientSpec
|
|
27
|
+
* @param {object} [opts.matcher] from createRouteMatcher (import --spec)
|
|
28
|
+
* @param {number} [opts.sinceMs] drop records older than this epoch ms
|
|
29
|
+
*/
|
|
30
|
+
constructor({ clientSpec, matcher = null, sinceMs = null, hosts = [] }) {
|
|
31
|
+
this.clientSpec = clientSpec;
|
|
32
|
+
this.matcher = matcher;
|
|
33
|
+
this.sinceMs = sinceMs;
|
|
34
|
+
this.hostTest = compileHostFilter(hosts);
|
|
35
|
+
this.hostsSeen = new Map();
|
|
36
|
+
this.endpoints = new Map();
|
|
37
|
+
this.unmatchedCount = 0;
|
|
38
|
+
this.stats = { requests: 0, matched: 0, unmatched: 0, notFound: 0, tooOld: 0, otherHost: 0, overflow: 0 };
|
|
39
|
+
this.from = null;
|
|
40
|
+
this.to = null;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
add(record) {
|
|
44
|
+
// One load balancer often fronts several APIs by host name. Requests for
|
|
45
|
+
// another host must not count as usage of this API.
|
|
46
|
+
if (record.host) {
|
|
47
|
+
if (this.hostTest && !this.hostTest(record.host)) { this.stats.otherHost++; return; }
|
|
48
|
+
if (this.hostsSeen.size < 50 || this.hostsSeen.has(record.host)) {
|
|
49
|
+
this.hostsSeen.set(record.host, (this.hostsSeen.get(record.host) || 0) + 1);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
if (this.sinceMs !== null && record.time !== null && record.time < this.sinceMs) {
|
|
53
|
+
this.stats.tooOld++;
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
let path;
|
|
58
|
+
let matched = null;
|
|
59
|
+
if (this.matcher) {
|
|
60
|
+
const m = this.matcher.match(record.method, record.path);
|
|
61
|
+
if (m && m.operation) {
|
|
62
|
+
path = m.template;
|
|
63
|
+
matched = true;
|
|
64
|
+
} else {
|
|
65
|
+
// 404/405 on a path the spec doesn't know is almost always a scanner
|
|
66
|
+
// probing /wp-login.php and friends — not an undocumented endpoint.
|
|
67
|
+
if (record.status === 404 || record.status === 405) { this.stats.notFound++; return; }
|
|
68
|
+
path = templatePath(record.path).templated;
|
|
69
|
+
matched = false;
|
|
70
|
+
}
|
|
71
|
+
} else {
|
|
72
|
+
path = templatePath(record.path).templated;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
const key = `${record.method} ${path}`;
|
|
76
|
+
let ep = this.endpoints.get(key);
|
|
77
|
+
if (!ep) {
|
|
78
|
+
if (this.endpoints.size >= MAX_ENDPOINTS || (matched === false && this.unmatchedCount >= MAX_UNMATCHED)) {
|
|
79
|
+
this.stats.overflow++;
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
ep = { method: record.method, path, matched, clients: new Map() };
|
|
83
|
+
this.endpoints.set(key, ep);
|
|
84
|
+
if (matched === false) this.unmatchedCount++;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
const clientId = clientIdOf(record, this.clientSpec);
|
|
88
|
+
let c = ep.clients.get(clientId);
|
|
89
|
+
if (!c) {
|
|
90
|
+
c = { calls: 0, firstSeen: null, lastSeen: null, params: new Map(), paramsTruncated: false };
|
|
91
|
+
ep.clients.set(clientId, c);
|
|
92
|
+
}
|
|
93
|
+
c.calls++;
|
|
94
|
+
if (record.time !== null) {
|
|
95
|
+
if (c.firstSeen === null || record.time < c.firstSeen) c.firstSeen = record.time;
|
|
96
|
+
if (c.lastSeen === null || record.time > c.lastSeen) c.lastSeen = record.time;
|
|
97
|
+
if (this.from === null || record.time < this.from) this.from = record.time;
|
|
98
|
+
if (this.to === null || record.time > this.to) this.to = record.time;
|
|
99
|
+
}
|
|
100
|
+
for (const name of queryParamNames(record.query)) {
|
|
101
|
+
if (c.params.has(name)) c.params.set(name, c.params.get(name) + 1);
|
|
102
|
+
else if (c.params.size < MAX_PARAMS) c.params.set(name, 1);
|
|
103
|
+
else c.paramsTruncated = true;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
this.stats.requests++;
|
|
107
|
+
if (matched === true) this.stats.matched++;
|
|
108
|
+
else if (matched === false) this.stats.unmatched++;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/** Fold a previously written usage.json into this aggregate (import --append). */
|
|
112
|
+
absorb(usage) {
|
|
113
|
+
for (const e of usage.endpoints) {
|
|
114
|
+
const key = `${e.method} ${e.path}`;
|
|
115
|
+
let ep = this.endpoints.get(key);
|
|
116
|
+
if (!ep) {
|
|
117
|
+
ep = { method: e.method, path: e.path, matched: e.matched ?? null, clients: new Map() };
|
|
118
|
+
this.endpoints.set(key, ep);
|
|
119
|
+
if (ep.matched === false) this.unmatchedCount++;
|
|
120
|
+
}
|
|
121
|
+
for (const ec of e.clients) {
|
|
122
|
+
let c = ep.clients.get(ec.id);
|
|
123
|
+
if (!c) {
|
|
124
|
+
c = { calls: 0, firstSeen: null, lastSeen: null, params: new Map(), paramsTruncated: false };
|
|
125
|
+
ep.clients.set(ec.id, c);
|
|
126
|
+
}
|
|
127
|
+
c.calls += ec.calls;
|
|
128
|
+
c.firstSeen = minTime(c.firstSeen, toMs(ec.firstSeen));
|
|
129
|
+
c.lastSeen = maxTime(c.lastSeen, toMs(ec.lastSeen));
|
|
130
|
+
for (const [name, n] of Object.entries(ec.params || {})) {
|
|
131
|
+
if (c.params.has(name)) c.params.set(name, c.params.get(name) + n);
|
|
132
|
+
else if (c.params.size < MAX_PARAMS) c.params.set(name, n);
|
|
133
|
+
else c.paramsTruncated = true;
|
|
134
|
+
}
|
|
135
|
+
if (ec.paramsTruncated) c.paramsTruncated = true;
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
this.from = minTime(this.from, toMs(usage.window && usage.window.from));
|
|
139
|
+
this.to = maxTime(this.to, toMs(usage.window && usage.window.to));
|
|
140
|
+
for (const k of Object.keys(this.stats)) this.stats[k] += (usage.stats && usage.stats[k]) || 0;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
toJSON({ generator, client, spec, basePath, sources }) {
|
|
144
|
+
const endpoints = [...this.endpoints.values()].map((ep) => {
|
|
145
|
+
const clients = [...ep.clients.entries()]
|
|
146
|
+
.map(([id, c]) => {
|
|
147
|
+
const out = { id, calls: c.calls, firstSeen: toIso(c.firstSeen), lastSeen: toIso(c.lastSeen) };
|
|
148
|
+
if (c.params.size > 0) out.params = Object.fromEntries([...c.params.entries()].sort((a, b) => b[1] - a[1]));
|
|
149
|
+
if (c.paramsTruncated) out.paramsTruncated = true;
|
|
150
|
+
return out;
|
|
151
|
+
})
|
|
152
|
+
.sort((a, b) => b.calls - a.calls || a.id.localeCompare(b.id));
|
|
153
|
+
const calls = clients.reduce((n, c) => n + c.calls, 0);
|
|
154
|
+
const lastSeen = clients.reduce((t, c) => maxTime(t, toMs(c.lastSeen)), null);
|
|
155
|
+
return { method: ep.method, path: ep.path, matched: ep.matched, calls, lastSeen: toIso(lastSeen), clients };
|
|
156
|
+
}).sort((a, b) => b.calls - a.calls || `${a.path} ${a.method}`.localeCompare(`${b.path} ${b.method}`));
|
|
157
|
+
|
|
158
|
+
return {
|
|
159
|
+
format: FORMAT,
|
|
160
|
+
version: VERSION,
|
|
161
|
+
generatedAt: new Date().toISOString(),
|
|
162
|
+
generator,
|
|
163
|
+
client,
|
|
164
|
+
spec: spec || null,
|
|
165
|
+
basePath: basePath || null,
|
|
166
|
+
window: { from: toIso(this.from), to: toIso(this.to) },
|
|
167
|
+
stats: { ...this.stats },
|
|
168
|
+
sources,
|
|
169
|
+
endpoints,
|
|
170
|
+
};
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/** --host values: exact host names, or globs like "*.example.com". Null = no filter. */
|
|
175
|
+
function compileHostFilter(hosts) {
|
|
176
|
+
const list = (hosts || []).filter(Boolean).map((h) => String(h).toLowerCase());
|
|
177
|
+
if (list.length === 0) return null;
|
|
178
|
+
const tests = list.map((h) => {
|
|
179
|
+
if (!h.includes('*')) return (x) => x === h;
|
|
180
|
+
const re = new RegExp('^' + h.split('*').map((p) => p.replace(/[.+?^${}()|[\]\\]/g, '\\$&')).join('.*') + '$');
|
|
181
|
+
return (x) => re.test(x);
|
|
182
|
+
});
|
|
183
|
+
return (host) => tests.some((t) => t(host));
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/** Reads and validates a usage.json. Throws with a readable message. */
|
|
187
|
+
async function loadUsage(file) {
|
|
188
|
+
if (!(await fsExtra.pathExists(file))) throw new Error(`Usage file not found: ${file}`);
|
|
189
|
+
let data;
|
|
190
|
+
try {
|
|
191
|
+
data = JSON.parse(await fsExtra.readFile(file, 'utf8'));
|
|
192
|
+
} catch (err) {
|
|
193
|
+
throw new Error(`Usage file "${file}" is not valid JSON: ${err.message}`);
|
|
194
|
+
}
|
|
195
|
+
return validateUsage(data, file);
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
function validateUsage(data, file = 'usage.json') {
|
|
199
|
+
if (!data || data.format !== FORMAT || !Array.isArray(data.endpoints)) {
|
|
200
|
+
throw new Error(`"${file}" is not a SpecShield usage file. Create one with: specshield usage import <access.log>`);
|
|
201
|
+
}
|
|
202
|
+
if (data.version !== VERSION) {
|
|
203
|
+
throw new Error(`"${file}" is usage format v${data.version}; this CLI reads v${VERSION}. Re-run specshield usage import.`);
|
|
204
|
+
}
|
|
205
|
+
for (const e of data.endpoints) {
|
|
206
|
+
if (typeof e.method !== 'string' || typeof e.path !== 'string' || !Array.isArray(e.clients)) {
|
|
207
|
+
throw new Error(`"${file}" has a malformed endpoint entry`);
|
|
208
|
+
}
|
|
209
|
+
for (const c of e.clients) {
|
|
210
|
+
if (typeof c.id !== 'string' || !Number.isFinite(c.calls) || c.calls < 0) {
|
|
211
|
+
throw new Error(`"${file}" has a malformed client entry under ${e.method} ${e.path}`);
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
return data;
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/** "30d" / "12h" / "2w" / "45m" → milliseconds. */
|
|
219
|
+
function parseDuration(s) {
|
|
220
|
+
const m = /^(\d+)\s*(m|h|d|w)$/i.exec(String(s || '').trim());
|
|
221
|
+
if (!m) throw new Error(`Invalid duration "${s}". Use e.g. 30d, 12h, 2w`);
|
|
222
|
+
const unit = { m: 60e3, h: 3600e3, d: 86400e3, w: 7 * 86400e3 }[m[2].toLowerCase()];
|
|
223
|
+
return Number(m[1]) * unit;
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
function toMs(v) {
|
|
227
|
+
if (v === null || v === undefined) return null;
|
|
228
|
+
const t = typeof v === 'number' ? v : Date.parse(v);
|
|
229
|
+
return Number.isNaN(t) ? null : t;
|
|
230
|
+
}
|
|
231
|
+
function toIso(ms) { return ms === null || ms === undefined ? null : new Date(ms).toISOString(); }
|
|
232
|
+
function minTime(a, b) { return a === null ? b : b === null ? a : Math.min(a, b); }
|
|
233
|
+
function maxTime(a, b) { return a === null ? b : b === null ? a : Math.max(a, b); }
|
|
234
|
+
|
|
235
|
+
module.exports = { UsageAggregator, loadUsage, validateUsage, parseDuration, toMs, FORMAT, VERSION };
|