specshield 3.4.10 → 3.4.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/README.md +161 -208
- package/package.json +1 -1
- package/src/cli.js +2 -0
- package/src/commands/compare.js +49 -0
- package/src/commands/usage.js +223 -0
- package/src/core/configLoader.js +25 -0
- package/src/core/outputFormatter.js +96 -3
- package/src/core/usage/clientIdentity.js +67 -0
- package/src/core/usage/format.js +31 -0
- package/src/core/usage/impact.js +222 -0
- package/src/core/usage/importLogs.js +85 -0
- package/src/core/usage/logParsers.js +262 -0
- package/src/core/usage/routeMatcher.js +127 -0
- package/src/core/usage/usageStore.js +213 -0
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const { createRouteMatcher } = require('./routeMatcher');
|
|
4
|
+
const { toMs } = require('./usageStore');
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Joins a compare result to real traffic: every breaking change (and warning)
|
|
8
|
+
* gets an `impact` saying which clients actually exercise it.
|
|
9
|
+
*
|
|
10
|
+
* impact.verdict 'used' — at least one call in the usage window
|
|
11
|
+
* 'unused' — zero calls; nobody will notice the change
|
|
12
|
+
* 'unknown' — can't tie the change to an endpoint, or the
|
|
13
|
+
* endpoint isn't in the base spec
|
|
14
|
+
* impact.basis 'endpoint' — clients calling the operation. Logs
|
|
15
|
+
* don't show bodies, so for a body/response
|
|
16
|
+
* field change this means "may be affected"
|
|
17
|
+
* 'query-parameter' — clients that send (or, for a param that
|
|
18
|
+
* became required, omit) the parameter
|
|
19
|
+
*
|
|
20
|
+
* Breaking changes judged 'unused' move to `unusedBreakingChanges` and stop
|
|
21
|
+
* failing the build — but only when the evidence is good enough (see
|
|
22
|
+
* downgradeBlocker). Otherwise they are annotated and stay breaking.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
const STALE_AFTER_DAYS = 30;
|
|
26
|
+
const DAY = 86400e3;
|
|
27
|
+
|
|
28
|
+
function applyUsageImpact(result, opts) {
|
|
29
|
+
const {
|
|
30
|
+
usage, baseSpec, baseNorm, file = 'usage.json',
|
|
31
|
+
ignoreClients = [], minDays = 7, downgrade = true, now = Date.now(),
|
|
32
|
+
} = opts;
|
|
33
|
+
|
|
34
|
+
const ignore = compileClientPatterns(ignoreClients);
|
|
35
|
+
const matcher = createRouteMatcher(baseSpec, { basePath: usage.basePath });
|
|
36
|
+
|
|
37
|
+
// "GET /orders/{id}" → Map(clientId → { calls, lastSeen, params, paramsTruncated })
|
|
38
|
+
const opIndex = new Map();
|
|
39
|
+
let totalCalls = 0;
|
|
40
|
+
let matchedCalls = 0;
|
|
41
|
+
let ignoredCalls = 0;
|
|
42
|
+
const clientIds = new Set();
|
|
43
|
+
|
|
44
|
+
for (const e of usage.endpoints) {
|
|
45
|
+
const m = matcher.match(e.method, e.path);
|
|
46
|
+
for (const c of e.clients) {
|
|
47
|
+
if (ignore(c.id)) { ignoredCalls += c.calls; continue; }
|
|
48
|
+
totalCalls += c.calls;
|
|
49
|
+
if (!m || !m.operation) continue;
|
|
50
|
+
matchedCalls += c.calls;
|
|
51
|
+
clientIds.add(c.id);
|
|
52
|
+
const key = `${m.method} ${m.template}`;
|
|
53
|
+
if (!opIndex.has(key)) opIndex.set(key, new Map());
|
|
54
|
+
const byClient = opIndex.get(key);
|
|
55
|
+
const agg = byClient.get(c.id) || { calls: 0, lastSeen: null, params: {}, paramsTruncated: false };
|
|
56
|
+
agg.calls += c.calls;
|
|
57
|
+
const seen = toMs(c.lastSeen);
|
|
58
|
+
if (seen !== null && (agg.lastSeen === null || seen > agg.lastSeen)) agg.lastSeen = seen;
|
|
59
|
+
for (const [name, n] of Object.entries(c.params || {})) agg.params[name] = (agg.params[name] || 0) + n;
|
|
60
|
+
if (c.paramsTruncated) agg.paramsTruncated = true;
|
|
61
|
+
byClient.set(c.id, agg);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
const from = toMs(usage.window && usage.window.from);
|
|
66
|
+
const to = toMs(usage.window && usage.window.to);
|
|
67
|
+
const meta = {
|
|
68
|
+
file,
|
|
69
|
+
window: { from: usage.window ? usage.window.from : null, to: usage.window ? usage.window.to : null },
|
|
70
|
+
days: from !== null && to !== null ? round1((to - from) / DAY) : null,
|
|
71
|
+
staleDays: to !== null ? round1(Math.max(0, now - to) / DAY) : null,
|
|
72
|
+
totalCalls,
|
|
73
|
+
matchedCalls,
|
|
74
|
+
ignoredCalls,
|
|
75
|
+
clients: clientIds.size,
|
|
76
|
+
matchRate: totalCalls > 0 ? round3(matchedCalls / totalCalls) : 0,
|
|
77
|
+
basePaths: matcher.basePaths,
|
|
78
|
+
trusted: matchedCalls > 0,
|
|
79
|
+
};
|
|
80
|
+
|
|
81
|
+
const blocker = downgradeBlocker(meta, { downgrade, minDays });
|
|
82
|
+
meta.downgrade = { enabled: blocker === null, reason: blocker };
|
|
83
|
+
|
|
84
|
+
const ctx = { opIndex, baseSpec, baseNorm, trusted: meta.trusted };
|
|
85
|
+
const annotate = (c) => (meta.trusted ? { ...c, impact: computeImpact(c, ctx) } : c);
|
|
86
|
+
|
|
87
|
+
let breakingChanges = result.breakingChanges.map(annotate);
|
|
88
|
+
const warnings = result.warnings.map(annotate);
|
|
89
|
+
let unusedBreakingChanges = [];
|
|
90
|
+
|
|
91
|
+
if (meta.downgrade.enabled) {
|
|
92
|
+
unusedBreakingChanges = breakingChanges.filter((c) => c.impact && c.impact.verdict === 'unused');
|
|
93
|
+
breakingChanges = breakingChanges.filter((c) => !(c.impact && c.impact.verdict === 'unused'));
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
return {
|
|
97
|
+
result: { ...result, breakingChanges, warnings, unusedBreakingChanges },
|
|
98
|
+
meta,
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** Why "zero calls" isn't strong enough evidence to stop failing the build — or null. */
|
|
103
|
+
function downgradeBlocker(meta, { downgrade, minDays }) {
|
|
104
|
+
if (!downgrade) return 'disabled by --no-usage-downgrade';
|
|
105
|
+
if (!meta.trusted) return 'no call in the usage file matches an operation in the base spec';
|
|
106
|
+
if (meta.days === null) return 'the logs had no timestamps, so the time window is unknown';
|
|
107
|
+
if (meta.days < minDays) {
|
|
108
|
+
return `the usage covers ${meta.days} day(s); at least ${minDays} are needed so weekly/monthly clients show up (--usage-min-days)`;
|
|
109
|
+
}
|
|
110
|
+
if (meta.staleDays > STALE_AFTER_DAYS) {
|
|
111
|
+
return `the newest log line is ${Math.floor(meta.staleDays)} days old — re-import recent logs`;
|
|
112
|
+
}
|
|
113
|
+
return null;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
function computeImpact(change, { opIndex, baseSpec, baseNorm }) {
|
|
117
|
+
const endpoints = endpointsOf(change);
|
|
118
|
+
if (endpoints.length === 0) {
|
|
119
|
+
return { verdict: 'unknown', basis: null, reason: 'not tied to a specific endpoint' };
|
|
120
|
+
}
|
|
121
|
+
if (!endpoints.every(([m, p]) => operationExists(baseSpec, m, p))) {
|
|
122
|
+
return { verdict: 'unknown', basis: null, reason: 'endpoint not found in the base spec' };
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
// Query-parameter changes can be judged per parameter: logs record the
|
|
126
|
+
// query string. Header/path/cookie params and bodies can't — fall back to
|
|
127
|
+
// "who calls this endpoint".
|
|
128
|
+
let param = null;
|
|
129
|
+
const pm = /^parameters\.(.+)$/.exec(change.field || '');
|
|
130
|
+
if (pm && endpoints.length === 1) {
|
|
131
|
+
const loc = paramLocation(baseNorm, endpoints[0][0], endpoints[0][1], pm[1]);
|
|
132
|
+
const byClient = opIndex.get(`${endpoints[0][0]} ${endpoints[0][1]}`);
|
|
133
|
+
const truncated = byClient && [...byClient.values()].some((c) => c.paramsTruncated);
|
|
134
|
+
if (loc === 'query' && !truncated) param = pm[1];
|
|
135
|
+
}
|
|
136
|
+
const omitters = change.type === 'PARAMETER_BECAME_REQUIRED';
|
|
137
|
+
|
|
138
|
+
const clients = new Map();
|
|
139
|
+
let endpointCalls = 0;
|
|
140
|
+
for (const [method, path] of endpoints) {
|
|
141
|
+
const byClient = opIndex.get(`${method} ${path}`);
|
|
142
|
+
if (!byClient) continue;
|
|
143
|
+
for (const [id, c] of byClient) {
|
|
144
|
+
endpointCalls += c.calls;
|
|
145
|
+
const sent = param ? (c.params[param] || 0) : 0;
|
|
146
|
+
const hit = !param ? c.calls : omitters ? c.calls - sent : sent;
|
|
147
|
+
if (hit <= 0) continue;
|
|
148
|
+
const agg = clients.get(id) || { id, calls: 0, lastSeen: null };
|
|
149
|
+
agg.calls += hit;
|
|
150
|
+
if (c.lastSeen !== null && (agg.lastSeen === null || c.lastSeen > agg.lastSeen)) agg.lastSeen = c.lastSeen;
|
|
151
|
+
clients.set(id, agg);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
const list = [...clients.values()]
|
|
156
|
+
.sort((a, b) => b.calls - a.calls || a.id.localeCompare(b.id))
|
|
157
|
+
.map((c) => ({
|
|
158
|
+
id: c.id,
|
|
159
|
+
calls: c.calls,
|
|
160
|
+
share: endpointCalls > 0 ? round3(c.calls / endpointCalls) : 0,
|
|
161
|
+
lastSeen: c.lastSeen === null ? null : new Date(c.lastSeen).toISOString(),
|
|
162
|
+
}));
|
|
163
|
+
const calls = list.reduce((n, c) => n + c.calls, 0);
|
|
164
|
+
const lastSeen = list.reduce((t, c) => (c.lastSeen && (!t || c.lastSeen > t) ? c.lastSeen : t), null);
|
|
165
|
+
|
|
166
|
+
const impact = {
|
|
167
|
+
verdict: calls > 0 ? 'used' : 'unused',
|
|
168
|
+
basis: param ? 'query-parameter' : 'endpoint',
|
|
169
|
+
calls,
|
|
170
|
+
endpointCalls,
|
|
171
|
+
clients: list,
|
|
172
|
+
lastSeen,
|
|
173
|
+
};
|
|
174
|
+
if (param) {
|
|
175
|
+
impact.parameter = param;
|
|
176
|
+
if (omitters) impact.parameterUse = 'omitted';
|
|
177
|
+
else impact.parameterUse = 'sent';
|
|
178
|
+
}
|
|
179
|
+
return impact;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
function endpointsOf(change) {
|
|
183
|
+
if (change.path && change.method) return [[change.method.toUpperCase(), change.path]];
|
|
184
|
+
const out = [];
|
|
185
|
+
for (const e of change.affectedEndpoints || []) {
|
|
186
|
+
const m = /^([A-Za-z]+) (\S+)$/.exec(e);
|
|
187
|
+
if (m) out.push([m[1].toUpperCase(), m[2]]);
|
|
188
|
+
}
|
|
189
|
+
return out;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
function operationExists(spec, method, path) {
|
|
193
|
+
const item = spec && spec.paths && spec.paths[path];
|
|
194
|
+
return Boolean(item && item[method.toLowerCase()]);
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
function paramLocation(baseNorm, method, path, name) {
|
|
198
|
+
const op = baseNorm && baseNorm.endpoints && baseNorm.endpoints[path] && baseNorm.endpoints[path][method.toLowerCase()];
|
|
199
|
+
const p = op && (op.parameters || []).find((x) => x.name === name);
|
|
200
|
+
return p ? p.in : null;
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/**
|
|
204
|
+
* --usage-ignore-client patterns. With "*" the pattern is a glob over the whole
|
|
205
|
+
* id ("kube-probe/*"); without, a case-insensitive substring ("probe").
|
|
206
|
+
*/
|
|
207
|
+
function compileClientPatterns(patterns) {
|
|
208
|
+
const tests = patterns.filter(Boolean).map((p) => {
|
|
209
|
+
if (p.includes('*')) {
|
|
210
|
+
const re = new RegExp('^' + p.split('*').map((s) => s.replace(/[.+?^${}()|[\]\\]/g, '\\$&')).join('.*') + '$', 'i');
|
|
211
|
+
return (id) => re.test(id);
|
|
212
|
+
}
|
|
213
|
+
const needle = p.toLowerCase();
|
|
214
|
+
return (id) => id.toLowerCase().includes(needle);
|
|
215
|
+
});
|
|
216
|
+
return (id) => tests.some((t) => t(id));
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
function round1(n) { return Math.round(n * 10) / 10; }
|
|
220
|
+
function round3(n) { return Math.round(n * 1000) / 1000; }
|
|
221
|
+
|
|
222
|
+
module.exports = { applyUsageImpact, compileClientPatterns };
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const fs = require('fs');
|
|
4
|
+
const path = require('path');
|
|
5
|
+
const zlib = require('zlib');
|
|
6
|
+
const crypto = require('crypto');
|
|
7
|
+
const readline = require('readline');
|
|
8
|
+
const { PARSERS, detectFormat } = require('./logParsers');
|
|
9
|
+
|
|
10
|
+
const SAMPLE_LINES = 50;
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Streams one log file (or '-' for stdin; .gz transparently) into the
|
|
14
|
+
* aggregator. Lines are processed one at a time, so multi-GB logs are fine.
|
|
15
|
+
*
|
|
16
|
+
* @returns {{ file, format, lines, parsed, skipped, fingerprint }}
|
|
17
|
+
*/
|
|
18
|
+
async function importLogFile(file, { format = 'auto', aggregator }) {
|
|
19
|
+
const isStdin = file === '-';
|
|
20
|
+
if (!isStdin && !fs.existsSync(file)) throw new Error(`Log file not found: ${file}`);
|
|
21
|
+
if (!isStdin && !fs.statSync(file).isFile()) throw new Error(`Not a file: ${file}`);
|
|
22
|
+
if (format !== 'auto' && !PARSERS[format]) {
|
|
23
|
+
throw new Error(`Unknown --format "${format}". Use auto, ${Object.keys(PARSERS).join(', ')}`);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
const fingerprint = isStdin ? null : await fingerprintFile(file);
|
|
27
|
+
let input = isStdin ? process.stdin : fs.createReadStream(file);
|
|
28
|
+
if (!isStdin && /\.gz$/i.test(file)) input = input.pipe(zlib.createGunzip());
|
|
29
|
+
|
|
30
|
+
const rl = readline.createInterface({ input, crlfDelay: Infinity });
|
|
31
|
+
const stats = { lines: 0, parsed: 0, skipped: 0 };
|
|
32
|
+
let parse = format === 'auto' ? null : PARSERS[format];
|
|
33
|
+
let chosen = format === 'auto' ? null : format;
|
|
34
|
+
const sample = [];
|
|
35
|
+
|
|
36
|
+
const consume = (line) => {
|
|
37
|
+
const rec = parse(line);
|
|
38
|
+
if (rec) { stats.parsed++; aggregator.add(rec); } else stats.skipped++;
|
|
39
|
+
};
|
|
40
|
+
|
|
41
|
+
for await (const raw of rl) {
|
|
42
|
+
const line = raw.trim();
|
|
43
|
+
if (!line || line.startsWith('#')) continue; // blank / W3C-style header
|
|
44
|
+
stats.lines++;
|
|
45
|
+
if (parse) { consume(line); continue; }
|
|
46
|
+
sample.push(line);
|
|
47
|
+
if (sample.length >= SAMPLE_LINES) {
|
|
48
|
+
chosen = detectFormat(sample);
|
|
49
|
+
if (!chosen) throw unrecognised(file, sample);
|
|
50
|
+
parse = PARSERS[chosen];
|
|
51
|
+
sample.splice(0).forEach(consume);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
if (!parse && sample.length > 0) {
|
|
55
|
+
chosen = detectFormat(sample);
|
|
56
|
+
if (!chosen) throw unrecognised(file, sample);
|
|
57
|
+
parse = PARSERS[chosen];
|
|
58
|
+
sample.splice(0).forEach(consume);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
return { file: isStdin ? '<stdin>' : path.basename(file), format: chosen, ...stats, fingerprint };
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function unrecognised(file, sample) {
|
|
65
|
+
const first = sample[0].length > 160 ? sample[0].slice(0, 160) + '…' : sample[0];
|
|
66
|
+
return new Error(
|
|
67
|
+
`Could not recognise the log format of ${file === '-' ? 'stdin' : file}.\n` +
|
|
68
|
+
` First line: ${first}\n` +
|
|
69
|
+
' Supported: nginx/Apache combined, AWS ALB, JSON lines. Force one with --format.');
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** size + sha256 of the first 64 KB — enough to spot the same file imported twice. */
|
|
73
|
+
async function fingerprintFile(file) {
|
|
74
|
+
const { size } = fs.statSync(file);
|
|
75
|
+
const fd = fs.openSync(file, 'r');
|
|
76
|
+
try {
|
|
77
|
+
const buf = Buffer.alloc(Math.min(size, 65536));
|
|
78
|
+
fs.readSync(fd, buf, 0, buf.length, 0);
|
|
79
|
+
return `${size}:${crypto.createHash('sha256').update(buf).digest('hex').slice(0, 16)}`;
|
|
80
|
+
} finally {
|
|
81
|
+
fs.closeSync(fd);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
module.exports = { importLogFile, fingerprintFile };
|
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Access-log line parsers for `specshield usage import`.
|
|
5
|
+
*
|
|
6
|
+
* Every parser turns one line into a request record, or null when the line
|
|
7
|
+
* isn't a request it understands (blank, comment, health-check noise in an
|
|
8
|
+
* unknown shape):
|
|
9
|
+
*
|
|
10
|
+
* { method, path, query, status, time, userAgent, extras, json }
|
|
11
|
+
*
|
|
12
|
+
* method upper-case HTTP method
|
|
13
|
+
* path URL path only — no scheme/host, no query string
|
|
14
|
+
* query raw query string without the leading '?' ('' when absent)
|
|
15
|
+
* status numeric status code, or null
|
|
16
|
+
* time epoch ms, or null when the line carries no parseable timestamp
|
|
17
|
+
* userAgent raw User-Agent, or null
|
|
18
|
+
* extras quoted fields after the User-Agent (combined format only) —
|
|
19
|
+
* where nginx `log_format` additions such as $http_x_api_key land
|
|
20
|
+
* json the parsed object (json format only), for --client field:<path>
|
|
21
|
+
*
|
|
22
|
+
* Supported formats:
|
|
23
|
+
* combined nginx / Apache "combined" and "common" log formats
|
|
24
|
+
* alb AWS Application Load Balancer access logs
|
|
25
|
+
* json one JSON object per line — nginx JSON log_format, Kong file-log,
|
|
26
|
+
* API Gateway JSON access logs, Envoy/Traefik JSON, ECS-style logs
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
const HTTP_METHODS = new Set(['GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS', 'TRACE', 'CONNECT']);
|
|
30
|
+
|
|
31
|
+
// ─── combined / common ──────────────────────────────────────────────────────
|
|
32
|
+
|
|
33
|
+
// host ident authuser [time] "request" status bytes ["referer" "user-agent"] [extras...]
|
|
34
|
+
const COMBINED_RE = /^(\S+) \S+ \S+ \[([^\]]+)\] "((?:[^"\\]|\\.)*)" (\d{3}|-) \S+(?: "((?:[^"\\]|\\.)*)" "((?:[^"\\]|\\.)*)")?(.*)$/;
|
|
35
|
+
|
|
36
|
+
function parseCombined(line) {
|
|
37
|
+
const m = COMBINED_RE.exec(line);
|
|
38
|
+
if (!m) return null;
|
|
39
|
+
const req = splitRequestLine(unescape(m[3]));
|
|
40
|
+
if (!req) return null;
|
|
41
|
+
return {
|
|
42
|
+
...req,
|
|
43
|
+
status: m[4] === '-' ? null : Number(m[4]),
|
|
44
|
+
time: parseClfTime(m[2]),
|
|
45
|
+
userAgent: m[6] !== undefined ? nullIfDash(unescape(m[6])) : null,
|
|
46
|
+
extras: m[7] ? tokenize(m[7]).map(nullIfDash) : [],
|
|
47
|
+
json: null,
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const MONTHS = { Jan: 0, Feb: 1, Mar: 2, Apr: 3, May: 4, Jun: 5, Jul: 6, Aug: 7, Sep: 8, Oct: 9, Nov: 10, Dec: 11 };
|
|
52
|
+
|
|
53
|
+
/** "10/Oct/2026:13:55:36 +0530" → epoch ms. */
|
|
54
|
+
function parseClfTime(s) {
|
|
55
|
+
const m = /^(\d{2})\/(\w{3})\/(\d{4}):(\d{2}):(\d{2}):(\d{2})(?: ([+-])(\d{2})(\d{2}))?$/.exec(s || '');
|
|
56
|
+
if (!m || MONTHS[m[2]] === undefined) return null;
|
|
57
|
+
const utc = Date.UTC(+m[3], MONTHS[m[2]], +m[1], +m[4], +m[5], +m[6]);
|
|
58
|
+
const offsetMin = m[7] ? (m[7] === '-' ? -1 : 1) * (+m[8] * 60 + +m[9]) : 0;
|
|
59
|
+
return utc - offsetMin * 60000;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
// ─── AWS ALB ────────────────────────────────────────────────────────────────
|
|
63
|
+
|
|
64
|
+
// type time elb client:port target:port req_t target_t resp_t elb_status
|
|
65
|
+
// target_status recv sent "request" "user_agent" ...
|
|
66
|
+
const ALB_TYPES = new Set(['http', 'https', 'h2', 'grpcs', 'ws', 'wss']);
|
|
67
|
+
|
|
68
|
+
function parseAlb(line) {
|
|
69
|
+
const f = tokenize(line);
|
|
70
|
+
if (f.length < 14 || !ALB_TYPES.has(f[0])) return null;
|
|
71
|
+
const req = splitRequestLine(f[12]);
|
|
72
|
+
if (!req) return null;
|
|
73
|
+
const time = Date.parse(f[1]);
|
|
74
|
+
return {
|
|
75
|
+
...req,
|
|
76
|
+
status: /^\d{3}$/.test(f[8]) ? Number(f[8]) : null,
|
|
77
|
+
time: Number.isNaN(time) ? null : time,
|
|
78
|
+
userAgent: nullIfDash(f[13]),
|
|
79
|
+
extras: [],
|
|
80
|
+
json: null,
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
// ─── JSON lines ─────────────────────────────────────────────────────────────
|
|
85
|
+
|
|
86
|
+
const JSON_KEYS = {
|
|
87
|
+
method: ['method', 'request_method', 'httpMethod', 'http_method', 'verb',
|
|
88
|
+
'request.method', 'req.method', 'http.method', 'http.request.method'],
|
|
89
|
+
path: ['path', 'uri', 'request_uri', 'url', 'request.uri', 'request.url', 'req.url',
|
|
90
|
+
'req.path', 'http.url', 'http.target', 'url.original', 'url.path'],
|
|
91
|
+
query: ['query', 'query_string', 'args', 'url.query'],
|
|
92
|
+
status: ['status', 'status_code', 'statusCode', 'response.status', 'res.statusCode',
|
|
93
|
+
'http.status_code', 'http.response.status_code'],
|
|
94
|
+
time: ['time', 'timestamp', '@timestamp', 'time_iso8601', 'time_local', 'requestTime',
|
|
95
|
+
'requestTimeEpoch', 'started_at', 'ts', 'start_time'],
|
|
96
|
+
userAgent: ['user_agent', 'http_user_agent', 'userAgent', 'user_agent.original',
|
|
97
|
+
'request.headers.user-agent', 'req.headers.user-agent', 'headers.user-agent',
|
|
98
|
+
'http.user_agent', 'request.headers.User-Agent'],
|
|
99
|
+
};
|
|
100
|
+
|
|
101
|
+
function parseJsonLine(line) {
|
|
102
|
+
const trimmed = line.trim();
|
|
103
|
+
if (!trimmed.startsWith('{')) return null;
|
|
104
|
+
let obj;
|
|
105
|
+
try { obj = JSON.parse(trimmed); } catch { return null; }
|
|
106
|
+
if (!obj || typeof obj !== 'object') return null;
|
|
107
|
+
|
|
108
|
+
let method = pick(obj, JSON_KEYS.method);
|
|
109
|
+
let target = pick(obj, JSON_KEYS.path);
|
|
110
|
+
|
|
111
|
+
// nginx JSON formats often log "$request" ("GET /x?y HTTP/1.1") instead of
|
|
112
|
+
// separate method and uri fields.
|
|
113
|
+
if ((!method || !target) && typeof obj.request === 'string') {
|
|
114
|
+
const req = splitRequestLine(obj.request);
|
|
115
|
+
if (req) { method = method || req.method; target = target || req.path + (req.query ? `?${req.query}` : ''); }
|
|
116
|
+
}
|
|
117
|
+
if (typeof method !== 'string' || typeof target !== 'string') return null;
|
|
118
|
+
method = method.toUpperCase();
|
|
119
|
+
if (!HTTP_METHODS.has(method)) return null;
|
|
120
|
+
|
|
121
|
+
const { path, query } = splitTarget(target);
|
|
122
|
+
if (!path) return null;
|
|
123
|
+
const explicitQuery = pick(obj, JSON_KEYS.query);
|
|
124
|
+
const status = Number(pick(obj, JSON_KEYS.status));
|
|
125
|
+
|
|
126
|
+
return {
|
|
127
|
+
method,
|
|
128
|
+
path,
|
|
129
|
+
query: query || (typeof explicitQuery === 'string' ? explicitQuery.replace(/^\?/, '') : ''),
|
|
130
|
+
status: Number.isInteger(status) && status >= 100 && status <= 599 ? status : null,
|
|
131
|
+
time: parseAnyTime(pick(obj, JSON_KEYS.time)),
|
|
132
|
+
userAgent: nullIfDash(asString(pick(obj, JSON_KEYS.userAgent))),
|
|
133
|
+
extras: [],
|
|
134
|
+
json: obj,
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/** Value at a dotted path. Tries the literal key first ("user_agent.original" as one key). */
|
|
139
|
+
function lookup(obj, dotted) {
|
|
140
|
+
if (obj == null) return undefined;
|
|
141
|
+
if (Object.prototype.hasOwnProperty.call(obj, dotted)) return obj[dotted];
|
|
142
|
+
let cur = obj;
|
|
143
|
+
for (const part of dotted.split('.')) {
|
|
144
|
+
if (cur == null || typeof cur !== 'object') return undefined;
|
|
145
|
+
cur = cur[part];
|
|
146
|
+
}
|
|
147
|
+
return cur;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
function pick(obj, keys) {
|
|
151
|
+
for (const k of keys) {
|
|
152
|
+
const v = lookup(obj, k);
|
|
153
|
+
if (v !== undefined && v !== null && v !== '') return v;
|
|
154
|
+
}
|
|
155
|
+
return undefined;
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
function parseAnyTime(v) {
|
|
159
|
+
if (v === undefined || v === null || v === '') return null;
|
|
160
|
+
if (typeof v === 'number' || /^\d+(\.\d+)?$/.test(String(v))) {
|
|
161
|
+
const n = Number(v);
|
|
162
|
+
return n > 1e12 ? Math.round(n) : Math.round(n * 1000); // ms vs seconds
|
|
163
|
+
}
|
|
164
|
+
const s = String(v);
|
|
165
|
+
const clf = parseClfTime(s);
|
|
166
|
+
if (clf !== null) return clf;
|
|
167
|
+
const t = Date.parse(s);
|
|
168
|
+
return Number.isNaN(t) ? null : t;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
// ─── Format detection ───────────────────────────────────────────────────────
|
|
172
|
+
|
|
173
|
+
const PARSERS = { combined: parseCombined, alb: parseAlb, json: parseJsonLine };
|
|
174
|
+
|
|
175
|
+
/** Picks the format that parses the most of the sample lines. */
|
|
176
|
+
function detectFormat(sampleLines) {
|
|
177
|
+
let best = null;
|
|
178
|
+
let bestHits = 0;
|
|
179
|
+
for (const [name, parse] of Object.entries(PARSERS)) {
|
|
180
|
+
const hits = sampleLines.filter((l) => parse(l)).length;
|
|
181
|
+
if (hits > bestHits) { best = name; bestHits = hits; }
|
|
182
|
+
}
|
|
183
|
+
return best;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
// ─── Helpers ────────────────────────────────────────────────────────────────
|
|
187
|
+
|
|
188
|
+
/** "GET /a/b?x=1 HTTP/1.1" (or an absolute URL as ALB logs it) → { method, path, query }. */
|
|
189
|
+
function splitRequestLine(s) {
|
|
190
|
+
if (typeof s !== 'string') return null;
|
|
191
|
+
const parts = s.trim().split(/\s+/);
|
|
192
|
+
if (parts.length < 2) return null;
|
|
193
|
+
const method = parts[0].toUpperCase();
|
|
194
|
+
if (!HTTP_METHODS.has(method)) return null;
|
|
195
|
+
const { path, query } = splitTarget(parts[1]);
|
|
196
|
+
if (!path) return null;
|
|
197
|
+
return { method, path, query };
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
function splitTarget(target) {
|
|
201
|
+
let t = target;
|
|
202
|
+
// Absolute URL (ALB, some proxies) → keep only the path + query.
|
|
203
|
+
const abs = /^[a-z][a-z0-9+.-]*:\/\/[^/?#]*/i.exec(t);
|
|
204
|
+
if (abs) t = t.slice(abs[0].length) || '/';
|
|
205
|
+
if (!t.startsWith('/')) return { path: null, query: '' };
|
|
206
|
+
const hash = t.indexOf('#');
|
|
207
|
+
if (hash >= 0) t = t.slice(0, hash);
|
|
208
|
+
const q = t.indexOf('?');
|
|
209
|
+
let path = q >= 0 ? t.slice(0, q) : t;
|
|
210
|
+
const query = q >= 0 ? t.slice(q + 1) : '';
|
|
211
|
+
path = path.replace(/\/{2,}/g, '/');
|
|
212
|
+
if (path.length > 1 && path.endsWith('/')) path = path.slice(0, -1);
|
|
213
|
+
return { path, query };
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/** Query-string parameter NAMES only — values are never kept. */
|
|
217
|
+
function queryParamNames(query) {
|
|
218
|
+
if (!query) return [];
|
|
219
|
+
const names = new Set();
|
|
220
|
+
for (const pair of query.split('&')) {
|
|
221
|
+
if (!pair) continue;
|
|
222
|
+
const raw = pair.split('=')[0];
|
|
223
|
+
let name;
|
|
224
|
+
try { name = decodeURIComponent(raw.replace(/\+/g, ' ')); } catch { name = raw; }
|
|
225
|
+
name = name.replace(/\[\]$/, ''); // tags[]=a → tags
|
|
226
|
+
if (name && name.length <= 100) names.add(name);
|
|
227
|
+
}
|
|
228
|
+
return [...names];
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
/** Space-separated tokens; "double quoted" tokens may contain spaces and \" escapes. */
|
|
232
|
+
function tokenize(s) {
|
|
233
|
+
const out = [];
|
|
234
|
+
const re = /"((?:[^"\\]|\\.)*)"|(\S+)/g;
|
|
235
|
+
let m;
|
|
236
|
+
while ((m = re.exec(s)) !== null) out.push(m[1] !== undefined ? unescape(m[1]) : m[2]);
|
|
237
|
+
return out;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
function unescape(s) {
|
|
241
|
+
return s.replace(/\\(["\\])/g, '$1');
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
function nullIfDash(s) {
|
|
245
|
+
return s === undefined || s === null || s === '' || s === '-' ? null : s;
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
function asString(v) {
|
|
249
|
+
if (v === undefined || v === null) return null;
|
|
250
|
+
return Array.isArray(v) ? String(v[0]) : String(v);
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
module.exports = {
|
|
254
|
+
PARSERS,
|
|
255
|
+
parseCombined,
|
|
256
|
+
parseAlb,
|
|
257
|
+
parseJsonLine,
|
|
258
|
+
detectFormat,
|
|
259
|
+
queryParamNames,
|
|
260
|
+
lookup,
|
|
261
|
+
splitTarget,
|
|
262
|
+
};
|