specshield 3.4.11 → 3.4.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,12 +1,14 @@
1
1
  'use strict';
2
2
 
3
3
  const chalk = require('chalk');
4
+ const { fmt, plural, pct, day, spanDays, ago } = require('./usage/format');
4
5
 
5
6
  function formatHuman(result) {
6
7
  const lines = [];
7
8
  const { breakingChanges, additions, modifications, warnings } = result;
9
+ const unused = result.unusedBreakingChanges || [];
8
10
 
9
- const total = breakingChanges.length + additions.length + modifications.length + warnings.length;
11
+ const total = breakingChanges.length + additions.length + modifications.length + warnings.length + unused.length;
10
12
 
11
13
  if (total === 0) {
12
14
  lines.push(chalk.green('✔ No changes detected between the two specs.'));
@@ -21,8 +23,10 @@ function formatHuman(result) {
21
23
  ` ${chalk.red.bold(`${breakingChanges.length} breaking`)} ` +
22
24
  `${chalk.green(`${additions.length} additions`)} ` +
23
25
  `${chalk.yellow(`${modifications.length} modifications`)} ` +
24
- `${chalk.gray(`${warnings.length} warnings`)}`
26
+ `${chalk.gray(`${warnings.length} warnings`)}` +
27
+ (result.usage ? ` ${chalk.green(`${unused.length} breaking but unused`)}` : '')
25
28
  );
29
+ if (result.usage) lines.push(...formatUsageHeader(result.usage));
26
30
  lines.push('');
27
31
 
28
32
  if (breakingChanges.length > 0) {
@@ -33,6 +37,21 @@ function formatHuman(result) {
33
37
  if (c.oldValue && c.newValue) {
34
38
  lines.push(` ${chalk.gray('from:')} ${chalk.red(c.oldValue)} ${chalk.gray('→')} ${chalk.green(c.newValue)}`);
35
39
  }
40
+ if (c.impact) lines.push(...formatImpact(c, { detailed: true }));
41
+ }
42
+ lines.push('');
43
+ }
44
+
45
+ if (unused.length > 0) {
46
+ const w = result.usage && result.usage.window;
47
+ lines.push(chalk.green.bold(' ✔ BREAKING BUT UNUSED — safe to ship'));
48
+ lines.push(chalk.gray(` 0 calls ${w && w.from ? `between ${day(w.from)} and ${day(w.to)}` : 'in the usage window'}; these do not fail the build`));
49
+ lines.push(chalk.gray(' ─────────────────────────────────────────'));
50
+ for (const c of unused) {
51
+ lines.push(` ${chalk.green('○')} ${c.description}`);
52
+ if (c.impact && c.impact.basis === 'query-parameter') {
53
+ lines.push(chalk.gray(` no client ${c.impact.parameterUse === 'omitted' ? 'omits' : 'sends'} "${c.impact.parameter}"`));
54
+ }
36
55
  }
37
56
  lines.push('');
38
57
  }
@@ -63,6 +82,7 @@ function formatHuman(result) {
63
82
  lines.push(chalk.gray(' ─────────────────────────────────────────'));
64
83
  for (const c of warnings) {
65
84
  lines.push(` ${chalk.gray('!')} ${c.description}`);
85
+ if (c.impact) lines.push(...formatImpact(c, { detailed: false }));
66
86
  }
67
87
  // Whether a constraint change actually breaks depends on which side of the
68
88
  // contract it sits on, which this engine does not track. Say so rather than
@@ -82,8 +102,74 @@ function formatHuman(result) {
82
102
  return lines.join('\n');
83
103
  }
84
104
 
105
+ // ─── Usage impact ───────────────────────────────────────────────────────────
106
+
107
+ const MAX_CLIENTS_SHOWN = 5;
108
+
109
+ function formatUsageHeader(u) {
110
+ const lines = [];
111
+ if (!u.trusted) {
112
+ lines.push(chalk.yellow(` Usage: ${u.file} was not applied — none of its calls match an operation in the base spec.`));
113
+ return lines;
114
+ }
115
+ const span = spanDays(u.window);
116
+ lines.push(chalk.gray(
117
+ ` Usage: ${u.file} · ${u.window.from ? `${day(u.window.from)} → ${day(u.window.to)}` : 'no timestamps'}` +
118
+ (span !== null ? ` (${span} days)` : '') +
119
+ ` · ${plural(u.matchedCalls, 'call')} · ${plural(u.clients, 'client')}` +
120
+ (u.operationsTotal ? ` · ${u.operationsSeen}/${u.operationsTotal} operations seen` : '') +
121
+ (u.ignoredCalls ? ` · ${fmt(u.ignoredCalls)} ignored` : '')));
122
+ if (!u.downgrade.enabled && u.downgrade.reason !== 'disabled by --no-usage-downgrade') {
123
+ lines.push(chalk.yellow(` Unused breaking changes still fail the build: ${u.downgrade.reason}.`));
124
+ }
125
+ return lines;
126
+ }
127
+
128
+ function formatImpact(c, { detailed }) {
129
+ const i = c.impact;
130
+ const lines = [];
131
+ if (i.verdict === 'unknown') {
132
+ lines.push(chalk.gray(` ↳ impact: unknown — ${i.reason}`));
133
+ return lines;
134
+ }
135
+ if (i.verdict === 'unused') {
136
+ lines.push(chalk.green(` ↳ impact: 0 calls${scope(i)} — nobody would notice this change`));
137
+ return lines;
138
+ }
139
+
140
+ const n = i.clients.length;
141
+ lines.push(chalk.yellow(
142
+ ` ↳ impact: ${plural(n, 'client')} · ${plural(i.calls, 'call')}${scope(i)}` +
143
+ ` · last seen ${ago(i.lastSeen)}`));
144
+ if (detailed) {
145
+ const width = Math.min(32, Math.max(...i.clients.slice(0, MAX_CLIENTS_SHOWN).map((x) => x.id.length)));
146
+ for (const cl of i.clients.slice(0, MAX_CLIENTS_SHOWN)) {
147
+ lines.push(
148
+ ` ${chalk.bold(cl.id.padEnd(width))} ${pct(cl.share).padStart(4)} of calls ` +
149
+ chalk.gray(`${fmt(cl.calls)} · last seen ${ago(cl.lastSeen)}`));
150
+ }
151
+ if (n > MAX_CLIENTS_SHOWN) lines.push(chalk.gray(` …and ${n - MAX_CLIENTS_SHOWN} more`));
152
+ if (i.basis === 'endpoint' && isFieldLevel(c)) {
153
+ lines.push(chalk.gray(' (clients calling this endpoint — access logs do not show which fields they use)'));
154
+ }
155
+ }
156
+ return lines;
157
+ }
158
+
159
+ function scope(i) {
160
+ if (i.basis !== 'query-parameter') return '';
161
+ return i.parameterUse === 'omitted'
162
+ ? ` without "${i.parameter}"`
163
+ : ` sending "${i.parameter}"`;
164
+ }
165
+
166
+ /** Changes below the operation level, where "calls the endpoint" over-approximates "is affected". */
167
+ function isFieldLevel(c) {
168
+ return !['ENDPOINT_REMOVED', 'METHOD_REMOVED'].includes(c.type);
169
+ }
170
+
85
171
  function formatJson(result) {
86
- return {
172
+ const out = {
87
173
  summary: {
88
174
  breaking: result.breakingChanges.length,
89
175
  additions: result.additions.length,
@@ -95,6 +181,14 @@ function formatJson(result) {
95
181
  modifications: result.modifications,
96
182
  warnings: result.warnings,
97
183
  };
184
+ // Additive only, and only with --usage: CI consumers parsing the existing
185
+ // shape see no difference unless they opt in.
186
+ if (result.usage) {
187
+ out.summary.unusedBreaking = (result.unusedBreakingChanges || []).length;
188
+ out.unusedBreakingChanges = result.unusedBreakingChanges || [];
189
+ out.usage = result.usage;
190
+ }
191
+ return out;
98
192
  }
99
193
 
100
194
  module.exports = { formatHuman, formatJson };
@@ -0,0 +1,70 @@
1
+ 'use strict';
2
+
3
+ const crypto = require('crypto');
4
+ const { lookup } = require('./logParsers');
5
+
6
+ /**
7
+ * Who made a request. The `--client` option picks the signal:
8
+ *
9
+ * ua User-Agent, reduced to "product/major.minor" (default)
10
+ * field:<dot.path> a field of a JSON log line, e.g. field:consumer.username (Kong)
11
+ * extra:<n> the n-th quoted field after the User-Agent in a combined
12
+ * log, e.g. an nginx log_format that appends "$http_x_client_id"
13
+ *
14
+ * Prefix any source with `hash:` (e.g. hash:field:request.headers.x-api-key)
15
+ * to store a one-way hash instead of the value. Use it for API keys: the key
16
+ * never lands in usage.json, but the same key always maps to the same client.
17
+ */
18
+ function parseClientSpec(spec) {
19
+ const raw = String(spec || 'ua').trim();
20
+ const hash = raw.startsWith('hash:');
21
+ const source = hash ? raw.slice(5) : raw;
22
+
23
+ if (source === 'ua') return { spec: raw, hash, kind: 'ua' };
24
+ let m = /^field:(.+)$/.exec(source);
25
+ if (m) return { spec: raw, hash, kind: 'field', field: m[1] };
26
+ m = /^extra:(\d+)$/.exec(source);
27
+ if (m && Number(m[1]) >= 1) return { spec: raw, hash, kind: 'extra', index: Number(m[1]) - 1 };
28
+
29
+ throw new Error(`Invalid --client "${raw}". Use ua, field:<path>, extra:<n> — optionally prefixed with hash:`);
30
+ }
31
+
32
+ function clientIdOf(record, clientSpec) {
33
+ let value;
34
+ if (clientSpec.kind === 'ua') {
35
+ value = normalizeUserAgent(record.userAgent);
36
+ if (value === null) return 'unknown';
37
+ } else if (clientSpec.kind === 'field') {
38
+ const v = record.json ? lookup(record.json, clientSpec.field) : undefined;
39
+ value = v === undefined || v === null || v === '' ? null : String(Array.isArray(v) ? v[0] : v);
40
+ } else {
41
+ value = record.extras[clientSpec.index] ?? null;
42
+ }
43
+ if (value === null) return 'unknown';
44
+ if (clientSpec.hash) return `hash:${crypto.createHash('sha256').update(value).digest('hex').slice(0, 12)}`;
45
+ return value.length > 120 ? value.slice(0, 120) : value;
46
+ }
47
+
48
+ /**
49
+ * "MyShop-iOS/5.2.1 CFNetwork/1410 Darwin/22.6" → "MyShop-iOS/5.2"
50
+ * "okhttp/4.12.0" → "okhttp/4.12"
51
+ * "Mozilla/5.0 (Windows NT 10.0; …) Chrome/…" → "browser"
52
+ *
53
+ * Keeping major.minor groups patch releases of one app version together —
54
+ * the useful question is "is anyone still on 5.2?", not "on 5.2.1 vs 5.2.3?".
55
+ */
56
+ function normalizeUserAgent(ua) {
57
+ if (!ua) return null;
58
+ const s = ua.trim();
59
+ if (!s) return null;
60
+ if (/^Mozilla\//.test(s)) return 'browser';
61
+ // The product is everything up to the first "/<digit>", so multi-word
62
+ // products survive: "Debian APT-HTTP/1.3 (…)" → "Debian APT-HTTP/1.3",
63
+ // "Chef Client/10.18.2 (…)" → "Chef Client/10.18".
64
+ const m = /^([^/();]{1,60}?)\/v?(\d+)(?:\.(\d+))?/.exec(s);
65
+ if (m) return `${m[1].trim()}/${m[2]}${m[3] !== undefined ? `.${m[3]}` : ''}`;
66
+ const name = s.split('(')[0].replace(/[\s;,]+$/, '') || s;
67
+ return name.length > 60 ? name.slice(0, 60) : name;
68
+ }
69
+
70
+ module.exports = { parseClientSpec, clientIdOf, normalizeUserAgent };
@@ -0,0 +1,31 @@
1
+ 'use strict';
2
+
3
+ /** Small formatting helpers shared by `usage show` and the compare impact report. */
4
+
5
+ function fmt(n) { return Number(n).toLocaleString('en-US'); }
6
+
7
+ /** plural(1, 'client') → "1 client"; plural(3, 'call') → "3 calls". */
8
+ function plural(n, word) { return `${fmt(n)} ${word}${n === 1 ? '' : 's'}`; }
9
+
10
+ function pct(x) {
11
+ if (x > 0 && x < 0.01) return '<1%';
12
+ return `${Math.round(x * 100)}%`;
13
+ }
14
+
15
+ function day(iso) { return iso ? iso.slice(0, 10) : '—'; }
16
+
17
+ function spanDays(w) {
18
+ if (!w || !w.from || !w.to) return null;
19
+ return Math.round(((Date.parse(w.to) - Date.parse(w.from)) / 86400e3) * 10) / 10;
20
+ }
21
+
22
+ function ago(iso, now = Date.now()) {
23
+ if (!iso) return 'unknown';
24
+ const s = Math.max(0, (now - Date.parse(iso)) / 1000);
25
+ if (s < 90) return 'just now';
26
+ if (s < 5400) return `${Math.round(s / 60)} min ago`;
27
+ if (s < 129600) return `${Math.round(s / 3600)} h ago`;
28
+ return `${Math.round(s / 86400)} days ago`;
29
+ }
30
+
31
+ module.exports = { fmt, plural, pct, day, spanDays, ago };
@@ -0,0 +1,243 @@
1
+ 'use strict';
2
+
3
+ const { createRouteMatcher } = require('./routeMatcher');
4
+ const { toMs } = require('./usageStore');
5
+
6
+ /**
7
+ * Joins a compare result to real traffic: every breaking change (and warning)
8
+ * gets an `impact` saying which clients actually exercise it.
9
+ *
10
+ * impact.verdict 'used' — at least one call in the usage window
11
+ * 'unused' — zero calls; nobody will notice the change
12
+ * 'unknown' — can't tie the change to an endpoint, or the
13
+ * endpoint isn't in the base spec
14
+ * impact.basis 'endpoint' — clients calling the operation. Logs
15
+ * don't show bodies, so for a body/response
16
+ * field change this means "may be affected"
17
+ * 'query-parameter' — clients that send (or, for a param that
18
+ * became required, omit) the parameter
19
+ *
20
+ * Breaking changes judged 'unused' move to `unusedBreakingChanges` and stop
21
+ * failing the build — but only when the evidence is good enough (see
22
+ * downgradeBlocker). Otherwise they are annotated and stay breaking.
23
+ */
24
+
25
+ const DAY = 86400e3;
26
+
27
+ function applyUsageImpact(result, opts) {
28
+ const {
29
+ usage, baseSpec, baseNorm, file = 'usage.json',
30
+ ignoreClients = [], minDays = 7, maxAgeDays = 30, minCoverage = 25,
31
+ downgrade = true, now = Date.now(),
32
+ } = opts;
33
+
34
+ const ignore = compileClientPatterns(ignoreClients);
35
+ const matcher = createRouteMatcher(baseSpec, { basePath: usage.basePath });
36
+
37
+ // "GET /orders/{id}" → Map(clientId → { calls, lastSeen, params, paramsTruncated })
38
+ const opIndex = new Map();
39
+ let totalCalls = 0;
40
+ let matchedCalls = 0;
41
+ let ignoredCalls = 0;
42
+ const clientIds = new Set();
43
+
44
+ for (const e of usage.endpoints) {
45
+ const m = matcher.match(e.method, e.path);
46
+ for (const c of e.clients) {
47
+ if (ignore(c.id)) { ignoredCalls += c.calls; continue; }
48
+ totalCalls += c.calls;
49
+ if (!m || !m.operation) continue;
50
+ matchedCalls += c.calls;
51
+ clientIds.add(c.id);
52
+ const key = `${m.method} ${m.template}`;
53
+ if (!opIndex.has(key)) opIndex.set(key, new Map());
54
+ const byClient = opIndex.get(key);
55
+ const agg = byClient.get(c.id) || { calls: 0, lastSeen: null, params: {}, paramsTruncated: false };
56
+ agg.calls += c.calls;
57
+ const seen = toMs(c.lastSeen);
58
+ if (seen !== null && (agg.lastSeen === null || seen > agg.lastSeen)) agg.lastSeen = seen;
59
+ for (const [name, n] of Object.entries(c.params || {})) agg.params[name] = (agg.params[name] || 0) + n;
60
+ if (c.paramsTruncated) agg.paramsTruncated = true;
61
+ byClient.set(c.id, agg);
62
+ }
63
+ }
64
+
65
+ const from = toMs(usage.window && usage.window.from);
66
+ const to = toMs(usage.window && usage.window.to);
67
+ const meta = {
68
+ file,
69
+ window: { from: usage.window ? usage.window.from : null, to: usage.window ? usage.window.to : null },
70
+ days: from !== null && to !== null ? round1((to - from) / DAY) : null,
71
+ staleDays: to !== null ? round1(Math.max(0, now - to) / DAY) : null,
72
+ totalCalls,
73
+ matchedCalls,
74
+ ignoredCalls,
75
+ clients: clientIds.size,
76
+ matchRate: totalCalls > 0 ? round3(matchedCalls / totalCalls) : 0,
77
+ basePaths: matcher.basePaths,
78
+ // Share of the spec's operations that appear in the logs. A low share
79
+ // usually means the logs are filtered (one route exported), sampled, or
80
+ // from one node of several — and then "0 calls" proves nothing.
81
+ operationsSeen: opIndex.size,
82
+ operationsTotal: matcher.operations.length,
83
+ trusted: matchedCalls > 0,
84
+ };
85
+
86
+ const blocker = downgradeBlocker(meta, { downgrade, minDays, maxAgeDays, minCoverage });
87
+ meta.downgrade = { enabled: blocker === null, reason: blocker };
88
+
89
+ const ctx = { opIndex, baseSpec, baseNorm, trusted: meta.trusted };
90
+ const annotate = (c) => (meta.trusted ? { ...c, impact: computeImpact(c, ctx) } : c);
91
+
92
+ let breakingChanges = result.breakingChanges.map(annotate);
93
+ const warnings = result.warnings.map(annotate);
94
+ let unusedBreakingChanges = [];
95
+
96
+ if (meta.downgrade.enabled) {
97
+ unusedBreakingChanges = breakingChanges.filter((c) => c.impact && c.impact.verdict === 'unused');
98
+ breakingChanges = breakingChanges.filter((c) => !(c.impact && c.impact.verdict === 'unused'));
99
+ }
100
+
101
+ return {
102
+ result: { ...result, breakingChanges, warnings, unusedBreakingChanges },
103
+ meta,
104
+ };
105
+ }
106
+
107
+ /** Why "zero calls" isn't strong enough evidence to stop failing the build — or null. */
108
+ function downgradeBlocker(meta, { downgrade, minDays, maxAgeDays, minCoverage }) {
109
+ if (!downgrade) return 'disabled by --no-usage-downgrade';
110
+ if (!meta.trusted) return 'no call in the usage file matches an operation in the base spec';
111
+ if (meta.days === null) return 'the logs had no timestamps, so the time window is unknown';
112
+ if (meta.days < minDays) {
113
+ return `the usage covers ${meta.days} day(s); at least ${minDays} are needed so weekly/monthly clients show up (--usage-min-days)`;
114
+ }
115
+ if (meta.staleDays > maxAgeDays) {
116
+ return `the newest log line is ${Math.floor(meta.staleDays)} days old — re-import recent logs (--usage-max-age)`;
117
+ }
118
+ const coverage = meta.operationsTotal > 0 ? (meta.operationsSeen / meta.operationsTotal) * 100 : 0;
119
+ if (coverage < minCoverage) {
120
+ return `only ${meta.operationsSeen} of ${meta.operationsTotal} operations in the spec appear in the logs — ` +
121
+ 'they look filtered, sampled or from one server of several (--usage-min-coverage)';
122
+ }
123
+ return null;
124
+ }
125
+
126
+ function computeImpact(change, { opIndex, baseSpec, baseNorm }) {
127
+ const endpoints = endpointsOf(change);
128
+ if (endpoints.length === 0) {
129
+ return { verdict: 'unknown', basis: null, reason: 'not tied to a specific endpoint' };
130
+ }
131
+ if (!endpoints.every(([m, p]) => operationExists(baseSpec, m, p))) {
132
+ return { verdict: 'unknown', basis: null, reason: 'endpoint not found in the base spec' };
133
+ }
134
+
135
+ // Query-parameter changes can be judged per parameter: logs record the
136
+ // query string. Header/path/cookie params and bodies can't — fall back to
137
+ // "who calls this endpoint".
138
+ let param = null;
139
+ const pm = /^parameters\.(.+)$/.exec(change.field || '');
140
+ if (pm && endpoints.length === 1) {
141
+ const p = findParam(baseNorm, endpoints[0][0], endpoints[0][1], pm[1]);
142
+ const byClient = opIndex.get(`${endpoints[0][0]} ${endpoints[0][1]}`);
143
+ const truncated = byClient && [...byClient.values()].some((c) => c.paramsTruncated);
144
+ // An object-typed query parameter (Spring's @ModelAttribute DTOs, form
145
+ // explode, deepObject) never appears in the URL under its own name — its
146
+ // properties do. Judging it by name would call a heavily used parameter
147
+ // unused, so judge it by who calls the endpoint instead.
148
+ if (p && p.in === 'query' && !isObjectSchema(p.schema) && !truncated) param = pm[1];
149
+ }
150
+ const omitters = change.type === 'PARAMETER_BECAME_REQUIRED';
151
+
152
+ const clients = new Map();
153
+ let endpointCalls = 0;
154
+ for (const [method, path] of endpoints) {
155
+ const byClient = opIndex.get(`${method} ${path}`);
156
+ if (!byClient) continue;
157
+ for (const [id, c] of byClient) {
158
+ endpointCalls += c.calls;
159
+ const sent = param ? (c.params[param] || 0) : 0;
160
+ const hit = !param ? c.calls : omitters ? c.calls - sent : sent;
161
+ if (hit <= 0) continue;
162
+ const agg = clients.get(id) || { id, calls: 0, lastSeen: null };
163
+ agg.calls += hit;
164
+ if (c.lastSeen !== null && (agg.lastSeen === null || c.lastSeen > agg.lastSeen)) agg.lastSeen = c.lastSeen;
165
+ clients.set(id, agg);
166
+ }
167
+ }
168
+
169
+ const list = [...clients.values()]
170
+ .sort((a, b) => b.calls - a.calls || a.id.localeCompare(b.id))
171
+ .map((c) => ({
172
+ id: c.id,
173
+ calls: c.calls,
174
+ share: endpointCalls > 0 ? round3(c.calls / endpointCalls) : 0,
175
+ lastSeen: c.lastSeen === null ? null : new Date(c.lastSeen).toISOString(),
176
+ }));
177
+ const calls = list.reduce((n, c) => n + c.calls, 0);
178
+ const lastSeen = list.reduce((t, c) => (c.lastSeen && (!t || c.lastSeen > t) ? c.lastSeen : t), null);
179
+
180
+ const impact = {
181
+ verdict: calls > 0 ? 'used' : 'unused',
182
+ basis: param ? 'query-parameter' : 'endpoint',
183
+ calls,
184
+ endpointCalls,
185
+ clients: list,
186
+ lastSeen,
187
+ };
188
+ if (param) {
189
+ impact.parameter = param;
190
+ if (omitters) impact.parameterUse = 'omitted';
191
+ else impact.parameterUse = 'sent';
192
+ }
193
+ return impact;
194
+ }
195
+
196
+ function endpointsOf(change) {
197
+ if (change.path && change.method) return [[change.method.toUpperCase(), change.path]];
198
+ const out = [];
199
+ for (const e of change.affectedEndpoints || []) {
200
+ const m = /^([A-Za-z]+) (\S+)$/.exec(e);
201
+ if (m) out.push([m[1].toUpperCase(), m[2]]);
202
+ }
203
+ return out;
204
+ }
205
+
206
+ function operationExists(spec, method, path) {
207
+ const item = spec && spec.paths && spec.paths[path];
208
+ return Boolean(item && item[method.toLowerCase()]);
209
+ }
210
+
211
+ function findParam(baseNorm, method, path, name) {
212
+ const op = baseNorm && baseNorm.endpoints && baseNorm.endpoints[path] && baseNorm.endpoints[path][method.toLowerCase()];
213
+ return (op && (op.parameters || []).find((x) => x.name === name)) || null;
214
+ }
215
+
216
+ /** Unknown schemas count as objects: when in doubt, don't judge by the name. */
217
+ function isObjectSchema(schema) {
218
+ if (!schema) return true;
219
+ // Scalars and arrays serialize under the parameter's own name (?tag=a&tag=b).
220
+ if (['string', 'integer', 'number', 'boolean', 'array'].includes(schema.type)) return false;
221
+ return true;
222
+ }
223
+
224
+ /**
225
+ * --usage-ignore-client patterns. With "*" the pattern is a glob over the whole
226
+ * id ("kube-probe/*"); without, a case-insensitive substring ("probe").
227
+ */
228
+ function compileClientPatterns(patterns) {
229
+ const tests = patterns.filter(Boolean).map((p) => {
230
+ if (p.includes('*')) {
231
+ const re = new RegExp('^' + p.split('*').map((s) => s.replace(/[.+?^${}()|[\]\\]/g, '\\$&')).join('.*') + '$', 'i');
232
+ return (id) => re.test(id);
233
+ }
234
+ const needle = p.toLowerCase();
235
+ return (id) => id.toLowerCase().includes(needle);
236
+ });
237
+ return (id) => tests.some((t) => t(id));
238
+ }
239
+
240
+ function round1(n) { return Math.round(n * 10) / 10; }
241
+ function round3(n) { return Math.round(n * 1000) / 1000; }
242
+
243
+ module.exports = { applyUsageImpact, compileClientPatterns };
@@ -0,0 +1,85 @@
1
+ 'use strict';
2
+
3
+ const fs = require('fs');
4
+ const path = require('path');
5
+ const zlib = require('zlib');
6
+ const crypto = require('crypto');
7
+ const readline = require('readline');
8
+ const { PARSERS, detectFormat } = require('./logParsers');
9
+
10
+ const SAMPLE_LINES = 50;
11
+
12
+ /**
13
+ * Streams one log file (or '-' for stdin; .gz transparently) into the
14
+ * aggregator. Lines are processed one at a time, so multi-GB logs are fine.
15
+ *
16
+ * @returns {{ file, format, lines, parsed, skipped, fingerprint }}
17
+ */
18
+ async function importLogFile(file, { format = 'auto', aggregator }) {
19
+ const isStdin = file === '-';
20
+ if (!isStdin && !fs.existsSync(file)) throw new Error(`Log file not found: ${file}`);
21
+ if (!isStdin && !fs.statSync(file).isFile()) throw new Error(`Not a file: ${file}`);
22
+ if (format !== 'auto' && !PARSERS[format]) {
23
+ throw new Error(`Unknown --format "${format}". Use auto, ${Object.keys(PARSERS).join(', ')}`);
24
+ }
25
+
26
+ const fingerprint = isStdin ? null : await fingerprintFile(file);
27
+ let input = isStdin ? process.stdin : fs.createReadStream(file);
28
+ if (!isStdin && /\.gz$/i.test(file)) input = input.pipe(zlib.createGunzip());
29
+
30
+ const rl = readline.createInterface({ input, crlfDelay: Infinity });
31
+ const stats = { lines: 0, parsed: 0, skipped: 0 };
32
+ let parse = format === 'auto' ? null : PARSERS[format];
33
+ let chosen = format === 'auto' ? null : format;
34
+ const sample = [];
35
+
36
+ const consume = (line) => {
37
+ const rec = parse(line);
38
+ if (rec) { stats.parsed++; aggregator.add(rec); } else stats.skipped++;
39
+ };
40
+
41
+ for await (const raw of rl) {
42
+ const line = raw.trim();
43
+ if (!line || line.startsWith('#')) continue; // blank / W3C-style header
44
+ stats.lines++;
45
+ if (parse) { consume(line); continue; }
46
+ sample.push(line);
47
+ if (sample.length >= SAMPLE_LINES) {
48
+ chosen = detectFormat(sample);
49
+ if (!chosen) throw unrecognised(file, sample);
50
+ parse = PARSERS[chosen];
51
+ sample.splice(0).forEach(consume);
52
+ }
53
+ }
54
+ if (!parse && sample.length > 0) {
55
+ chosen = detectFormat(sample);
56
+ if (!chosen) throw unrecognised(file, sample);
57
+ parse = PARSERS[chosen];
58
+ sample.splice(0).forEach(consume);
59
+ }
60
+
61
+ return { file: isStdin ? '<stdin>' : path.basename(file), format: chosen, ...stats, fingerprint };
62
+ }
63
+
64
+ function unrecognised(file, sample) {
65
+ const first = sample[0].length > 160 ? sample[0].slice(0, 160) + '…' : sample[0];
66
+ return new Error(
67
+ `Could not recognise the log format of ${file === '-' ? 'stdin' : file}.\n` +
68
+ ` First line: ${first}\n` +
69
+ ' Supported: nginx/Apache combined, AWS ALB, JSON lines. Force one with --format.');
70
+ }
71
+
72
+ /** size + sha256 of the first 64 KB — enough to spot the same file imported twice. */
73
+ async function fingerprintFile(file) {
74
+ const { size } = fs.statSync(file);
75
+ const fd = fs.openSync(file, 'r');
76
+ try {
77
+ const buf = Buffer.alloc(Math.min(size, 65536));
78
+ fs.readSync(fd, buf, 0, buf.length, 0);
79
+ return `${size}:${crypto.createHash('sha256').update(buf).digest('hex').slice(0, 16)}`;
80
+ } finally {
81
+ fs.closeSync(fd);
82
+ }
83
+ }
84
+
85
+ module.exports = { importLogFile, fingerprintFile };