specshield 3.4.13 → 3.4.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +55 -1
- package/README.md +114 -4
- package/package.json +1 -1
- package/src/api/bdctClient.js +8 -0
- package/src/cli.js +2 -0
- package/src/commands/compare.js +7 -1
- package/src/commands/deprecations.js +236 -0
- package/src/core/configLoader.js +6 -1
- package/src/core/normalizeSpec.js +67 -11
- package/src/core/outputFormatter.js +22 -1
- package/src/core/usage/deprecations.js +438 -0
- package/src/core/usage/impact.js +62 -28
- package/src/core/usage/logParsers.js +39 -1
- package/src/core/usage/usageStore.js +32 -5
package/src/core/usage/impact.js
CHANGED
|
@@ -24,6 +24,17 @@ const { toMs } = require('./usageStore');
|
|
|
24
24
|
|
|
25
25
|
const DAY = 86400e3;
|
|
26
26
|
|
|
27
|
+
// Changes that only matter to clients SENDING a body. GET/HEAD/OPTIONS carry
|
|
28
|
+
// no body, so their calls can't be affected — counting them (list/watch
|
|
29
|
+
// traffic dominates real logs) inflated impact several-fold in real Kubernetes
|
|
30
|
+
// audit data. FIELD_BECAME_REQUIRED is listed because it only breaks senders;
|
|
31
|
+
// on the response side it is a stronger guarantee, not a break.
|
|
32
|
+
const REQUEST_BODY_TYPES = new Set([
|
|
33
|
+
'REQUEST_FIELD_REMOVED', 'REQUEST_FIELD_TYPE_CHANGED', 'REQUEST_REQUIRED_FIELD_ADDED',
|
|
34
|
+
'REQUEST_TYPE_CHANGED', 'FIELD_BECAME_REQUIRED',
|
|
35
|
+
]);
|
|
36
|
+
const BODYLESS_METHODS = new Set(['GET', 'HEAD', 'OPTIONS']);
|
|
37
|
+
|
|
27
38
|
function applyUsageImpact(result, opts) {
|
|
28
39
|
const {
|
|
29
40
|
usage, baseSpec, baseNorm, file = 'usage.json',
|
|
@@ -31,10 +42,38 @@ function applyUsageImpact(result, opts) {
|
|
|
31
42
|
downgrade = true, now = Date.now(),
|
|
32
43
|
} = opts;
|
|
33
44
|
|
|
45
|
+
const { opIndex, meta } = indexUsage(usage, baseSpec, { file, ignoreClients, now });
|
|
46
|
+
|
|
47
|
+
const blocker = downgradeBlocker(meta, { downgrade, minDays, maxAgeDays, minCoverage });
|
|
48
|
+
meta.downgrade = { enabled: blocker === null, reason: blocker };
|
|
49
|
+
|
|
50
|
+
const ctx = { opIndex, baseSpec, baseNorm, trusted: meta.trusted };
|
|
51
|
+
const annotate = (c) => (meta.trusted ? { ...c, impact: computeImpact(c, ctx) } : c);
|
|
52
|
+
|
|
53
|
+
let breakingChanges = result.breakingChanges.map(annotate);
|
|
54
|
+
const warnings = result.warnings.map(annotate);
|
|
55
|
+
let unusedBreakingChanges = [];
|
|
56
|
+
|
|
57
|
+
if (meta.downgrade.enabled) {
|
|
58
|
+
unusedBreakingChanges = breakingChanges.filter((c) => c.impact && c.impact.verdict === 'unused');
|
|
59
|
+
breakingChanges = breakingChanges.filter((c) => !(c.impact && c.impact.verdict === 'unused'));
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
return {
|
|
63
|
+
result: { ...result, breakingChanges, warnings, unusedBreakingChanges },
|
|
64
|
+
meta,
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Maps a usage file onto the operations of `spec`:
|
|
70
|
+
* opIndex "GET /orders/{id}" → Map(clientId → { calls, lastSeen, params, paramsLastSeen, weeks, paramsTruncated })
|
|
71
|
+
* meta window, totals, coverage — the evidence the downgrade guards judge
|
|
72
|
+
*/
|
|
73
|
+
function indexUsage(usage, spec, { file = 'usage.json', ignoreClients = [], now = Date.now() } = {}) {
|
|
34
74
|
const ignore = compileClientPatterns(ignoreClients);
|
|
35
|
-
const matcher = createRouteMatcher(
|
|
75
|
+
const matcher = createRouteMatcher(spec, { basePath: usage.basePath });
|
|
36
76
|
|
|
37
|
-
// "GET /orders/{id}" → Map(clientId → { calls, lastSeen, params, paramsTruncated })
|
|
38
77
|
const opIndex = new Map();
|
|
39
78
|
let totalCalls = 0;
|
|
40
79
|
let matchedCalls = 0;
|
|
@@ -52,11 +91,17 @@ function applyUsageImpact(result, opts) {
|
|
|
52
91
|
const key = `${m.method} ${m.template}`;
|
|
53
92
|
if (!opIndex.has(key)) opIndex.set(key, new Map());
|
|
54
93
|
const byClient = opIndex.get(key);
|
|
55
|
-
const agg = byClient.get(c.id) ||
|
|
94
|
+
const agg = byClient.get(c.id) ||
|
|
95
|
+
{ calls: 0, lastSeen: null, params: {}, paramsLastSeen: {}, weeks: {}, paramsTruncated: false };
|
|
56
96
|
agg.calls += c.calls;
|
|
57
97
|
const seen = toMs(c.lastSeen);
|
|
58
98
|
if (seen !== null && (agg.lastSeen === null || seen > agg.lastSeen)) agg.lastSeen = seen;
|
|
59
99
|
for (const [name, n] of Object.entries(c.params || {})) agg.params[name] = (agg.params[name] || 0) + n;
|
|
100
|
+
for (const [name, t] of Object.entries(c.paramsLastSeen || {})) {
|
|
101
|
+
const ms = toMs(t);
|
|
102
|
+
if (ms !== null && !(agg.paramsLastSeen[name] >= ms)) agg.paramsLastSeen[name] = ms;
|
|
103
|
+
}
|
|
104
|
+
for (const [week, n] of Object.entries(c.weeks || {})) agg.weeks[week] = (agg.weeks[week] || 0) + n;
|
|
60
105
|
if (c.paramsTruncated) agg.paramsTruncated = true;
|
|
61
106
|
byClient.set(c.id, agg);
|
|
62
107
|
}
|
|
@@ -82,26 +127,7 @@ function applyUsageImpact(result, opts) {
|
|
|
82
127
|
operationsTotal: matcher.operations.length,
|
|
83
128
|
trusted: matchedCalls > 0,
|
|
84
129
|
};
|
|
85
|
-
|
|
86
|
-
const blocker = downgradeBlocker(meta, { downgrade, minDays, maxAgeDays, minCoverage });
|
|
87
|
-
meta.downgrade = { enabled: blocker === null, reason: blocker };
|
|
88
|
-
|
|
89
|
-
const ctx = { opIndex, baseSpec, baseNorm, trusted: meta.trusted };
|
|
90
|
-
const annotate = (c) => (meta.trusted ? { ...c, impact: computeImpact(c, ctx) } : c);
|
|
91
|
-
|
|
92
|
-
let breakingChanges = result.breakingChanges.map(annotate);
|
|
93
|
-
const warnings = result.warnings.map(annotate);
|
|
94
|
-
let unusedBreakingChanges = [];
|
|
95
|
-
|
|
96
|
-
if (meta.downgrade.enabled) {
|
|
97
|
-
unusedBreakingChanges = breakingChanges.filter((c) => c.impact && c.impact.verdict === 'unused');
|
|
98
|
-
breakingChanges = breakingChanges.filter((c) => !(c.impact && c.impact.verdict === 'unused'));
|
|
99
|
-
}
|
|
100
|
-
|
|
101
|
-
return {
|
|
102
|
-
result: { ...result, breakingChanges, warnings, unusedBreakingChanges },
|
|
103
|
-
meta,
|
|
104
|
-
};
|
|
130
|
+
return { opIndex, meta, matcher };
|
|
105
131
|
}
|
|
106
132
|
|
|
107
133
|
/** Why "zero calls" isn't strong enough evidence to stop failing the build — or null. */
|
|
@@ -124,13 +150,16 @@ function downgradeBlocker(meta, { downgrade, minDays, maxAgeDays, minCoverage })
|
|
|
124
150
|
}
|
|
125
151
|
|
|
126
152
|
function computeImpact(change, { opIndex, baseSpec, baseNorm }) {
|
|
127
|
-
|
|
153
|
+
let endpoints = endpointsOf(change);
|
|
128
154
|
if (endpoints.length === 0) {
|
|
129
155
|
return { verdict: 'unknown', basis: null, reason: 'not tied to a specific endpoint' };
|
|
130
156
|
}
|
|
131
157
|
if (!endpoints.every(([m, p]) => operationExists(baseSpec, m, p))) {
|
|
132
158
|
return { verdict: 'unknown', basis: null, reason: 'endpoint not found in the base spec' };
|
|
133
159
|
}
|
|
160
|
+
if (REQUEST_BODY_TYPES.has(change.type)) {
|
|
161
|
+
endpoints = endpoints.filter(([m]) => !BODYLESS_METHODS.has(m));
|
|
162
|
+
}
|
|
134
163
|
|
|
135
164
|
// Query-parameter changes can be judged per parameter: logs record the
|
|
136
165
|
// query string. Header/path/cookie params and bodies can't — fall back to
|
|
@@ -195,12 +224,14 @@ function computeImpact(change, { opIndex, baseSpec, baseNorm }) {
|
|
|
195
224
|
|
|
196
225
|
function endpointsOf(change) {
|
|
197
226
|
if (change.path && change.method) return [[change.method.toUpperCase(), change.path]];
|
|
198
|
-
|
|
227
|
+
// A merged $ref change lists an endpoint once per occurrence (request AND
|
|
228
|
+
// response); count each endpoint's traffic once.
|
|
229
|
+
const seen = new Map();
|
|
199
230
|
for (const e of change.affectedEndpoints || []) {
|
|
200
231
|
const m = /^([A-Za-z]+) (\S+)$/.exec(e);
|
|
201
|
-
if (m)
|
|
232
|
+
if (m) seen.set(`${m[1].toUpperCase()} ${m[2]}`, [m[1].toUpperCase(), m[2]]);
|
|
202
233
|
}
|
|
203
|
-
return
|
|
234
|
+
return [...seen.values()];
|
|
204
235
|
}
|
|
205
236
|
|
|
206
237
|
function operationExists(spec, method, path) {
|
|
@@ -240,4 +271,7 @@ function compileClientPatterns(patterns) {
|
|
|
240
271
|
function round1(n) { return Math.round(n * 10) / 10; }
|
|
241
272
|
function round3(n) { return Math.round(n * 1000) / 1000; }
|
|
242
273
|
|
|
243
|
-
module.exports = {
|
|
274
|
+
module.exports = {
|
|
275
|
+
applyUsageImpact, indexUsage, downgradeBlocker, endpointsOf, isObjectSchema, compileClientPatterns,
|
|
276
|
+
REQUEST_BODY_TYPES, BODYLESS_METHODS,
|
|
277
|
+
};
|
|
@@ -25,7 +25,8 @@
|
|
|
25
25
|
* combined nginx / Apache "combined" and "common" log formats
|
|
26
26
|
* alb AWS Application Load Balancer access logs
|
|
27
27
|
* json one JSON object per line — nginx JSON log_format, Kong file-log,
|
|
28
|
-
* API Gateway JSON access logs, Envoy/Traefik JSON, ECS-style logs
|
|
28
|
+
* API Gateway JSON access logs, Envoy/Traefik JSON, ECS-style logs,
|
|
29
|
+
* and Kubernetes API-server audit logs (audit.k8s.io Events)
|
|
29
30
|
*/
|
|
30
31
|
|
|
31
32
|
const HTTP_METHODS = new Set(['GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS', 'TRACE', 'CONNECT']);
|
|
@@ -114,6 +115,9 @@ function parseJsonLine(line) {
|
|
|
114
115
|
let obj;
|
|
115
116
|
try { obj = JSON.parse(trimmed); } catch { return null; }
|
|
116
117
|
if (!obj || typeof obj !== 'object') return null;
|
|
118
|
+
if (obj.kind === 'Event' && typeof obj.apiVersion === 'string' && obj.apiVersion.startsWith('audit.k8s.io/')) {
|
|
119
|
+
return parseK8sAuditEvent(obj);
|
|
120
|
+
}
|
|
117
121
|
|
|
118
122
|
let method = pick(obj, JSON_KEYS.method);
|
|
119
123
|
let target = pick(obj, JSON_KEYS.path);
|
|
@@ -146,6 +150,40 @@ function parseJsonLine(line) {
|
|
|
146
150
|
};
|
|
147
151
|
}
|
|
148
152
|
|
|
153
|
+
// ─── Kubernetes audit events ────────────────────────────────────────────────
|
|
154
|
+
|
|
155
|
+
// Kubernetes verbs → HTTP methods (kubernetes.io/docs/reference/access-authn-authz/authorization/#determine-the-request-verb)
|
|
156
|
+
const K8S_VERBS = {
|
|
157
|
+
get: 'GET', list: 'GET', watch: 'GET',
|
|
158
|
+
create: 'POST', update: 'PUT', patch: 'PATCH',
|
|
159
|
+
delete: 'DELETE', deletecollection: 'DELETE',
|
|
160
|
+
};
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* One request is logged once per audit stage (RequestReceived,
|
|
164
|
+
* ResponseStarted for long-running calls, ResponseComplete). Count only the
|
|
165
|
+
* final stage so each request is counted once.
|
|
166
|
+
*/
|
|
167
|
+
function parseK8sAuditEvent(obj) {
|
|
168
|
+
if (obj.stage !== 'ResponseComplete' && obj.stage !== 'Panic') return null;
|
|
169
|
+
const method = K8S_VERBS[obj.verb];
|
|
170
|
+
if (!method || typeof obj.requestURI !== 'string') return null;
|
|
171
|
+
const { path, query } = splitTarget(obj.requestURI);
|
|
172
|
+
if (!path) return null;
|
|
173
|
+
const code = obj.responseStatus && Number(obj.responseStatus.code);
|
|
174
|
+
return {
|
|
175
|
+
method,
|
|
176
|
+
path,
|
|
177
|
+
query,
|
|
178
|
+
host: null,
|
|
179
|
+
status: Number.isInteger(code) && code >= 100 && code <= 599 ? code : null,
|
|
180
|
+
time: parseAnyTime(obj.requestReceivedTimestamp || obj.stageTimestamp),
|
|
181
|
+
userAgent: nullIfDash(asString(obj.userAgent)),
|
|
182
|
+
extras: [],
|
|
183
|
+
json: obj,
|
|
184
|
+
};
|
|
185
|
+
}
|
|
186
|
+
|
|
149
187
|
/** Value at a dotted path. Tries the literal key first ("user_agent.original" as one key). */
|
|
150
188
|
function lookup(obj, dotted) {
|
|
151
189
|
if (obj == null) return undefined;
|
|
@@ -9,7 +9,8 @@ const { clientIdOf } = require('./clientIdentity');
|
|
|
9
9
|
* Aggregates request records into the usage summary written to usage.json.
|
|
10
10
|
*
|
|
11
11
|
* Only aggregates are kept: per (method, path template, client) a call count,
|
|
12
|
-
* first/last seen, and how often
|
|
12
|
+
* first/last seen, calls per week, and how often (and how recently) each
|
|
13
|
+
* query-parameter NAME was sent. Concrete
|
|
13
14
|
* paths (which carry IDs), query values, IPs and bodies are never stored — the
|
|
14
15
|
* file is safe to commit or upload.
|
|
15
16
|
*/
|
|
@@ -19,6 +20,7 @@ const VERSION = 1;
|
|
|
19
20
|
const MAX_ENDPOINTS = 5000;
|
|
20
21
|
const MAX_UNMATCHED = 500;
|
|
21
22
|
const MAX_PARAMS = 50;
|
|
23
|
+
const MAX_WEEKS = 26;
|
|
22
24
|
|
|
23
25
|
class UsageAggregator {
|
|
24
26
|
/**
|
|
@@ -87,7 +89,7 @@ class UsageAggregator {
|
|
|
87
89
|
const clientId = clientIdOf(record, this.clientSpec);
|
|
88
90
|
let c = ep.clients.get(clientId);
|
|
89
91
|
if (!c) {
|
|
90
|
-
c =
|
|
92
|
+
c = newClient();
|
|
91
93
|
ep.clients.set(clientId, c);
|
|
92
94
|
}
|
|
93
95
|
c.calls++;
|
|
@@ -96,11 +98,14 @@ class UsageAggregator {
|
|
|
96
98
|
if (c.lastSeen === null || record.time > c.lastSeen) c.lastSeen = record.time;
|
|
97
99
|
if (this.from === null || record.time < this.from) this.from = record.time;
|
|
98
100
|
if (this.to === null || record.time > this.to) this.to = record.time;
|
|
101
|
+
const week = weekOf(record.time);
|
|
102
|
+
c.weeks.set(week, (c.weeks.get(week) || 0) + 1);
|
|
99
103
|
}
|
|
100
104
|
for (const name of queryParamNames(record.query)) {
|
|
101
105
|
if (c.params.has(name)) c.params.set(name, c.params.get(name) + 1);
|
|
102
106
|
else if (c.params.size < MAX_PARAMS) c.params.set(name, 1);
|
|
103
|
-
else c.paramsTruncated = true;
|
|
107
|
+
else { c.paramsTruncated = true; continue; }
|
|
108
|
+
if (record.time !== null) c.paramsLastSeen.set(name, maxTime(c.paramsLastSeen.get(name) ?? null, record.time));
|
|
104
109
|
}
|
|
105
110
|
|
|
106
111
|
this.stats.requests++;
|
|
@@ -121,7 +126,7 @@ class UsageAggregator {
|
|
|
121
126
|
for (const ec of e.clients) {
|
|
122
127
|
let c = ep.clients.get(ec.id);
|
|
123
128
|
if (!c) {
|
|
124
|
-
c =
|
|
129
|
+
c = newClient();
|
|
125
130
|
ep.clients.set(ec.id, c);
|
|
126
131
|
}
|
|
127
132
|
c.calls += ec.calls;
|
|
@@ -132,6 +137,10 @@ class UsageAggregator {
|
|
|
132
137
|
else if (c.params.size < MAX_PARAMS) c.params.set(name, n);
|
|
133
138
|
else c.paramsTruncated = true;
|
|
134
139
|
}
|
|
140
|
+
for (const [name, t] of Object.entries(ec.paramsLastSeen || {})) {
|
|
141
|
+
if (c.params.has(name)) c.paramsLastSeen.set(name, maxTime(c.paramsLastSeen.get(name) ?? null, toMs(t)));
|
|
142
|
+
}
|
|
143
|
+
for (const [week, n] of Object.entries(ec.weeks || {})) c.weeks.set(week, (c.weeks.get(week) || 0) + n);
|
|
135
144
|
if (ec.paramsTruncated) c.paramsTruncated = true;
|
|
136
145
|
}
|
|
137
146
|
}
|
|
@@ -146,7 +155,11 @@ class UsageAggregator {
|
|
|
146
155
|
.map(([id, c]) => {
|
|
147
156
|
const out = { id, calls: c.calls, firstSeen: toIso(c.firstSeen), lastSeen: toIso(c.lastSeen) };
|
|
148
157
|
if (c.params.size > 0) out.params = Object.fromEntries([...c.params.entries()].sort((a, b) => b[1] - a[1]));
|
|
158
|
+
if (c.paramsLastSeen.size > 0) {
|
|
159
|
+
out.paramsLastSeen = Object.fromEntries([...c.paramsLastSeen.entries()].map(([n, t]) => [n, toIso(t)]));
|
|
160
|
+
}
|
|
149
161
|
if (c.paramsTruncated) out.paramsTruncated = true;
|
|
162
|
+
if (c.weeks.size > 0) out.weeks = Object.fromEntries([...c.weeks.entries()].sort().slice(-MAX_WEEKS));
|
|
150
163
|
return out;
|
|
151
164
|
})
|
|
152
165
|
.sort((a, b) => b.calls - a.calls || a.id.localeCompare(b.id));
|
|
@@ -171,6 +184,20 @@ class UsageAggregator {
|
|
|
171
184
|
}
|
|
172
185
|
}
|
|
173
186
|
|
|
187
|
+
function newClient() {
|
|
188
|
+
return {
|
|
189
|
+
calls: 0, firstSeen: null, lastSeen: null,
|
|
190
|
+
params: new Map(), paramsLastSeen: new Map(), paramsTruncated: false, weeks: new Map(),
|
|
191
|
+
};
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/** The Monday (UTC) starting the week of `ms`, as "YYYY-MM-DD" — the key of usage.json `weeks`. */
|
|
195
|
+
function weekOf(ms) {
|
|
196
|
+
const d = new Date(ms);
|
|
197
|
+
const monday = Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate()) - ((d.getUTCDay() + 6) % 7) * 86400e3;
|
|
198
|
+
return new Date(monday).toISOString().slice(0, 10);
|
|
199
|
+
}
|
|
200
|
+
|
|
174
201
|
/** --host values: exact host names, or globs like "*.example.com". Null = no filter. */
|
|
175
202
|
function compileHostFilter(hosts) {
|
|
176
203
|
const list = (hosts || []).filter(Boolean).map((h) => String(h).toLowerCase());
|
|
@@ -232,4 +259,4 @@ function toIso(ms) { return ms === null || ms === undefined ? null : new Date(ms
|
|
|
232
259
|
function minTime(a, b) { return a === null ? b : b === null ? a : Math.min(a, b); }
|
|
233
260
|
function maxTime(a, b) { return a === null ? b : b === null ? a : Math.max(a, b); }
|
|
234
261
|
|
|
235
|
-
module.exports = { UsageAggregator, loadUsage, validateUsage, parseDuration, toMs, FORMAT, VERSION };
|
|
262
|
+
module.exports = { UsageAggregator, loadUsage, validateUsage, parseDuration, toMs, weekOf, FORMAT, VERSION };
|