specshield 3.4.13 → 3.4.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -24,6 +24,17 @@ const { toMs } = require('./usageStore');
24
24
 
25
25
  const DAY = 86400e3;
26
26
 
27
+ // Changes that only matter to clients SENDING a body. GET/HEAD/OPTIONS carry
28
+ // no body, so their calls can't be affected — counting them (list/watch
29
+ // traffic dominates real logs) inflated impact several-fold in real Kubernetes
30
+ // audit data. FIELD_BECAME_REQUIRED is listed because it only breaks senders;
31
+ // on the response side it is a stronger guarantee, not a break.
32
+ const REQUEST_BODY_TYPES = new Set([
33
+ 'REQUEST_FIELD_REMOVED', 'REQUEST_FIELD_TYPE_CHANGED', 'REQUEST_REQUIRED_FIELD_ADDED',
34
+ 'REQUEST_TYPE_CHANGED', 'FIELD_BECAME_REQUIRED',
35
+ ]);
36
+ const BODYLESS_METHODS = new Set(['GET', 'HEAD', 'OPTIONS']);
37
+
27
38
  function applyUsageImpact(result, opts) {
28
39
  const {
29
40
  usage, baseSpec, baseNorm, file = 'usage.json',
@@ -31,10 +42,38 @@ function applyUsageImpact(result, opts) {
31
42
  downgrade = true, now = Date.now(),
32
43
  } = opts;
33
44
 
45
+ const { opIndex, meta } = indexUsage(usage, baseSpec, { file, ignoreClients, now });
46
+
47
+ const blocker = downgradeBlocker(meta, { downgrade, minDays, maxAgeDays, minCoverage });
48
+ meta.downgrade = { enabled: blocker === null, reason: blocker };
49
+
50
+ const ctx = { opIndex, baseSpec, baseNorm, trusted: meta.trusted };
51
+ const annotate = (c) => (meta.trusted ? { ...c, impact: computeImpact(c, ctx) } : c);
52
+
53
+ let breakingChanges = result.breakingChanges.map(annotate);
54
+ const warnings = result.warnings.map(annotate);
55
+ let unusedBreakingChanges = [];
56
+
57
+ if (meta.downgrade.enabled) {
58
+ unusedBreakingChanges = breakingChanges.filter((c) => c.impact && c.impact.verdict === 'unused');
59
+ breakingChanges = breakingChanges.filter((c) => !(c.impact && c.impact.verdict === 'unused'));
60
+ }
61
+
62
+ return {
63
+ result: { ...result, breakingChanges, warnings, unusedBreakingChanges },
64
+ meta,
65
+ };
66
+ }
67
+
68
+ /**
69
+ * Maps a usage file onto the operations of `spec`:
70
+ * opIndex "GET /orders/{id}" → Map(clientId → { calls, lastSeen, params, paramsLastSeen, weeks, paramsTruncated })
71
+ * meta window, totals, coverage — the evidence the downgrade guards judge
72
+ */
73
+ function indexUsage(usage, spec, { file = 'usage.json', ignoreClients = [], now = Date.now() } = {}) {
34
74
  const ignore = compileClientPatterns(ignoreClients);
35
- const matcher = createRouteMatcher(baseSpec, { basePath: usage.basePath });
75
+ const matcher = createRouteMatcher(spec, { basePath: usage.basePath });
36
76
 
37
- // "GET /orders/{id}" → Map(clientId → { calls, lastSeen, params, paramsTruncated })
38
77
  const opIndex = new Map();
39
78
  let totalCalls = 0;
40
79
  let matchedCalls = 0;
@@ -52,11 +91,17 @@ function applyUsageImpact(result, opts) {
52
91
  const key = `${m.method} ${m.template}`;
53
92
  if (!opIndex.has(key)) opIndex.set(key, new Map());
54
93
  const byClient = opIndex.get(key);
55
- const agg = byClient.get(c.id) || { calls: 0, lastSeen: null, params: {}, paramsTruncated: false };
94
+ const agg = byClient.get(c.id) ||
95
+ { calls: 0, lastSeen: null, params: {}, paramsLastSeen: {}, weeks: {}, paramsTruncated: false };
56
96
  agg.calls += c.calls;
57
97
  const seen = toMs(c.lastSeen);
58
98
  if (seen !== null && (agg.lastSeen === null || seen > agg.lastSeen)) agg.lastSeen = seen;
59
99
  for (const [name, n] of Object.entries(c.params || {})) agg.params[name] = (agg.params[name] || 0) + n;
100
+ for (const [name, t] of Object.entries(c.paramsLastSeen || {})) {
101
+ const ms = toMs(t);
102
+ if (ms !== null && !(agg.paramsLastSeen[name] >= ms)) agg.paramsLastSeen[name] = ms;
103
+ }
104
+ for (const [week, n] of Object.entries(c.weeks || {})) agg.weeks[week] = (agg.weeks[week] || 0) + n;
60
105
  if (c.paramsTruncated) agg.paramsTruncated = true;
61
106
  byClient.set(c.id, agg);
62
107
  }
@@ -82,26 +127,7 @@ function applyUsageImpact(result, opts) {
82
127
  operationsTotal: matcher.operations.length,
83
128
  trusted: matchedCalls > 0,
84
129
  };
85
-
86
- const blocker = downgradeBlocker(meta, { downgrade, minDays, maxAgeDays, minCoverage });
87
- meta.downgrade = { enabled: blocker === null, reason: blocker };
88
-
89
- const ctx = { opIndex, baseSpec, baseNorm, trusted: meta.trusted };
90
- const annotate = (c) => (meta.trusted ? { ...c, impact: computeImpact(c, ctx) } : c);
91
-
92
- let breakingChanges = result.breakingChanges.map(annotate);
93
- const warnings = result.warnings.map(annotate);
94
- let unusedBreakingChanges = [];
95
-
96
- if (meta.downgrade.enabled) {
97
- unusedBreakingChanges = breakingChanges.filter((c) => c.impact && c.impact.verdict === 'unused');
98
- breakingChanges = breakingChanges.filter((c) => !(c.impact && c.impact.verdict === 'unused'));
99
- }
100
-
101
- return {
102
- result: { ...result, breakingChanges, warnings, unusedBreakingChanges },
103
- meta,
104
- };
130
+ return { opIndex, meta, matcher };
105
131
  }
106
132
 
107
133
  /** Why "zero calls" isn't strong enough evidence to stop failing the build — or null. */
@@ -124,13 +150,16 @@ function downgradeBlocker(meta, { downgrade, minDays, maxAgeDays, minCoverage })
124
150
  }
125
151
 
126
152
  function computeImpact(change, { opIndex, baseSpec, baseNorm }) {
127
- const endpoints = endpointsOf(change);
153
+ let endpoints = endpointsOf(change);
128
154
  if (endpoints.length === 0) {
129
155
  return { verdict: 'unknown', basis: null, reason: 'not tied to a specific endpoint' };
130
156
  }
131
157
  if (!endpoints.every(([m, p]) => operationExists(baseSpec, m, p))) {
132
158
  return { verdict: 'unknown', basis: null, reason: 'endpoint not found in the base spec' };
133
159
  }
160
+ if (REQUEST_BODY_TYPES.has(change.type)) {
161
+ endpoints = endpoints.filter(([m]) => !BODYLESS_METHODS.has(m));
162
+ }
134
163
 
135
164
  // Query-parameter changes can be judged per parameter: logs record the
136
165
  // query string. Header/path/cookie params and bodies can't — fall back to
@@ -195,12 +224,14 @@ function computeImpact(change, { opIndex, baseSpec, baseNorm }) {
195
224
 
196
225
  function endpointsOf(change) {
197
226
  if (change.path && change.method) return [[change.method.toUpperCase(), change.path]];
198
- const out = [];
227
+ // A merged $ref change lists an endpoint once per occurrence (request AND
228
+ // response); count each endpoint's traffic once.
229
+ const seen = new Map();
199
230
  for (const e of change.affectedEndpoints || []) {
200
231
  const m = /^([A-Za-z]+) (\S+)$/.exec(e);
201
- if (m) out.push([m[1].toUpperCase(), m[2]]);
232
+ if (m) seen.set(`${m[1].toUpperCase()} ${m[2]}`, [m[1].toUpperCase(), m[2]]);
202
233
  }
203
- return out;
234
+ return [...seen.values()];
204
235
  }
205
236
 
206
237
  function operationExists(spec, method, path) {
@@ -240,4 +271,7 @@ function compileClientPatterns(patterns) {
240
271
  function round1(n) { return Math.round(n * 10) / 10; }
241
272
  function round3(n) { return Math.round(n * 1000) / 1000; }
242
273
 
243
- module.exports = { applyUsageImpact, compileClientPatterns };
274
+ module.exports = {
275
+ applyUsageImpact, indexUsage, downgradeBlocker, endpointsOf, isObjectSchema, compileClientPatterns,
276
+ REQUEST_BODY_TYPES, BODYLESS_METHODS,
277
+ };
@@ -25,7 +25,8 @@
25
25
  * combined nginx / Apache "combined" and "common" log formats
26
26
  * alb AWS Application Load Balancer access logs
27
27
  * json one JSON object per line — nginx JSON log_format, Kong file-log,
28
- * API Gateway JSON access logs, Envoy/Traefik JSON, ECS-style logs
28
+ * API Gateway JSON access logs, Envoy/Traefik JSON, ECS-style logs,
29
+ * and Kubernetes API-server audit logs (audit.k8s.io Events)
29
30
  */
30
31
 
31
32
  const HTTP_METHODS = new Set(['GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS', 'TRACE', 'CONNECT']);
@@ -114,6 +115,9 @@ function parseJsonLine(line) {
114
115
  let obj;
115
116
  try { obj = JSON.parse(trimmed); } catch { return null; }
116
117
  if (!obj || typeof obj !== 'object') return null;
118
+ if (obj.kind === 'Event' && typeof obj.apiVersion === 'string' && obj.apiVersion.startsWith('audit.k8s.io/')) {
119
+ return parseK8sAuditEvent(obj);
120
+ }
117
121
 
118
122
  let method = pick(obj, JSON_KEYS.method);
119
123
  let target = pick(obj, JSON_KEYS.path);
@@ -146,6 +150,40 @@ function parseJsonLine(line) {
146
150
  };
147
151
  }
148
152
 
153
+ // ─── Kubernetes audit events ────────────────────────────────────────────────
154
+
155
+ // Kubernetes verbs → HTTP methods (kubernetes.io/docs/reference/access-authn-authz/authorization/#determine-the-request-verb)
156
+ const K8S_VERBS = {
157
+ get: 'GET', list: 'GET', watch: 'GET',
158
+ create: 'POST', update: 'PUT', patch: 'PATCH',
159
+ delete: 'DELETE', deletecollection: 'DELETE',
160
+ };
161
+
162
+ /**
163
+ * One request is logged once per audit stage (RequestReceived,
164
+ * ResponseStarted for long-running calls, ResponseComplete). Count only the
165
+ * final stage so each request is counted once.
166
+ */
167
+ function parseK8sAuditEvent(obj) {
168
+ if (obj.stage !== 'ResponseComplete' && obj.stage !== 'Panic') return null;
169
+ const method = K8S_VERBS[obj.verb];
170
+ if (!method || typeof obj.requestURI !== 'string') return null;
171
+ const { path, query } = splitTarget(obj.requestURI);
172
+ if (!path) return null;
173
+ const code = obj.responseStatus && Number(obj.responseStatus.code);
174
+ return {
175
+ method,
176
+ path,
177
+ query,
178
+ host: null,
179
+ status: Number.isInteger(code) && code >= 100 && code <= 599 ? code : null,
180
+ time: parseAnyTime(obj.requestReceivedTimestamp || obj.stageTimestamp),
181
+ userAgent: nullIfDash(asString(obj.userAgent)),
182
+ extras: [],
183
+ json: obj,
184
+ };
185
+ }
186
+
149
187
  /** Value at a dotted path. Tries the literal key first ("user_agent.original" as one key). */
150
188
  function lookup(obj, dotted) {
151
189
  if (obj == null) return undefined;
@@ -9,7 +9,8 @@ const { clientIdOf } = require('./clientIdentity');
9
9
  * Aggregates request records into the usage summary written to usage.json.
10
10
  *
11
11
  * Only aggregates are kept: per (method, path template, client) a call count,
12
- * first/last seen, and how often each query-parameter NAME was sent. Concrete
12
+ * first/last seen, calls per week, and how often (and how recently) each
13
+ * query-parameter NAME was sent. Concrete
13
14
  * paths (which carry IDs), query values, IPs and bodies are never stored — the
14
15
  * file is safe to commit or upload.
15
16
  */
@@ -19,6 +20,7 @@ const VERSION = 1;
19
20
  const MAX_ENDPOINTS = 5000;
20
21
  const MAX_UNMATCHED = 500;
21
22
  const MAX_PARAMS = 50;
23
+ const MAX_WEEKS = 26;
22
24
 
23
25
  class UsageAggregator {
24
26
  /**
@@ -87,7 +89,7 @@ class UsageAggregator {
87
89
  const clientId = clientIdOf(record, this.clientSpec);
88
90
  let c = ep.clients.get(clientId);
89
91
  if (!c) {
90
- c = { calls: 0, firstSeen: null, lastSeen: null, params: new Map(), paramsTruncated: false };
92
+ c = newClient();
91
93
  ep.clients.set(clientId, c);
92
94
  }
93
95
  c.calls++;
@@ -96,11 +98,14 @@ class UsageAggregator {
96
98
  if (c.lastSeen === null || record.time > c.lastSeen) c.lastSeen = record.time;
97
99
  if (this.from === null || record.time < this.from) this.from = record.time;
98
100
  if (this.to === null || record.time > this.to) this.to = record.time;
101
+ const week = weekOf(record.time);
102
+ c.weeks.set(week, (c.weeks.get(week) || 0) + 1);
99
103
  }
100
104
  for (const name of queryParamNames(record.query)) {
101
105
  if (c.params.has(name)) c.params.set(name, c.params.get(name) + 1);
102
106
  else if (c.params.size < MAX_PARAMS) c.params.set(name, 1);
103
- else c.paramsTruncated = true;
107
+ else { c.paramsTruncated = true; continue; }
108
+ if (record.time !== null) c.paramsLastSeen.set(name, maxTime(c.paramsLastSeen.get(name) ?? null, record.time));
104
109
  }
105
110
 
106
111
  this.stats.requests++;
@@ -121,7 +126,7 @@ class UsageAggregator {
121
126
  for (const ec of e.clients) {
122
127
  let c = ep.clients.get(ec.id);
123
128
  if (!c) {
124
- c = { calls: 0, firstSeen: null, lastSeen: null, params: new Map(), paramsTruncated: false };
129
+ c = newClient();
125
130
  ep.clients.set(ec.id, c);
126
131
  }
127
132
  c.calls += ec.calls;
@@ -132,6 +137,10 @@ class UsageAggregator {
132
137
  else if (c.params.size < MAX_PARAMS) c.params.set(name, n);
133
138
  else c.paramsTruncated = true;
134
139
  }
140
+ for (const [name, t] of Object.entries(ec.paramsLastSeen || {})) {
141
+ if (c.params.has(name)) c.paramsLastSeen.set(name, maxTime(c.paramsLastSeen.get(name) ?? null, toMs(t)));
142
+ }
143
+ for (const [week, n] of Object.entries(ec.weeks || {})) c.weeks.set(week, (c.weeks.get(week) || 0) + n);
135
144
  if (ec.paramsTruncated) c.paramsTruncated = true;
136
145
  }
137
146
  }
@@ -146,7 +155,11 @@ class UsageAggregator {
146
155
  .map(([id, c]) => {
147
156
  const out = { id, calls: c.calls, firstSeen: toIso(c.firstSeen), lastSeen: toIso(c.lastSeen) };
148
157
  if (c.params.size > 0) out.params = Object.fromEntries([...c.params.entries()].sort((a, b) => b[1] - a[1]));
158
+ if (c.paramsLastSeen.size > 0) {
159
+ out.paramsLastSeen = Object.fromEntries([...c.paramsLastSeen.entries()].map(([n, t]) => [n, toIso(t)]));
160
+ }
149
161
  if (c.paramsTruncated) out.paramsTruncated = true;
162
+ if (c.weeks.size > 0) out.weeks = Object.fromEntries([...c.weeks.entries()].sort().slice(-MAX_WEEKS));
150
163
  return out;
151
164
  })
152
165
  .sort((a, b) => b.calls - a.calls || a.id.localeCompare(b.id));
@@ -171,6 +184,20 @@ class UsageAggregator {
171
184
  }
172
185
  }
173
186
 
187
+ function newClient() {
188
+ return {
189
+ calls: 0, firstSeen: null, lastSeen: null,
190
+ params: new Map(), paramsLastSeen: new Map(), paramsTruncated: false, weeks: new Map(),
191
+ };
192
+ }
193
+
194
+ /** The Monday (UTC) starting the week of `ms`, as "YYYY-MM-DD" — the key of usage.json `weeks`. */
195
+ function weekOf(ms) {
196
+ const d = new Date(ms);
197
+ const monday = Date.UTC(d.getUTCFullYear(), d.getUTCMonth(), d.getUTCDate()) - ((d.getUTCDay() + 6) % 7) * 86400e3;
198
+ return new Date(monday).toISOString().slice(0, 10);
199
+ }
200
+
174
201
  /** --host values: exact host names, or globs like "*.example.com". Null = no filter. */
175
202
  function compileHostFilter(hosts) {
176
203
  const list = (hosts || []).filter(Boolean).map((h) => String(h).toLowerCase());
@@ -232,4 +259,4 @@ function toIso(ms) { return ms === null || ms === undefined ? null : new Date(ms
232
259
  function minTime(a, b) { return a === null ? b : b === null ? a : Math.min(a, b); }
233
260
  function maxTime(a, b) { return a === null ? b : b === null ? a : Math.max(a, b); }
234
261
 
235
- module.exports = { UsageAggregator, loadUsage, validateUsage, parseDuration, toMs, FORMAT, VERSION };
262
+ module.exports = { UsageAggregator, loadUsage, validateUsage, parseDuration, toMs, weekOf, FORMAT, VERSION };