specshield 3.4.12 → 3.4.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -15,8 +15,23 @@
15
15
  that exercise them (exact for query parameters). Breaking changes with zero calls move to
16
16
  `unusedBreakingChanges` and don't fail `--fail-on-breaking`, only when the usage covers
17
17
  ≥ `--usage-min-days` (default 7), is ≤ 30 days old and matches the base spec.
18
- Also `--usage-ignore-client`, `--no-usage-downgrade`, and a `usage:` block in
19
- `.specshield.yml`. JSON output is unchanged unless `--usage` is given.
18
+ Also `--usage-ignore-client`, `--usage-max-age`, `--usage-min-coverage`,
19
+ `--no-usage-downgrade`, and a `usage:` block in `.specshield.yml`. JSON output is
20
+ unchanged unless `--usage` is given.
21
+
22
+ ### Fixed (found by testing on real logs — SEART-GHS API + live spec, Elastic, AWS ALB examples)
23
+
24
+ - Object-typed query parameters (Spring DTOs) were judged by their own name, which never
25
+ appears in URLs, and reported as unused while 1,491 calls sent their fields.
26
+ - Filtered or sampled logs made unseen endpoints look unused: unused changes now pass only
27
+ when ≥ 25% of the spec's operations appear in the logs (`--usage-min-coverage`).
28
+ - The 30-day staleness guard couldn't be relaxed for archived logs (`--usage-max-age`).
29
+ - JSON logs: added the field names of Elastic (`agent`, `response`), Caddy, Traefik, Envoy
30
+ and Google Cloud LB; previously Elastic JSON gave 1 client where the same events in
31
+ combined format gave 36.
32
+ - Multi-word User-Agent products are kept ("Debian APT-HTTP/1.3", not "Debian").
33
+ - `usage import --host` filters by host; ALB/JSON logs from a load balancer serving
34
+ several APIs no longer mix their traffic, and the import warns when hosts are mixed.
20
35
 
21
36
  ## 3.4.0 — 2026-07-29 — `govern` command (API governance in the CLI)
22
37
 
package/README.md CHANGED
@@ -290,6 +290,11 @@ specshield usage import /var/log/nginx/access.log* --spec api/openapi.yaml
290
290
  (base paths from `servers` / `basePath`, or `--base-path`). Without it, IDs are guessed.
291
291
  - Add more days later with `--append` (a file imported twice is detected and skipped);
292
292
  limit the window with `--since 30d`.
293
+ - **One load balancer, several APIs?** ALB and JSON logs record the host; pass
294
+ `--host api.example.com` (repeatable, `*.example.com` globs) so another API's traffic
295
+ doesn't count as usage of this one. The import warns when it sees mixed hosts.
296
+ - **Import logs from every server.** If several instances each write their own log,
297
+ import all of them — logs from one node make the others' clients look like non-users.
293
298
 
294
299
  **2. Compare with impact:**
295
300
 
@@ -304,9 +309,13 @@ specshield compare base.yaml target.yaml --usage usage.json --fail-on-breaking
304
309
  *send* it; a parameter that became required affects only clients that *omit* it.
305
310
  - **Zero calls → "breaking but unused"**: listed separately and excluded from
306
311
  `--fail-on-breaking`. SpecShield only does this when the evidence is good enough — at
307
- least `--usage-min-days` (default 7) of logs, the newest no older than 30 days, and calls
308
- that actually match the base spec. Otherwise the change stays breaking and the report
309
- says why. `--no-usage-downgrade` keeps everything breaking and only annotates.
312
+ least `--usage-min-days` (default 7) of logs, the newest no older than `--usage-max-age`
313
+ (default 30) days, calls that match the base spec, and at least `--usage-min-coverage`
314
+ (default 25) percent of the spec's operations seen in the logs — fewer usually means the
315
+ logs are filtered, sampled or from one server of several. Otherwise the change stays
316
+ breaking and the report says why. `--no-usage-downgrade` keeps everything breaking.
317
+ - Object-typed query parameters (e.g. a Spring `@ModelAttribute` DTO) are sent as their
318
+ properties, not under their own name, so they're judged by who calls the endpoint.
310
319
  - Leave health checks and bots out: `--usage-ignore-client 'kube-probe/*' --usage-ignore-client healthcheck`.
311
320
  - `--json` adds `impact` to each change, plus `unusedBreakingChanges`, `summary.unusedBreaking`
312
321
  and a `usage` block. Without `--usage` the JSON is unchanged.
@@ -1275,6 +1284,8 @@ ignore:
1275
1284
  # file: usage.json
1276
1285
  # ignoreClients: ["kube-probe/*"]
1277
1286
  # minDays: 7
1287
+ # maxAgeDays: 30
1288
+ # minCoverage: 25
1278
1289
  # downgrade: true
1279
1290
 
1280
1291
  # Hosted compare (set --remote on the CLI to override).
@@ -1337,6 +1348,8 @@ specshield compare <base> <target> [options]
1337
1348
  | `--usage <file>` | `usage.json` from `specshield usage import` — show which clients each breaking change affects |
1338
1349
  | `--usage-ignore-client <pattern>` | Leave a client out of the impact (repeatable; `*` globs, otherwise substring) |
1339
1350
  | `--usage-min-days <n>` | Days of usage needed before an unused breaking change stops failing the build (default `7`) |
1351
+ | `--usage-max-age <days>` | The newest log line may be at most this old for unused changes to pass (default `30`) |
1352
+ | `--usage-min-coverage <pct>` | Percent of spec operations that must appear in the logs for unused changes to pass (default `25`) |
1340
1353
  | `--no-usage-downgrade` | Annotate impact only; keep unused breaking changes breaking |
1341
1354
  | `--timeout <ms>` | Request timeout for remote mode (default `10000`) |
1342
1355
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "specshield",
3
- "version": "3.4.12",
3
+ "version": "3.4.13",
4
4
  "description": "Contract compatibility testing for APIs — catch breaking OpenAPI changes before they reach your consumers, with can-i-deploy deploy gating and GitHub PR checks. (a.k.a. bidirectional contract testing.)",
5
5
  "main": "src/cli.js",
6
6
  "homepage": "https://specshield.io",
@@ -46,6 +46,8 @@ compare
46
46
  .option('--usage <file>', 'usage.json from "specshield usage import" — show which clients each breaking change affects')
47
47
  .option('--usage-ignore-client <pattern>', 'Leave a client out of the impact, e.g. health checks (repeatable; * globs)', collect, [])
48
48
  .option('--usage-min-days <n>', 'Days of usage needed before an unused breaking change stops failing the build (default: 7)')
49
+ .option('--usage-max-age <days>', 'Newest log line may be at most this old for unused changes to pass (default: 30)')
50
+ .option('--usage-min-coverage <pct>', 'Percent of spec operations that must appear in the logs for unused changes to pass (default: 25)')
49
51
  .option('--no-usage-downgrade', 'Annotate impact only; keep unused breaking changes as breaking')
50
52
  .action(async (base, target, opts) => {
51
53
  try {
@@ -156,10 +158,14 @@ async function applyUsage(result, base, options) {
156
158
 
157
159
  const usage = await loadUsage(options.usage);
158
160
  const baseSpec = parseSpec(await loadSpec(base), base);
159
- const minDays = Number(options.usageMinDays);
160
- if (!Number.isFinite(minDays) || minDays < 0) {
161
- throw new Error(`--usage-min-days must be a number ≥ 0, got "${options.usageMinDays}"`);
162
- }
161
+ const num = (value, flag, max = Infinity) => {
162
+ const n = Number(value);
163
+ if (!Number.isFinite(n) || n < 0 || n > max) throw new Error(`${flag} must be a number from 0 to ${max}, got "${value}"`);
164
+ return n;
165
+ };
166
+ const minDays = num(options.usageMinDays, '--usage-min-days');
167
+ const maxAgeDays = num(options.usageMaxAge, '--usage-max-age');
168
+ const minCoverage = num(options.usageMinCoverage, '--usage-min-coverage', 100);
163
169
 
164
170
  const { result: annotated, meta } = applyUsageImpact(result, {
165
171
  usage,
@@ -168,6 +174,8 @@ async function applyUsage(result, base, options) {
168
174
  file: path.basename(options.usage),
169
175
  ignoreClients: options.usageIgnoreClient,
170
176
  minDays,
177
+ maxAgeDays,
178
+ minCoverage,
171
179
  downgrade: options.usageDowngrade,
172
180
  });
173
181
  if (!meta.trusted && meta.totalCalls > 0) {
@@ -303,6 +311,12 @@ function mergeOptions(config, cliOpts) {
303
311
  usageMinDays: cliOpts.usageMinDays !== undefined
304
312
  ? cliOpts.usageMinDays
305
313
  : config.usage && config.usage.minDays !== undefined ? config.usage.minDays : 7,
314
+ usageMaxAge: cliOpts.usageMaxAge !== undefined
315
+ ? cliOpts.usageMaxAge
316
+ : config.usage && config.usage.maxAgeDays !== undefined ? config.usage.maxAgeDays : 30,
317
+ usageMinCoverage: cliOpts.usageMinCoverage !== undefined
318
+ ? cliOpts.usageMinCoverage
319
+ : config.usage && config.usage.minCoverage !== undefined ? config.usage.minCoverage : 25,
306
320
  usageDowngrade: cliOpts.usageDowngrade === false
307
321
  ? false
308
322
  : !(config.usage && config.usage.downgrade === false),
@@ -28,6 +28,7 @@ usage
28
28
  .option('--format <format>', 'Log format: auto | combined | alb | json', 'auto')
29
29
  .option('--client <source>', 'How to identify clients: ua | field:<path> | extra:<n>, optionally hash:-prefixed', 'ua')
30
30
  .option('--base-path <path>', 'Path prefix the API is served under, e.g. /api/v1 (default: from the spec servers)')
31
+ .option('--host <name>', 'Only count requests for this host, e.g. api.example.com or *.example.com (repeatable)', collect, [])
31
32
  .option('--since <duration>', 'Only count requests newer than this, e.g. 30d')
32
33
  .option('--append', 'Add to an existing --out file instead of replacing it')
33
34
  .action(async (logs, opts) => {
@@ -50,7 +51,7 @@ usage
50
51
  }
51
52
  }
52
53
 
53
- const aggregator = new UsageAggregator({ clientSpec, matcher, sinceMs });
54
+ const aggregator = new UsageAggregator({ clientSpec, matcher, sinceMs, hosts: opts.host });
54
55
  const sources = existing ? [...(existing.sources || [])] : [];
55
56
  const seen = new Set(sources.map((s) => s.fingerprint).filter(Boolean));
56
57
  const imported = [];
@@ -85,7 +86,9 @@ usage
85
86
  });
86
87
  await fsExtra.outputFile(opts.out, JSON.stringify(out, null, 2) + '\n');
87
88
 
88
- printImportSummary(out, imported, runStats, { outFile: opts.out, matcher, appended: !!existing });
89
+ printImportSummary(out, imported, runStats, {
90
+ outFile: opts.out, matcher, appended: !!existing, hostsSeen: aggregator.hostsSeen, hostFilter: opts.host.length > 0,
91
+ });
89
92
  exitAfterFlush(0);
90
93
  } catch (err) {
91
94
  logger.error(`Error: ${err.message}`);
@@ -93,7 +96,7 @@ usage
93
96
  }
94
97
  });
95
98
 
96
- function printImportSummary(out, imported, run, { outFile, matcher, appended }) {
99
+ function printImportSummary(out, imported, run, { outFile, matcher, appended, hostsSeen, hostFilter }) {
97
100
  const lines = [];
98
101
  const parsed = imported.reduce((n, r) => n + r.parsed, 0);
99
102
  const skipped = imported.reduce((n, r) => n + r.skipped, 0);
@@ -110,6 +113,12 @@ function printImportSummary(out, imported, run, { outFile, matcher, appended })
110
113
  lines.push(` ${plural(out.endpoints.length, 'endpoint')} · ${plural(clients.size, 'client')}` +
111
114
  (matcher ? ` · ${fmt(run.matched)} matched the spec · ${fmt(run.unmatched)} not in the spec` : ''));
112
115
  if (run.notFound) lines.push(chalk.gray(` ${plural(run.notFound, 'request')} to unknown paths answered 404/405 skipped (scanners)`));
116
+ if (run.otherHost) lines.push(chalk.gray(` ${plural(run.otherHost, 'request')} for other hosts skipped (--host)`));
117
+ if (!hostFilter && hostsSeen.size > 1) {
118
+ const top = [...hostsSeen.entries()].sort((a, b) => b[1] - a[1]).slice(0, 4).map(([h, n]) => `${h} (${fmt(n)})`);
119
+ lines.push(chalk.yellow(` Requests for ${hostsSeen.size} hosts are mixed: ${top.join(', ')}${hostsSeen.size > 4 ? ', …' : ''}.`));
120
+ lines.push(chalk.yellow(' If they are different APIs, re-import with --host <name> so they don\'t count as usage of this one.'));
121
+ }
113
122
  if (run.tooOld) lines.push(chalk.gray(` ${plural(run.tooOld, 'request')} older than --since skipped`));
114
123
  if (run.overflow) lines.push(chalk.yellow(` ${plural(run.overflow, 'request')} dropped: endpoint limit reached (use --spec to group paths)`));
115
124
  if (skipped) lines.push(chalk.gray(` ${plural(skipped, 'line')} skipped (not a request)`));
@@ -144,6 +153,7 @@ usage
144
153
  const top = Math.max(1, parseInt(opts.top, 10) || 15);
145
154
  let unused = null;
146
155
  let undocumented = null;
156
+ let calledCount = 0;
147
157
 
148
158
  if (opts.spec) {
149
159
  const spec = parseSpec(await loadSpec(opts.spec), opts.spec);
@@ -155,6 +165,7 @@ usage
155
165
  if (m && m.operation) called.add(`${m.method} ${m.template}`);
156
166
  else undocumented.push({ method: e.method, path: e.path, calls: e.calls, lastSeen: e.lastSeen, clients: e.clients.length });
157
167
  }
168
+ calledCount = called.size;
158
169
  unused = matcher.operations
159
170
  .filter((o) => !called.has(`${o.method} ${o.template}`))
160
171
  .map((o) => ({ method: o.method, path: o.template }));
@@ -170,7 +181,7 @@ usage
170
181
  return exitAfterFlush(0);
171
182
  }
172
183
 
173
- process.stdout.write(renderShow(data, { top, unused, undocumented }) + '\n');
184
+ process.stdout.write(renderShow(data, { top, unused, undocumented, seenOps: calledCount }) + '\n');
174
185
  exitAfterFlush(0);
175
186
  } catch (err) {
176
187
  logger.error(`Error: ${err.message}`);
@@ -178,7 +189,7 @@ usage
178
189
  }
179
190
  });
180
191
 
181
- function renderShow(data, { top, unused, undocumented }) {
192
+ function renderShow(data, { top, unused, undocumented, seenOps }) {
182
193
  const lines = [];
183
194
  const total = data.endpoints.reduce((n, e) => n + e.calls, 0);
184
195
  lines.push('');
@@ -195,7 +206,7 @@ function renderShow(data, { top, unused, undocumented }) {
195
206
  for (const c of shown) {
196
207
  lines.push(chalk.gray(` ${c.id.padEnd(28)} ${pct(c.calls / e.calls).padStart(4)} last seen ${ago(c.lastSeen)}`));
197
208
  }
198
- if (e.clients.length > shown.length) lines.push(chalk.gray(` …and ${e.clients.length - shown.length} more client(s)`));
209
+ if (e.clients.length > shown.length) lines.push(chalk.gray(` …and ${plural(e.clients.length - shown.length, 'more client')}`));
199
210
  }
200
211
  if (data.endpoints.length > top) lines.push(chalk.gray(` …${data.endpoints.length - top} more endpoint(s) — raise --top`));
201
212
 
@@ -203,6 +214,11 @@ function renderShow(data, { top, unused, undocumented }) {
203
214
  lines.push('');
204
215
  lines.push(chalk.green.bold(` ✔ NEVER CALLED IN THIS WINDOW (${unused.length}) — candidates for deprecation`));
205
216
  lines.push(chalk.gray(' ─────────────────────────────────────────'));
217
+ const total = unused.length + seenOps;
218
+ if (total > 0 && seenOps / total < 0.25) {
219
+ lines.push(chalk.yellow(` Only ${seenOps} of ${total} operations appear in these logs. If the logs are filtered,`));
220
+ lines.push(chalk.yellow(' sampled or from one server of several, this list is not reliable.'));
221
+ }
206
222
  if (unused.length === 0) lines.push(chalk.gray(' Every operation in the spec was called.'));
207
223
  for (const o of unused) lines.push(` ${chalk.green('○')} ${o.method} ${o.path}`);
208
224
  }
@@ -220,4 +236,8 @@ function renderShow(data, { top, unused, undocumented }) {
220
236
  return lines.join('\n');
221
237
  }
222
238
 
239
+ function collect(value, previous) {
240
+ return previous.concat([value]);
241
+ }
242
+
223
243
  module.exports = usage;
@@ -116,6 +116,11 @@ function validateConfig(config, filePath) {
116
116
  if (u.minDays !== undefined && !(Number(u.minDays) >= 0)) {
117
117
  throw new Error(`Config error in "${filePath}": usage.minDays must be a number ≥ 0`);
118
118
  }
119
+ for (const key of ['maxAgeDays', 'minCoverage']) {
120
+ if (u[key] !== undefined && !(Number(u[key]) >= 0)) {
121
+ throw new Error(`Config error in "${filePath}": usage.${key} must be a number ≥ 0`);
122
+ }
123
+ }
119
124
  if (u.downgrade !== undefined && typeof u.downgrade !== 'boolean') {
120
125
  throw new Error(`Config error in "${filePath}": usage.downgrade must be a boolean`);
121
126
  }
@@ -123,6 +128,8 @@ function validateConfig(config, filePath) {
123
128
  file: u.file || null,
124
129
  ignoreClients: (u.ignoreClients || []).filter(Boolean).map(String),
125
130
  minDays: u.minDays !== undefined ? Number(u.minDays) : undefined,
131
+ maxAgeDays: u.maxAgeDays !== undefined ? Number(u.maxAgeDays) : undefined,
132
+ minCoverage: u.minCoverage !== undefined ? Number(u.minCoverage) : undefined,
126
133
  downgrade: u.downgrade,
127
134
  };
128
135
  } else {
@@ -117,6 +117,7 @@ function formatUsageHeader(u) {
117
117
  ` Usage: ${u.file} · ${u.window.from ? `${day(u.window.from)} → ${day(u.window.to)}` : 'no timestamps'}` +
118
118
  (span !== null ? ` (${span} days)` : '') +
119
119
  ` · ${plural(u.matchedCalls, 'call')} · ${plural(u.clients, 'client')}` +
120
+ (u.operationsTotal ? ` · ${u.operationsSeen}/${u.operationsTotal} operations seen` : '') +
120
121
  (u.ignoredCalls ? ` · ${fmt(u.ignoredCalls)} ignored` : '')));
121
122
  if (!u.downgrade.enabled && u.downgrade.reason !== 'disabled by --no-usage-downgrade') {
122
123
  lines.push(chalk.yellow(` Unused breaking changes still fail the build: ${u.downgrade.reason}.`));
@@ -58,10 +58,13 @@ function normalizeUserAgent(ua) {
58
58
  const s = ua.trim();
59
59
  if (!s) return null;
60
60
  if (/^Mozilla\//.test(s)) return 'browser';
61
- const first = s.split(/\s+/)[0];
62
- const m = /^([^/]+)\/(\d+)(?:\.(\d+))?/.exec(first);
63
- if (m) return `${m[1]}/${m[2]}${m[3] !== undefined ? `.${m[3]}` : ''}`;
64
- return first.length > 60 ? first.slice(0, 60) : first;
61
+ // The product is everything up to the first "/<digit>", so multi-word
62
+ // products survive: "Debian APT-HTTP/1.3 (…)" → "Debian APT-HTTP/1.3",
63
+ // "Chef Client/10.18.2 (…)" → "Chef Client/10.18".
64
+ const m = /^([^/();]{1,60}?)\/v?(\d+)(?:\.(\d+))?/.exec(s);
65
+ if (m) return `${m[1].trim()}/${m[2]}${m[3] !== undefined ? `.${m[3]}` : ''}`;
66
+ const name = s.split('(')[0].replace(/[\s;,]+$/, '') || s;
67
+ return name.length > 60 ? name.slice(0, 60) : name;
65
68
  }
66
69
 
67
70
  module.exports = { parseClientSpec, clientIdOf, normalizeUserAgent };
@@ -22,13 +22,13 @@ const { toMs } = require('./usageStore');
22
22
  * downgradeBlocker). Otherwise they are annotated and stay breaking.
23
23
  */
24
24
 
25
- const STALE_AFTER_DAYS = 30;
26
25
  const DAY = 86400e3;
27
26
 
28
27
  function applyUsageImpact(result, opts) {
29
28
  const {
30
29
  usage, baseSpec, baseNorm, file = 'usage.json',
31
- ignoreClients = [], minDays = 7, downgrade = true, now = Date.now(),
30
+ ignoreClients = [], minDays = 7, maxAgeDays = 30, minCoverage = 25,
31
+ downgrade = true, now = Date.now(),
32
32
  } = opts;
33
33
 
34
34
  const ignore = compileClientPatterns(ignoreClients);
@@ -75,10 +75,15 @@ function applyUsageImpact(result, opts) {
75
75
  clients: clientIds.size,
76
76
  matchRate: totalCalls > 0 ? round3(matchedCalls / totalCalls) : 0,
77
77
  basePaths: matcher.basePaths,
78
+ // Share of the spec's operations that appear in the logs. A low share
79
+ // usually means the logs are filtered (one route exported), sampled, or
80
+ // from one node of several — and then "0 calls" proves nothing.
81
+ operationsSeen: opIndex.size,
82
+ operationsTotal: matcher.operations.length,
78
83
  trusted: matchedCalls > 0,
79
84
  };
80
85
 
81
- const blocker = downgradeBlocker(meta, { downgrade, minDays });
86
+ const blocker = downgradeBlocker(meta, { downgrade, minDays, maxAgeDays, minCoverage });
82
87
  meta.downgrade = { enabled: blocker === null, reason: blocker };
83
88
 
84
89
  const ctx = { opIndex, baseSpec, baseNorm, trusted: meta.trusted };
@@ -100,15 +105,20 @@ function applyUsageImpact(result, opts) {
100
105
  }
101
106
 
102
107
  /** Why "zero calls" isn't strong enough evidence to stop failing the build — or null. */
103
- function downgradeBlocker(meta, { downgrade, minDays }) {
108
+ function downgradeBlocker(meta, { downgrade, minDays, maxAgeDays, minCoverage }) {
104
109
  if (!downgrade) return 'disabled by --no-usage-downgrade';
105
110
  if (!meta.trusted) return 'no call in the usage file matches an operation in the base spec';
106
111
  if (meta.days === null) return 'the logs had no timestamps, so the time window is unknown';
107
112
  if (meta.days < minDays) {
108
113
  return `the usage covers ${meta.days} day(s); at least ${minDays} are needed so weekly/monthly clients show up (--usage-min-days)`;
109
114
  }
110
- if (meta.staleDays > STALE_AFTER_DAYS) {
111
- return `the newest log line is ${Math.floor(meta.staleDays)} days old — re-import recent logs`;
115
+ if (meta.staleDays > maxAgeDays) {
116
+ return `the newest log line is ${Math.floor(meta.staleDays)} days old — re-import recent logs (--usage-max-age)`;
117
+ }
118
+ const coverage = meta.operationsTotal > 0 ? (meta.operationsSeen / meta.operationsTotal) * 100 : 0;
119
+ if (coverage < minCoverage) {
120
+ return `only ${meta.operationsSeen} of ${meta.operationsTotal} operations in the spec appear in the logs — ` +
121
+ 'they look filtered, sampled or from one server of several (--usage-min-coverage)';
112
122
  }
113
123
  return null;
114
124
  }
@@ -128,10 +138,14 @@ function computeImpact(change, { opIndex, baseSpec, baseNorm }) {
128
138
  let param = null;
129
139
  const pm = /^parameters\.(.+)$/.exec(change.field || '');
130
140
  if (pm && endpoints.length === 1) {
131
- const loc = paramLocation(baseNorm, endpoints[0][0], endpoints[0][1], pm[1]);
141
+ const p = findParam(baseNorm, endpoints[0][0], endpoints[0][1], pm[1]);
132
142
  const byClient = opIndex.get(`${endpoints[0][0]} ${endpoints[0][1]}`);
133
143
  const truncated = byClient && [...byClient.values()].some((c) => c.paramsTruncated);
134
- if (loc === 'query' && !truncated) param = pm[1];
144
+ // An object-typed query parameter (Spring's @ModelAttribute DTOs, form
145
+ // explode, deepObject) never appears in the URL under its own name — its
146
+ // properties do. Judging it by name would call a heavily used parameter
147
+ // unused, so judge it by who calls the endpoint instead.
148
+ if (p && p.in === 'query' && !isObjectSchema(p.schema) && !truncated) param = pm[1];
135
149
  }
136
150
  const omitters = change.type === 'PARAMETER_BECAME_REQUIRED';
137
151
 
@@ -194,10 +208,17 @@ function operationExists(spec, method, path) {
194
208
  return Boolean(item && item[method.toLowerCase()]);
195
209
  }
196
210
 
197
- function paramLocation(baseNorm, method, path, name) {
211
+ function findParam(baseNorm, method, path, name) {
198
212
  const op = baseNorm && baseNorm.endpoints && baseNorm.endpoints[path] && baseNorm.endpoints[path][method.toLowerCase()];
199
- const p = op && (op.parameters || []).find((x) => x.name === name);
200
- return p ? p.in : null;
213
+ return (op && (op.parameters || []).find((x) => x.name === name)) || null;
214
+ }
215
+
216
+ /** Unknown schemas count as objects: when in doubt, don't judge by the name. */
217
+ function isObjectSchema(schema) {
218
+ if (!schema) return true;
219
+ // Scalars and arrays serialize under the parameter's own name (?tag=a&tag=b).
220
+ if (['string', 'integer', 'number', 'boolean', 'array'].includes(schema.type)) return false;
221
+ return true;
201
222
  }
202
223
 
203
224
  /**
@@ -11,6 +11,8 @@
11
11
  *
12
12
  * method upper-case HTTP method
13
13
  * path URL path only — no scheme/host, no query string
14
+ * host lower-case host without port, when the log records it (ALB,
15
+ * JSON with a host field, absolute request URLs), else null
14
16
  * query raw query string without the leading '?' ('' when absent)
15
17
  * status numeric status code, or null
16
18
  * time epoch ms, or null when the line carries no parseable timestamp
@@ -83,19 +85,27 @@ function parseAlb(line) {
83
85
 
84
86
  // ─── JSON lines ─────────────────────────────────────────────────────────────
85
87
 
88
+ // Field names used by common producers: nginx/Apache JSON log_format, Kong
89
+ // file-log, Caddy, Traefik, Envoy, Google Cloud HTTP(S) LB, AWS API Gateway,
90
+ // ECS/OpenTelemetry, Elastic examples. Order matters — first non-empty wins,
91
+ // so nested keys come before same-named containers ("response.status" before
92
+ // "response").
86
93
  const JSON_KEYS = {
87
- method: ['method', 'request_method', 'httpMethod', 'http_method', 'verb',
88
- 'request.method', 'req.method', 'http.method', 'http.request.method'],
89
- path: ['path', 'uri', 'request_uri', 'url', 'request.uri', 'request.url', 'req.url',
90
- 'req.path', 'http.url', 'http.target', 'url.original', 'url.path'],
94
+ method: ['method', 'request_method', 'httpMethod', 'http_method', 'verb', 'RequestMethod',
95
+ 'request.method', 'req.method', 'http.method', 'http.request.method', 'httpRequest.requestMethod'],
96
+ path: ['path', 'uri', 'request_uri', 'url', 'RequestPath', 'request.uri', 'request.url', 'req.url',
97
+ 'req.path', 'http.url', 'http.target', 'url.original', 'url.path', 'httpRequest.requestUrl'],
91
98
  query: ['query', 'query_string', 'args', 'url.query'],
92
- status: ['status', 'status_code', 'statusCode', 'response.status', 'res.statusCode',
93
- 'http.status_code', 'http.response.status_code'],
99
+ host: ['host', 'http_host', 'RequestHost', 'request.host', 'request.headers.host', 'request.headers.Host',
100
+ 'req.headers.host', 'headers.host', 'url.domain', 'http.host', 'domainName'],
101
+ status: ['status', 'status_code', 'statusCode', 'response_code', 'DownstreamStatus', 'OriginStatus',
102
+ 'response.status', 'res.statusCode', 'http.status_code', 'http.response.status_code',
103
+ 'httpRequest.status', 'response'],
94
104
  time: ['time', 'timestamp', '@timestamp', 'time_iso8601', 'time_local', 'requestTime',
95
- 'requestTimeEpoch', 'started_at', 'ts', 'start_time'],
96
- userAgent: ['user_agent', 'http_user_agent', 'userAgent', 'user_agent.original',
97
- 'request.headers.user-agent', 'req.headers.user-agent', 'headers.user-agent',
98
- 'http.user_agent', 'request.headers.User-Agent'],
105
+ 'requestTimeEpoch', 'started_at', 'ts', 'start_time', 'StartUTC'],
106
+ userAgent: ['user_agent', 'http_user_agent', 'userAgent', 'agent', 'user_agent.original',
107
+ 'request.headers.user-agent', 'request.headers.User-Agent', 'req.headers.user-agent',
108
+ 'headers.user-agent', 'http.user_agent', 'request_User-Agent', 'httpRequest.userAgent'],
99
109
  };
100
110
 
101
111
  function parseJsonLine(line) {
@@ -118,7 +128,7 @@ function parseJsonLine(line) {
118
128
  method = method.toUpperCase();
119
129
  if (!HTTP_METHODS.has(method)) return null;
120
130
 
121
- const { path, query } = splitTarget(target);
131
+ const { path, query, host } = splitTarget(target);
122
132
  if (!path) return null;
123
133
  const explicitQuery = pick(obj, JSON_KEYS.query);
124
134
  const status = Number(pick(obj, JSON_KEYS.status));
@@ -127,6 +137,7 @@ function parseJsonLine(line) {
127
137
  method,
128
138
  path,
129
139
  query: query || (typeof explicitQuery === 'string' ? explicitQuery.replace(/^\?/, '') : ''),
140
+ host: host || normalizeHost(asString(pick(obj, JSON_KEYS.host))),
130
141
  status: Number.isInteger(status) && status >= 100 && status <= 599 ? status : null,
131
142
  time: parseAnyTime(pick(obj, JSON_KEYS.time)),
132
143
  userAgent: nullIfDash(asString(pick(obj, JSON_KEYS.userAgent))),
@@ -192,17 +203,21 @@ function splitRequestLine(s) {
192
203
  if (parts.length < 2) return null;
193
204
  const method = parts[0].toUpperCase();
194
205
  if (!HTTP_METHODS.has(method)) return null;
195
- const { path, query } = splitTarget(parts[1]);
206
+ const { path, query, host } = splitTarget(parts[1]);
196
207
  if (!path) return null;
197
- return { method, path, query };
208
+ return { method, path, query, host };
198
209
  }
199
210
 
200
211
  function splitTarget(target) {
201
212
  let t = target;
202
- // Absolute URL (ALB, some proxies) → keep only the path + query.
203
- const abs = /^[a-z][a-z0-9+.-]*:\/\/[^/?#]*/i.exec(t);
204
- if (abs) t = t.slice(abs[0].length) || '/';
205
- if (!t.startsWith('/')) return { path: null, query: '' };
213
+ let host = null;
214
+ // Absolute URL (ALB, some proxies) → keep only the path + query, note the host.
215
+ const abs = /^[a-z][a-z0-9+.-]*:\/\/([^/?#]*)/i.exec(t);
216
+ if (abs) {
217
+ host = normalizeHost(abs[1]);
218
+ t = t.slice(abs[0].length) || '/';
219
+ }
220
+ if (!t.startsWith('/')) return { path: null, query: '', host };
206
221
  const hash = t.indexOf('#');
207
222
  if (hash >= 0) t = t.slice(0, hash);
208
223
  const q = t.indexOf('?');
@@ -210,7 +225,15 @@ function splitTarget(target) {
210
225
  const query = q >= 0 ? t.slice(q + 1) : '';
211
226
  path = path.replace(/\/{2,}/g, '/');
212
227
  if (path.length > 1 && path.endsWith('/')) path = path.slice(0, -1);
213
- return { path, query };
228
+ return { path, query, host };
229
+ }
230
+
231
+ /** "API.Example.com:443" → "api.example.com"; "[::1]:80" → "[::1]". */
232
+ function normalizeHost(h) {
233
+ if (!h || h === '-') return null;
234
+ const s = String(h).trim().toLowerCase().replace(/^.*@/, '');
235
+ const m = /^(\[[^\]]+\]|[^:]+)(?::\d+)?$/.exec(s);
236
+ return m ? m[1] : null;
214
237
  }
215
238
 
216
239
  /** Query-string parameter NAMES only — values are never kept. */
@@ -27,18 +27,28 @@ class UsageAggregator {
27
27
  * @param {object} [opts.matcher] from createRouteMatcher (import --spec)
28
28
  * @param {number} [opts.sinceMs] drop records older than this epoch ms
29
29
  */
30
- constructor({ clientSpec, matcher = null, sinceMs = null }) {
30
+ constructor({ clientSpec, matcher = null, sinceMs = null, hosts = [] }) {
31
31
  this.clientSpec = clientSpec;
32
32
  this.matcher = matcher;
33
33
  this.sinceMs = sinceMs;
34
+ this.hostTest = compileHostFilter(hosts);
35
+ this.hostsSeen = new Map();
34
36
  this.endpoints = new Map();
35
37
  this.unmatchedCount = 0;
36
- this.stats = { requests: 0, matched: 0, unmatched: 0, notFound: 0, tooOld: 0, overflow: 0 };
38
+ this.stats = { requests: 0, matched: 0, unmatched: 0, notFound: 0, tooOld: 0, otherHost: 0, overflow: 0 };
37
39
  this.from = null;
38
40
  this.to = null;
39
41
  }
40
42
 
41
43
  add(record) {
44
+ // One load balancer often fronts several APIs by host name. Requests for
45
+ // another host must not count as usage of this API.
46
+ if (record.host) {
47
+ if (this.hostTest && !this.hostTest(record.host)) { this.stats.otherHost++; return; }
48
+ if (this.hostsSeen.size < 50 || this.hostsSeen.has(record.host)) {
49
+ this.hostsSeen.set(record.host, (this.hostsSeen.get(record.host) || 0) + 1);
50
+ }
51
+ }
42
52
  if (this.sinceMs !== null && record.time !== null && record.time < this.sinceMs) {
43
53
  this.stats.tooOld++;
44
54
  return;
@@ -161,6 +171,18 @@ class UsageAggregator {
161
171
  }
162
172
  }
163
173
 
174
+ /** --host values: exact host names, or globs like "*.example.com". Null = no filter. */
175
+ function compileHostFilter(hosts) {
176
+ const list = (hosts || []).filter(Boolean).map((h) => String(h).toLowerCase());
177
+ if (list.length === 0) return null;
178
+ const tests = list.map((h) => {
179
+ if (!h.includes('*')) return (x) => x === h;
180
+ const re = new RegExp('^' + h.split('*').map((p) => p.replace(/[.+?^${}()|[\]\\]/g, '\\$&')).join('.*') + '$');
181
+ return (x) => re.test(x);
182
+ });
183
+ return (host) => tests.some((t) => t(host));
184
+ }
185
+
164
186
  /** Reads and validates a usage.json. Throws with a readable message. */
165
187
  async function loadUsage(file) {
166
188
  if (!(await fsExtra.pathExists(file))) throw new Error(`Usage file not found: ${file}`);