specshield 3.4.12 → 3.4.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +44 -3
- package/README.md +26 -6
- package/package.json +1 -1
- package/src/commands/compare.js +18 -4
- package/src/commands/usage.js +26 -6
- package/src/core/configLoader.js +7 -0
- package/src/core/normalizeSpec.js +67 -11
- package/src/core/outputFormatter.js +1 -0
- package/src/core/usage/clientIdentity.js +7 -4
- package/src/core/usage/impact.js +52 -15
- package/src/core/usage/logParsers.js +80 -19
- package/src/core/usage/usageStore.js +24 -2
package/CHANGELOG.md
CHANGED
|
@@ -1,6 +1,32 @@
|
|
|
1
1
|
# SpecShield CLI changelog
|
|
2
2
|
|
|
3
|
-
## Unreleased —
|
|
3
|
+
## Unreleased — Kubernetes audit logs, Swagger 2.0 bodies
|
|
4
|
+
|
|
5
|
+
Tested on the Kubernetes project's own published data: the API-server audit log of
|
|
6
|
+
CI job `ci-kubernetes-audit-kind-conformance` (219,507 events, k8s master `765dd1d`) and
|
|
7
|
+
the official `api/openapi-spec/swagger.json` at `765dd1d` and `v1.37.1`. Impact numbers
|
|
8
|
+
were checked against counts taken directly from the raw audit events.
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
|
|
12
|
+
- `usage import` reads **Kubernetes API-server audit logs** (`audit.k8s.io` Events):
|
|
13
|
+
verbs map to HTTP methods, and only the final stage (`ResponseComplete`/`Panic`) is
|
|
14
|
+
counted — the same request is otherwise logged up to three times.
|
|
15
|
+
|
|
16
|
+
### Fixed
|
|
17
|
+
|
|
18
|
+
- **Swagger 2.0 request and response bodies were not compared.** `#/definitions/` refs,
|
|
19
|
+
the `in: body` parameter and `responses.<code>.schema` were ignored, so a removed or
|
|
20
|
+
newly required field reported "No changes". Comparing Kubernetes v1.37.1 → master now
|
|
21
|
+
finds the 18 schemas whose fields became required.
|
|
22
|
+
- Path-level parameters and `$ref` parameters/responses (OpenAPI 3 and Swagger 2.0) are
|
|
23
|
+
now resolved; they were skipped before.
|
|
24
|
+
- Usage impact of request-body changes no longer counts `GET`/`HEAD`/`OPTIONS` calls, and a
|
|
25
|
+
merged change listing the same endpoint twice no longer double-counts its traffic (on
|
|
26
|
+
Kubernetes data: 551 → 12 calls for `events.k8s.io/v1` Event fields, matching the raw
|
|
27
|
+
log exactly).
|
|
28
|
+
|
|
29
|
+
## 3.4.13 — 2026-10-25 — usage-aware compare: "who does this change break?"
|
|
4
30
|
|
|
5
31
|
### Added
|
|
6
32
|
|
|
@@ -15,8 +41,23 @@
|
|
|
15
41
|
that exercise them (exact for query parameters). Breaking changes with zero calls move to
|
|
16
42
|
`unusedBreakingChanges` and don't fail `--fail-on-breaking`, only when the usage covers
|
|
17
43
|
≥ `--usage-min-days` (default 7), is ≤ 30 days old and matches the base spec.
|
|
18
|
-
Also `--usage-ignore-client`, `--
|
|
19
|
-
`.specshield.yml`. JSON output is
|
|
44
|
+
Also `--usage-ignore-client`, `--usage-max-age`, `--usage-min-coverage`,
|
|
45
|
+
`--no-usage-downgrade`, and a `usage:` block in `.specshield.yml`. JSON output is
|
|
46
|
+
unchanged unless `--usage` is given.
|
|
47
|
+
|
|
48
|
+
### Fixed (found by testing on real logs — SEART-GHS API + live spec, Elastic, AWS ALB examples)
|
|
49
|
+
|
|
50
|
+
- Object-typed query parameters (Spring DTOs) were judged by their own name, which never
|
|
51
|
+
appears in URLs, and reported as unused while 1,491 calls sent their fields.
|
|
52
|
+
- Filtered or sampled logs made unseen endpoints look unused: unused changes now pass only
|
|
53
|
+
when ≥ 25% of the spec's operations appear in the logs (`--usage-min-coverage`).
|
|
54
|
+
- The 30-day staleness guard couldn't be relaxed for archived logs (`--usage-max-age`).
|
|
55
|
+
- JSON logs: added the field names of Elastic (`agent`, `response`), Caddy, Traefik, Envoy
|
|
56
|
+
and Google Cloud LB; previously Elastic JSON gave 1 client where the same events in
|
|
57
|
+
combined format gave 36.
|
|
58
|
+
- Multi-word User-Agent products are kept ("Debian APT-HTTP/1.3", not "Debian").
|
|
59
|
+
- `usage import --host` filters by host; ALB/JSON logs from a load balancer serving
|
|
60
|
+
several APIs no longer mix their traffic, and the import warns when hosts are mixed.
|
|
20
61
|
|
|
21
62
|
## 3.4.0 — 2026-07-29 — `govern` command (API governance in the CLI)
|
|
22
63
|
|
package/README.md
CHANGED
|
@@ -278,8 +278,13 @@ specshield usage import /var/log/nginx/access.log* --spec api/openapi.yaml
|
|
|
278
278
|
```
|
|
279
279
|
|
|
280
280
|
- **Formats:** nginx/Apache combined & common, AWS ALB, and JSON lines (nginx JSON
|
|
281
|
-
`log_format`, Kong file-log, API Gateway,
|
|
282
|
-
|
|
281
|
+
`log_format`, Kong file-log, API Gateway, Caddy, Traefik, Envoy, Google Cloud LB, ECS,
|
|
282
|
+
and **Kubernetes API-server audit logs**). Auto-detected; `.gz` files and stdin (`-`)
|
|
283
|
+
work. Force one with `--format combined|alb|json`.
|
|
284
|
+
- **Kubernetes:** point it at the API-server audit log and the cluster's
|
|
285
|
+
`swagger.json`, and identify clients by their authenticated identity:
|
|
286
|
+
`specshield usage import audit.log --spec swagger.json --client field:user.username`.
|
|
287
|
+
Each request is counted once (final audit stage only).
|
|
283
288
|
- **Clients** are identified by User-Agent by default (`MyShop-iOS/5.2.1` → `MyShop-iOS/5.2`).
|
|
284
289
|
Use a better signal if you log one: `--client field:consumer.username` (JSON logs),
|
|
285
290
|
`--client extra:1` (first quoted field after the User-Agent in combined logs), and prefix
|
|
@@ -290,6 +295,11 @@ specshield usage import /var/log/nginx/access.log* --spec api/openapi.yaml
|
|
|
290
295
|
(base paths from `servers` / `basePath`, or `--base-path`). Without it, IDs are guessed.
|
|
291
296
|
- Add more days later with `--append` (a file imported twice is detected and skipped);
|
|
292
297
|
limit the window with `--since 30d`.
|
|
298
|
+
- **One load balancer, several APIs?** ALB and JSON logs record the host; pass
|
|
299
|
+
`--host api.example.com` (repeatable, `*.example.com` globs) so another API's traffic
|
|
300
|
+
doesn't count as usage of this one. The import warns when it sees mixed hosts.
|
|
301
|
+
- **Import logs from every server.** If several instances each write their own log,
|
|
302
|
+
import all of them — logs from one node make the others' clients look like non-users.
|
|
293
303
|
|
|
294
304
|
**2. Compare with impact:**
|
|
295
305
|
|
|
@@ -299,14 +309,20 @@ specshield compare base.yaml target.yaml --usage usage.json --fail-on-breaking
|
|
|
299
309
|
|
|
300
310
|
- **Endpoint and method removals, body/response field changes** → the clients calling that
|
|
301
311
|
operation. Access logs don't show which response fields a client reads, so for field
|
|
302
|
-
changes this is "clients that may be affected", and the report says so.
|
|
312
|
+
changes this is "clients that may be affected", and the report says so. Request-body
|
|
313
|
+
changes (a field removed or made required) count only calls that send a body — `GET`
|
|
314
|
+
traffic can't be affected by them.
|
|
303
315
|
- **Query parameters** are judged exactly: a removed parameter affects only clients that
|
|
304
316
|
*send* it; a parameter that became required affects only clients that *omit* it.
|
|
305
317
|
- **Zero calls → "breaking but unused"**: listed separately and excluded from
|
|
306
318
|
`--fail-on-breaking`. SpecShield only does this when the evidence is good enough — at
|
|
307
|
-
least `--usage-min-days` (default 7) of logs, the newest no older than
|
|
308
|
-
that
|
|
309
|
-
|
|
319
|
+
least `--usage-min-days` (default 7) of logs, the newest no older than `--usage-max-age`
|
|
320
|
+
(default 30) days, calls that match the base spec, and at least `--usage-min-coverage`
|
|
321
|
+
(default 25) percent of the spec's operations seen in the logs — fewer usually means the
|
|
322
|
+
logs are filtered, sampled or from one server of several. Otherwise the change stays
|
|
323
|
+
breaking and the report says why. `--no-usage-downgrade` keeps everything breaking.
|
|
324
|
+
- Object-typed query parameters (e.g. a Spring `@ModelAttribute` DTO) are sent as their
|
|
325
|
+
properties, not under their own name, so they're judged by who calls the endpoint.
|
|
310
326
|
- Leave health checks and bots out: `--usage-ignore-client 'kube-probe/*' --usage-ignore-client healthcheck`.
|
|
311
327
|
- `--json` adds `impact` to each change, plus `unusedBreakingChanges`, `summary.unusedBreaking`
|
|
312
328
|
and a `usage` block. Without `--usage` the JSON is unchanged.
|
|
@@ -1275,6 +1291,8 @@ ignore:
|
|
|
1275
1291
|
# file: usage.json
|
|
1276
1292
|
# ignoreClients: ["kube-probe/*"]
|
|
1277
1293
|
# minDays: 7
|
|
1294
|
+
# maxAgeDays: 30
|
|
1295
|
+
# minCoverage: 25
|
|
1278
1296
|
# downgrade: true
|
|
1279
1297
|
|
|
1280
1298
|
# Hosted compare (set --remote on the CLI to override).
|
|
@@ -1337,6 +1355,8 @@ specshield compare <base> <target> [options]
|
|
|
1337
1355
|
| `--usage <file>` | `usage.json` from `specshield usage import` — show which clients each breaking change affects |
|
|
1338
1356
|
| `--usage-ignore-client <pattern>` | Leave a client out of the impact (repeatable; `*` globs, otherwise substring) |
|
|
1339
1357
|
| `--usage-min-days <n>` | Days of usage needed before an unused breaking change stops failing the build (default `7`) |
|
|
1358
|
+
| `--usage-max-age <days>` | The newest log line may be at most this old for unused changes to pass (default `30`) |
|
|
1359
|
+
| `--usage-min-coverage <pct>` | Percent of spec operations that must appear in the logs for unused changes to pass (default `25`) |
|
|
1340
1360
|
| `--no-usage-downgrade` | Annotate impact only; keep unused breaking changes breaking |
|
|
1341
1361
|
| `--timeout <ms>` | Request timeout for remote mode (default `10000`) |
|
|
1342
1362
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "specshield",
|
|
3
|
-
"version": "3.4.
|
|
3
|
+
"version": "3.4.14",
|
|
4
4
|
"description": "Contract compatibility testing for APIs — catch breaking OpenAPI changes before they reach your consumers, with can-i-deploy deploy gating and GitHub PR checks. (a.k.a. bidirectional contract testing.)",
|
|
5
5
|
"main": "src/cli.js",
|
|
6
6
|
"homepage": "https://specshield.io",
|
package/src/commands/compare.js
CHANGED
|
@@ -46,6 +46,8 @@ compare
|
|
|
46
46
|
.option('--usage <file>', 'usage.json from "specshield usage import" — show which clients each breaking change affects')
|
|
47
47
|
.option('--usage-ignore-client <pattern>', 'Leave a client out of the impact, e.g. health checks (repeatable; * globs)', collect, [])
|
|
48
48
|
.option('--usage-min-days <n>', 'Days of usage needed before an unused breaking change stops failing the build (default: 7)')
|
|
49
|
+
.option('--usage-max-age <days>', 'Newest log line may be at most this old for unused changes to pass (default: 30)')
|
|
50
|
+
.option('--usage-min-coverage <pct>', 'Percent of spec operations that must appear in the logs for unused changes to pass (default: 25)')
|
|
49
51
|
.option('--no-usage-downgrade', 'Annotate impact only; keep unused breaking changes as breaking')
|
|
50
52
|
.action(async (base, target, opts) => {
|
|
51
53
|
try {
|
|
@@ -156,10 +158,14 @@ async function applyUsage(result, base, options) {
|
|
|
156
158
|
|
|
157
159
|
const usage = await loadUsage(options.usage);
|
|
158
160
|
const baseSpec = parseSpec(await loadSpec(base), base);
|
|
159
|
-
const
|
|
160
|
-
|
|
161
|
-
throw new Error(
|
|
162
|
-
|
|
161
|
+
const num = (value, flag, max = Infinity) => {
|
|
162
|
+
const n = Number(value);
|
|
163
|
+
if (!Number.isFinite(n) || n < 0 || n > max) throw new Error(`${flag} must be a number from 0 to ${max}, got "${value}"`);
|
|
164
|
+
return n;
|
|
165
|
+
};
|
|
166
|
+
const minDays = num(options.usageMinDays, '--usage-min-days');
|
|
167
|
+
const maxAgeDays = num(options.usageMaxAge, '--usage-max-age');
|
|
168
|
+
const minCoverage = num(options.usageMinCoverage, '--usage-min-coverage', 100);
|
|
163
169
|
|
|
164
170
|
const { result: annotated, meta } = applyUsageImpact(result, {
|
|
165
171
|
usage,
|
|
@@ -168,6 +174,8 @@ async function applyUsage(result, base, options) {
|
|
|
168
174
|
file: path.basename(options.usage),
|
|
169
175
|
ignoreClients: options.usageIgnoreClient,
|
|
170
176
|
minDays,
|
|
177
|
+
maxAgeDays,
|
|
178
|
+
minCoverage,
|
|
171
179
|
downgrade: options.usageDowngrade,
|
|
172
180
|
});
|
|
173
181
|
if (!meta.trusted && meta.totalCalls > 0) {
|
|
@@ -303,6 +311,12 @@ function mergeOptions(config, cliOpts) {
|
|
|
303
311
|
usageMinDays: cliOpts.usageMinDays !== undefined
|
|
304
312
|
? cliOpts.usageMinDays
|
|
305
313
|
: config.usage && config.usage.minDays !== undefined ? config.usage.minDays : 7,
|
|
314
|
+
usageMaxAge: cliOpts.usageMaxAge !== undefined
|
|
315
|
+
? cliOpts.usageMaxAge
|
|
316
|
+
: config.usage && config.usage.maxAgeDays !== undefined ? config.usage.maxAgeDays : 30,
|
|
317
|
+
usageMinCoverage: cliOpts.usageMinCoverage !== undefined
|
|
318
|
+
? cliOpts.usageMinCoverage
|
|
319
|
+
: config.usage && config.usage.minCoverage !== undefined ? config.usage.minCoverage : 25,
|
|
306
320
|
usageDowngrade: cliOpts.usageDowngrade === false
|
|
307
321
|
? false
|
|
308
322
|
: !(config.usage && config.usage.downgrade === false),
|
package/src/commands/usage.js
CHANGED
|
@@ -28,6 +28,7 @@ usage
|
|
|
28
28
|
.option('--format <format>', 'Log format: auto | combined | alb | json', 'auto')
|
|
29
29
|
.option('--client <source>', 'How to identify clients: ua | field:<path> | extra:<n>, optionally hash:-prefixed', 'ua')
|
|
30
30
|
.option('--base-path <path>', 'Path prefix the API is served under, e.g. /api/v1 (default: from the spec servers)')
|
|
31
|
+
.option('--host <name>', 'Only count requests for this host, e.g. api.example.com or *.example.com (repeatable)', collect, [])
|
|
31
32
|
.option('--since <duration>', 'Only count requests newer than this, e.g. 30d')
|
|
32
33
|
.option('--append', 'Add to an existing --out file instead of replacing it')
|
|
33
34
|
.action(async (logs, opts) => {
|
|
@@ -50,7 +51,7 @@ usage
|
|
|
50
51
|
}
|
|
51
52
|
}
|
|
52
53
|
|
|
53
|
-
const aggregator = new UsageAggregator({ clientSpec, matcher, sinceMs });
|
|
54
|
+
const aggregator = new UsageAggregator({ clientSpec, matcher, sinceMs, hosts: opts.host });
|
|
54
55
|
const sources = existing ? [...(existing.sources || [])] : [];
|
|
55
56
|
const seen = new Set(sources.map((s) => s.fingerprint).filter(Boolean));
|
|
56
57
|
const imported = [];
|
|
@@ -85,7 +86,9 @@ usage
|
|
|
85
86
|
});
|
|
86
87
|
await fsExtra.outputFile(opts.out, JSON.stringify(out, null, 2) + '\n');
|
|
87
88
|
|
|
88
|
-
printImportSummary(out, imported, runStats, {
|
|
89
|
+
printImportSummary(out, imported, runStats, {
|
|
90
|
+
outFile: opts.out, matcher, appended: !!existing, hostsSeen: aggregator.hostsSeen, hostFilter: opts.host.length > 0,
|
|
91
|
+
});
|
|
89
92
|
exitAfterFlush(0);
|
|
90
93
|
} catch (err) {
|
|
91
94
|
logger.error(`Error: ${err.message}`);
|
|
@@ -93,7 +96,7 @@ usage
|
|
|
93
96
|
}
|
|
94
97
|
});
|
|
95
98
|
|
|
96
|
-
function printImportSummary(out, imported, run, { outFile, matcher, appended }) {
|
|
99
|
+
function printImportSummary(out, imported, run, { outFile, matcher, appended, hostsSeen, hostFilter }) {
|
|
97
100
|
const lines = [];
|
|
98
101
|
const parsed = imported.reduce((n, r) => n + r.parsed, 0);
|
|
99
102
|
const skipped = imported.reduce((n, r) => n + r.skipped, 0);
|
|
@@ -110,6 +113,12 @@ function printImportSummary(out, imported, run, { outFile, matcher, appended })
|
|
|
110
113
|
lines.push(` ${plural(out.endpoints.length, 'endpoint')} · ${plural(clients.size, 'client')}` +
|
|
111
114
|
(matcher ? ` · ${fmt(run.matched)} matched the spec · ${fmt(run.unmatched)} not in the spec` : ''));
|
|
112
115
|
if (run.notFound) lines.push(chalk.gray(` ${plural(run.notFound, 'request')} to unknown paths answered 404/405 skipped (scanners)`));
|
|
116
|
+
if (run.otherHost) lines.push(chalk.gray(` ${plural(run.otherHost, 'request')} for other hosts skipped (--host)`));
|
|
117
|
+
if (!hostFilter && hostsSeen.size > 1) {
|
|
118
|
+
const top = [...hostsSeen.entries()].sort((a, b) => b[1] - a[1]).slice(0, 4).map(([h, n]) => `${h} (${fmt(n)})`);
|
|
119
|
+
lines.push(chalk.yellow(` Requests for ${hostsSeen.size} hosts are mixed: ${top.join(', ')}${hostsSeen.size > 4 ? ', …' : ''}.`));
|
|
120
|
+
lines.push(chalk.yellow(' If they are different APIs, re-import with --host <name> so they don\'t count as usage of this one.'));
|
|
121
|
+
}
|
|
113
122
|
if (run.tooOld) lines.push(chalk.gray(` ${plural(run.tooOld, 'request')} older than --since skipped`));
|
|
114
123
|
if (run.overflow) lines.push(chalk.yellow(` ${plural(run.overflow, 'request')} dropped: endpoint limit reached (use --spec to group paths)`));
|
|
115
124
|
if (skipped) lines.push(chalk.gray(` ${plural(skipped, 'line')} skipped (not a request)`));
|
|
@@ -144,6 +153,7 @@ usage
|
|
|
144
153
|
const top = Math.max(1, parseInt(opts.top, 10) || 15);
|
|
145
154
|
let unused = null;
|
|
146
155
|
let undocumented = null;
|
|
156
|
+
let calledCount = 0;
|
|
147
157
|
|
|
148
158
|
if (opts.spec) {
|
|
149
159
|
const spec = parseSpec(await loadSpec(opts.spec), opts.spec);
|
|
@@ -155,6 +165,7 @@ usage
|
|
|
155
165
|
if (m && m.operation) called.add(`${m.method} ${m.template}`);
|
|
156
166
|
else undocumented.push({ method: e.method, path: e.path, calls: e.calls, lastSeen: e.lastSeen, clients: e.clients.length });
|
|
157
167
|
}
|
|
168
|
+
calledCount = called.size;
|
|
158
169
|
unused = matcher.operations
|
|
159
170
|
.filter((o) => !called.has(`${o.method} ${o.template}`))
|
|
160
171
|
.map((o) => ({ method: o.method, path: o.template }));
|
|
@@ -170,7 +181,7 @@ usage
|
|
|
170
181
|
return exitAfterFlush(0);
|
|
171
182
|
}
|
|
172
183
|
|
|
173
|
-
process.stdout.write(renderShow(data, { top, unused, undocumented }) + '\n');
|
|
184
|
+
process.stdout.write(renderShow(data, { top, unused, undocumented, seenOps: calledCount }) + '\n');
|
|
174
185
|
exitAfterFlush(0);
|
|
175
186
|
} catch (err) {
|
|
176
187
|
logger.error(`Error: ${err.message}`);
|
|
@@ -178,7 +189,7 @@ usage
|
|
|
178
189
|
}
|
|
179
190
|
});
|
|
180
191
|
|
|
181
|
-
function renderShow(data, { top, unused, undocumented }) {
|
|
192
|
+
function renderShow(data, { top, unused, undocumented, seenOps }) {
|
|
182
193
|
const lines = [];
|
|
183
194
|
const total = data.endpoints.reduce((n, e) => n + e.calls, 0);
|
|
184
195
|
lines.push('');
|
|
@@ -195,7 +206,7 @@ function renderShow(data, { top, unused, undocumented }) {
|
|
|
195
206
|
for (const c of shown) {
|
|
196
207
|
lines.push(chalk.gray(` ${c.id.padEnd(28)} ${pct(c.calls / e.calls).padStart(4)} last seen ${ago(c.lastSeen)}`));
|
|
197
208
|
}
|
|
198
|
-
if (e.clients.length > shown.length) lines.push(chalk.gray(` …and ${e.clients.length - shown.length
|
|
209
|
+
if (e.clients.length > shown.length) lines.push(chalk.gray(` …and ${plural(e.clients.length - shown.length, 'more client')}`));
|
|
199
210
|
}
|
|
200
211
|
if (data.endpoints.length > top) lines.push(chalk.gray(` …${data.endpoints.length - top} more endpoint(s) — raise --top`));
|
|
201
212
|
|
|
@@ -203,6 +214,11 @@ function renderShow(data, { top, unused, undocumented }) {
|
|
|
203
214
|
lines.push('');
|
|
204
215
|
lines.push(chalk.green.bold(` ✔ NEVER CALLED IN THIS WINDOW (${unused.length}) — candidates for deprecation`));
|
|
205
216
|
lines.push(chalk.gray(' ─────────────────────────────────────────'));
|
|
217
|
+
const total = unused.length + seenOps;
|
|
218
|
+
if (total > 0 && seenOps / total < 0.25) {
|
|
219
|
+
lines.push(chalk.yellow(` Only ${seenOps} of ${total} operations appear in these logs. If the logs are filtered,`));
|
|
220
|
+
lines.push(chalk.yellow(' sampled or from one server of several, this list is not reliable.'));
|
|
221
|
+
}
|
|
206
222
|
if (unused.length === 0) lines.push(chalk.gray(' Every operation in the spec was called.'));
|
|
207
223
|
for (const o of unused) lines.push(` ${chalk.green('○')} ${o.method} ${o.path}`);
|
|
208
224
|
}
|
|
@@ -220,4 +236,8 @@ function renderShow(data, { top, unused, undocumented }) {
|
|
|
220
236
|
return lines.join('\n');
|
|
221
237
|
}
|
|
222
238
|
|
|
239
|
+
function collect(value, previous) {
|
|
240
|
+
return previous.concat([value]);
|
|
241
|
+
}
|
|
242
|
+
|
|
223
243
|
module.exports = usage;
|
package/src/core/configLoader.js
CHANGED
|
@@ -116,6 +116,11 @@ function validateConfig(config, filePath) {
|
|
|
116
116
|
if (u.minDays !== undefined && !(Number(u.minDays) >= 0)) {
|
|
117
117
|
throw new Error(`Config error in "${filePath}": usage.minDays must be a number ≥ 0`);
|
|
118
118
|
}
|
|
119
|
+
for (const key of ['maxAgeDays', 'minCoverage']) {
|
|
120
|
+
if (u[key] !== undefined && !(Number(u[key]) >= 0)) {
|
|
121
|
+
throw new Error(`Config error in "${filePath}": usage.${key} must be a number ≥ 0`);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
119
124
|
if (u.downgrade !== undefined && typeof u.downgrade !== 'boolean') {
|
|
120
125
|
throw new Error(`Config error in "${filePath}": usage.downgrade must be a boolean`);
|
|
121
126
|
}
|
|
@@ -123,6 +128,8 @@ function validateConfig(config, filePath) {
|
|
|
123
128
|
file: u.file || null,
|
|
124
129
|
ignoreClients: (u.ignoreClients || []).filter(Boolean).map(String),
|
|
125
130
|
minDays: u.minDays !== undefined ? Number(u.minDays) : undefined,
|
|
131
|
+
maxAgeDays: u.maxAgeDays !== undefined ? Number(u.maxAgeDays) : undefined,
|
|
132
|
+
minCoverage: u.minCoverage !== undefined ? Number(u.minCoverage) : undefined,
|
|
126
133
|
downgrade: u.downgrade,
|
|
127
134
|
};
|
|
128
135
|
} else {
|
|
@@ -3,6 +3,12 @@
|
|
|
3
3
|
/**
|
|
4
4
|
* Normalize an OpenAPI spec into a flat, consistent internal model.
|
|
5
5
|
* Resolves simple $ref references from components.schemas.
|
|
6
|
+
*
|
|
7
|
+
* Swagger 2.0 is mapped onto the same model: `definitions` stand in for
|
|
8
|
+
* components.schemas, the `in: body` parameter becomes the request body,
|
|
9
|
+
* `responses.<code>.schema` is the response schema, and non-body parameters
|
|
10
|
+
* carry their type inline instead of under `schema`. Path-level parameters
|
|
11
|
+
* and `$ref` parameters/responses are resolved for both versions.
|
|
6
12
|
*/
|
|
7
13
|
function normalizeSpec(spec) {
|
|
8
14
|
if (!spec || typeof spec !== 'object') {
|
|
@@ -10,7 +16,12 @@ function normalizeSpec(spec) {
|
|
|
10
16
|
}
|
|
11
17
|
|
|
12
18
|
const components = spec.components || {};
|
|
13
|
-
const schemas = components.schemas || {};
|
|
19
|
+
const schemas = components.schemas || spec.definitions || {};
|
|
20
|
+
const ctx = {
|
|
21
|
+
schemas,
|
|
22
|
+
parameters: components.parameters || spec.parameters || {},
|
|
23
|
+
responses: components.responses || spec.responses || {},
|
|
24
|
+
};
|
|
14
25
|
|
|
15
26
|
const endpoints = {};
|
|
16
27
|
|
|
@@ -24,7 +35,7 @@ function normalizeSpec(spec) {
|
|
|
24
35
|
for (const method of HTTP_METHODS) {
|
|
25
36
|
if (!pathItem[method]) continue;
|
|
26
37
|
const operation = pathItem[method];
|
|
27
|
-
methods[method] = normalizeOperation(operation,
|
|
38
|
+
methods[method] = normalizeOperation(operation, pathItem.parameters, ctx);
|
|
28
39
|
}
|
|
29
40
|
|
|
30
41
|
if (Object.keys(methods).length > 0) {
|
|
@@ -39,25 +50,62 @@ function normalizeSpec(spec) {
|
|
|
39
50
|
};
|
|
40
51
|
}
|
|
41
52
|
|
|
42
|
-
function normalizeOperation(operation,
|
|
53
|
+
function normalizeOperation(operation, pathParams, ctx) {
|
|
54
|
+
const { schemas } = ctx;
|
|
55
|
+
const params = mergeParameters(pathParams, operation.parameters, ctx);
|
|
56
|
+
const body = params.find((p) => p.in === 'body'); // Swagger 2.0
|
|
43
57
|
return {
|
|
44
58
|
operationId: operation.operationId || null,
|
|
45
59
|
summary: operation.summary || null,
|
|
46
|
-
parameters: normalizeParameters(
|
|
47
|
-
requestBody:
|
|
48
|
-
|
|
60
|
+
parameters: normalizeParameters(params.filter((p) => p.in !== 'body'), schemas),
|
|
61
|
+
requestBody: body
|
|
62
|
+
? (body.schema ? resolveSchema(body.schema, schemas) : null)
|
|
63
|
+
: normalizeRequestBody(operation.requestBody, schemas),
|
|
64
|
+
responses: normalizeResponses(operation.responses || {}, ctx),
|
|
49
65
|
};
|
|
50
66
|
}
|
|
51
67
|
|
|
68
|
+
/** Path-level parameters apply to every operation; an operation-level one with the same (in, name) overrides. */
|
|
69
|
+
function mergeParameters(pathParams, opParams, ctx) {
|
|
70
|
+
const byKey = new Map();
|
|
71
|
+
for (const raw of [...(Array.isArray(pathParams) ? pathParams : []), ...(Array.isArray(opParams) ? opParams : [])]) {
|
|
72
|
+
const p = derefParameter(raw, ctx);
|
|
73
|
+
if (!p || !p.name || !p.in) continue;
|
|
74
|
+
byKey.set(`${p.in}:${p.name}`, p);
|
|
75
|
+
}
|
|
76
|
+
return [...byKey.values()];
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function derefParameter(p, ctx) {
|
|
80
|
+
if (!p || typeof p !== 'object' || !p.$ref) return p;
|
|
81
|
+
const name = refName(p.$ref, ['#/components/parameters/', '#/parameters/']);
|
|
82
|
+
return name ? ctx.parameters[name] || null : null;
|
|
83
|
+
}
|
|
84
|
+
|
|
52
85
|
function normalizeParameters(params, schemas) {
|
|
53
86
|
return params.map((p) => ({
|
|
54
87
|
name: p.name,
|
|
55
88
|
in: p.in,
|
|
56
89
|
required: Boolean(p.required),
|
|
57
|
-
|
|
90
|
+
// Swagger 2.0 non-body parameters declare type/format/items/enum inline.
|
|
91
|
+
schema: p.schema
|
|
92
|
+
? resolveSchema(p.schema, schemas)
|
|
93
|
+
: p.type ? resolveSchema(inlineParamSchema(p), schemas) : null,
|
|
58
94
|
}));
|
|
59
95
|
}
|
|
60
96
|
|
|
97
|
+
function inlineParamSchema(p) {
|
|
98
|
+
const { name, in: _in, required, description, collectionFormat, allowEmptyValue, ...schema } = p;
|
|
99
|
+
return schema;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
function refName(ref, prefixes) {
|
|
103
|
+
for (const prefix of prefixes) {
|
|
104
|
+
if (typeof ref === 'string' && ref.startsWith(prefix)) return ref.slice(prefix.length);
|
|
105
|
+
}
|
|
106
|
+
return null;
|
|
107
|
+
}
|
|
108
|
+
|
|
61
109
|
function normalizeRequestBody(requestBody, schemas) {
|
|
62
110
|
if (!requestBody) return null;
|
|
63
111
|
const content = requestBody.content || {};
|
|
@@ -66,10 +114,18 @@ function normalizeRequestBody(requestBody, schemas) {
|
|
|
66
114
|
return resolveSchema(mediaType.schema, schemas);
|
|
67
115
|
}
|
|
68
116
|
|
|
69
|
-
function normalizeResponses(responses,
|
|
117
|
+
function normalizeResponses(responses, ctx) {
|
|
118
|
+
const { schemas } = ctx;
|
|
70
119
|
const result = {};
|
|
71
|
-
for (const [statusCode,
|
|
120
|
+
for (const [statusCode, raw] of Object.entries(responses)) {
|
|
121
|
+
const response = raw && raw.$ref
|
|
122
|
+
? ctx.responses[refName(raw.$ref, ['#/components/responses/', '#/responses/'])]
|
|
123
|
+
: raw;
|
|
72
124
|
if (!response) continue;
|
|
125
|
+
if (response.schema) { // Swagger 2.0
|
|
126
|
+
result[statusCode] = resolveSchema(response.schema, schemas);
|
|
127
|
+
continue;
|
|
128
|
+
}
|
|
73
129
|
const content = response.content || {};
|
|
74
130
|
const mediaType = content['application/json'] || Object.values(content)[0];
|
|
75
131
|
result[statusCode] = mediaType && mediaType.schema
|
|
@@ -96,8 +152,8 @@ function resolveSchema(schema, schemas, depth = 0) {
|
|
|
96
152
|
if (depth > 10) return { type: 'circular' };
|
|
97
153
|
|
|
98
154
|
if (schema.$ref) {
|
|
99
|
-
const
|
|
100
|
-
const resolved = schemas[
|
|
155
|
+
const name = refName(schema.$ref, ['#/components/schemas/', '#/definitions/']);
|
|
156
|
+
const resolved = name ? schemas[name] : undefined;
|
|
101
157
|
if (!resolved) return { type: 'unknown', ref: schema.$ref };
|
|
102
158
|
return resolveSchema(resolved, schemas, depth + 1);
|
|
103
159
|
}
|
|
@@ -117,6 +117,7 @@ function formatUsageHeader(u) {
|
|
|
117
117
|
` Usage: ${u.file} · ${u.window.from ? `${day(u.window.from)} → ${day(u.window.to)}` : 'no timestamps'}` +
|
|
118
118
|
(span !== null ? ` (${span} days)` : '') +
|
|
119
119
|
` · ${plural(u.matchedCalls, 'call')} · ${plural(u.clients, 'client')}` +
|
|
120
|
+
(u.operationsTotal ? ` · ${u.operationsSeen}/${u.operationsTotal} operations seen` : '') +
|
|
120
121
|
(u.ignoredCalls ? ` · ${fmt(u.ignoredCalls)} ignored` : '')));
|
|
121
122
|
if (!u.downgrade.enabled && u.downgrade.reason !== 'disabled by --no-usage-downgrade') {
|
|
122
123
|
lines.push(chalk.yellow(` Unused breaking changes still fail the build: ${u.downgrade.reason}.`));
|
|
@@ -58,10 +58,13 @@ function normalizeUserAgent(ua) {
|
|
|
58
58
|
const s = ua.trim();
|
|
59
59
|
if (!s) return null;
|
|
60
60
|
if (/^Mozilla\//.test(s)) return 'browser';
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
61
|
+
// The product is everything up to the first "/<digit>", so multi-word
|
|
62
|
+
// products survive: "Debian APT-HTTP/1.3 (…)" → "Debian APT-HTTP/1.3",
|
|
63
|
+
// "Chef Client/10.18.2 (…)" → "Chef Client/10.18".
|
|
64
|
+
const m = /^([^/();]{1,60}?)\/v?(\d+)(?:\.(\d+))?/.exec(s);
|
|
65
|
+
if (m) return `${m[1].trim()}/${m[2]}${m[3] !== undefined ? `.${m[3]}` : ''}`;
|
|
66
|
+
const name = s.split('(')[0].replace(/[\s;,]+$/, '') || s;
|
|
67
|
+
return name.length > 60 ? name.slice(0, 60) : name;
|
|
65
68
|
}
|
|
66
69
|
|
|
67
70
|
module.exports = { parseClientSpec, clientIdOf, normalizeUserAgent };
|
package/src/core/usage/impact.js
CHANGED
|
@@ -22,13 +22,24 @@ const { toMs } = require('./usageStore');
|
|
|
22
22
|
* downgradeBlocker). Otherwise they are annotated and stay breaking.
|
|
23
23
|
*/
|
|
24
24
|
|
|
25
|
-
const STALE_AFTER_DAYS = 30;
|
|
26
25
|
const DAY = 86400e3;
|
|
27
26
|
|
|
27
|
+
// Changes that only matter to clients SENDING a body. GET/HEAD/OPTIONS carry
|
|
28
|
+
// no body, so their calls can't be affected — counting them (list/watch
|
|
29
|
+
// traffic dominates real logs) inflated impact several-fold in real Kubernetes
|
|
30
|
+
// audit data. FIELD_BECAME_REQUIRED is listed because it only breaks senders;
|
|
31
|
+
// on the response side it is a stronger guarantee, not a break.
|
|
32
|
+
const REQUEST_BODY_TYPES = new Set([
|
|
33
|
+
'REQUEST_FIELD_REMOVED', 'REQUEST_FIELD_TYPE_CHANGED', 'REQUEST_REQUIRED_FIELD_ADDED',
|
|
34
|
+
'REQUEST_TYPE_CHANGED', 'FIELD_BECAME_REQUIRED',
|
|
35
|
+
]);
|
|
36
|
+
const BODYLESS_METHODS = new Set(['GET', 'HEAD', 'OPTIONS']);
|
|
37
|
+
|
|
28
38
|
function applyUsageImpact(result, opts) {
|
|
29
39
|
const {
|
|
30
40
|
usage, baseSpec, baseNorm, file = 'usage.json',
|
|
31
|
-
ignoreClients = [], minDays = 7,
|
|
41
|
+
ignoreClients = [], minDays = 7, maxAgeDays = 30, minCoverage = 25,
|
|
42
|
+
downgrade = true, now = Date.now(),
|
|
32
43
|
} = opts;
|
|
33
44
|
|
|
34
45
|
const ignore = compileClientPatterns(ignoreClients);
|
|
@@ -75,10 +86,15 @@ function applyUsageImpact(result, opts) {
|
|
|
75
86
|
clients: clientIds.size,
|
|
76
87
|
matchRate: totalCalls > 0 ? round3(matchedCalls / totalCalls) : 0,
|
|
77
88
|
basePaths: matcher.basePaths,
|
|
89
|
+
// Share of the spec's operations that appear in the logs. A low share
|
|
90
|
+
// usually means the logs are filtered (one route exported), sampled, or
|
|
91
|
+
// from one node of several — and then "0 calls" proves nothing.
|
|
92
|
+
operationsSeen: opIndex.size,
|
|
93
|
+
operationsTotal: matcher.operations.length,
|
|
78
94
|
trusted: matchedCalls > 0,
|
|
79
95
|
};
|
|
80
96
|
|
|
81
|
-
const blocker = downgradeBlocker(meta, { downgrade, minDays });
|
|
97
|
+
const blocker = downgradeBlocker(meta, { downgrade, minDays, maxAgeDays, minCoverage });
|
|
82
98
|
meta.downgrade = { enabled: blocker === null, reason: blocker };
|
|
83
99
|
|
|
84
100
|
const ctx = { opIndex, baseSpec, baseNorm, trusted: meta.trusted };
|
|
@@ -100,27 +116,35 @@ function applyUsageImpact(result, opts) {
|
|
|
100
116
|
}
|
|
101
117
|
|
|
102
118
|
/** Why "zero calls" isn't strong enough evidence to stop failing the build — or null. */
|
|
103
|
-
function downgradeBlocker(meta, { downgrade, minDays }) {
|
|
119
|
+
function downgradeBlocker(meta, { downgrade, minDays, maxAgeDays, minCoverage }) {
|
|
104
120
|
if (!downgrade) return 'disabled by --no-usage-downgrade';
|
|
105
121
|
if (!meta.trusted) return 'no call in the usage file matches an operation in the base spec';
|
|
106
122
|
if (meta.days === null) return 'the logs had no timestamps, so the time window is unknown';
|
|
107
123
|
if (meta.days < minDays) {
|
|
108
124
|
return `the usage covers ${meta.days} day(s); at least ${minDays} are needed so weekly/monthly clients show up (--usage-min-days)`;
|
|
109
125
|
}
|
|
110
|
-
if (meta.staleDays >
|
|
111
|
-
return `the newest log line is ${Math.floor(meta.staleDays)} days old — re-import recent logs`;
|
|
126
|
+
if (meta.staleDays > maxAgeDays) {
|
|
127
|
+
return `the newest log line is ${Math.floor(meta.staleDays)} days old — re-import recent logs (--usage-max-age)`;
|
|
128
|
+
}
|
|
129
|
+
const coverage = meta.operationsTotal > 0 ? (meta.operationsSeen / meta.operationsTotal) * 100 : 0;
|
|
130
|
+
if (coverage < minCoverage) {
|
|
131
|
+
return `only ${meta.operationsSeen} of ${meta.operationsTotal} operations in the spec appear in the logs — ` +
|
|
132
|
+
'they look filtered, sampled or from one server of several (--usage-min-coverage)';
|
|
112
133
|
}
|
|
113
134
|
return null;
|
|
114
135
|
}
|
|
115
136
|
|
|
116
137
|
function computeImpact(change, { opIndex, baseSpec, baseNorm }) {
|
|
117
|
-
|
|
138
|
+
let endpoints = endpointsOf(change);
|
|
118
139
|
if (endpoints.length === 0) {
|
|
119
140
|
return { verdict: 'unknown', basis: null, reason: 'not tied to a specific endpoint' };
|
|
120
141
|
}
|
|
121
142
|
if (!endpoints.every(([m, p]) => operationExists(baseSpec, m, p))) {
|
|
122
143
|
return { verdict: 'unknown', basis: null, reason: 'endpoint not found in the base spec' };
|
|
123
144
|
}
|
|
145
|
+
if (REQUEST_BODY_TYPES.has(change.type)) {
|
|
146
|
+
endpoints = endpoints.filter(([m]) => !BODYLESS_METHODS.has(m));
|
|
147
|
+
}
|
|
124
148
|
|
|
125
149
|
// Query-parameter changes can be judged per parameter: logs record the
|
|
126
150
|
// query string. Header/path/cookie params and bodies can't — fall back to
|
|
@@ -128,10 +152,14 @@ function computeImpact(change, { opIndex, baseSpec, baseNorm }) {
|
|
|
128
152
|
let param = null;
|
|
129
153
|
const pm = /^parameters\.(.+)$/.exec(change.field || '');
|
|
130
154
|
if (pm && endpoints.length === 1) {
|
|
131
|
-
const
|
|
155
|
+
const p = findParam(baseNorm, endpoints[0][0], endpoints[0][1], pm[1]);
|
|
132
156
|
const byClient = opIndex.get(`${endpoints[0][0]} ${endpoints[0][1]}`);
|
|
133
157
|
const truncated = byClient && [...byClient.values()].some((c) => c.paramsTruncated);
|
|
134
|
-
|
|
158
|
+
// An object-typed query parameter (Spring's @ModelAttribute DTOs, form
|
|
159
|
+
// explode, deepObject) never appears in the URL under its own name — its
|
|
160
|
+
// properties do. Judging it by name would call a heavily used parameter
|
|
161
|
+
// unused, so judge it by who calls the endpoint instead.
|
|
162
|
+
if (p && p.in === 'query' && !isObjectSchema(p.schema) && !truncated) param = pm[1];
|
|
135
163
|
}
|
|
136
164
|
const omitters = change.type === 'PARAMETER_BECAME_REQUIRED';
|
|
137
165
|
|
|
@@ -181,12 +209,14 @@ function computeImpact(change, { opIndex, baseSpec, baseNorm }) {
|
|
|
181
209
|
|
|
182
210
|
function endpointsOf(change) {
|
|
183
211
|
if (change.path && change.method) return [[change.method.toUpperCase(), change.path]];
|
|
184
|
-
|
|
212
|
+
// A merged $ref change lists an endpoint once per occurrence (request AND
|
|
213
|
+
// response); count each endpoint's traffic once.
|
|
214
|
+
const seen = new Map();
|
|
185
215
|
for (const e of change.affectedEndpoints || []) {
|
|
186
216
|
const m = /^([A-Za-z]+) (\S+)$/.exec(e);
|
|
187
|
-
if (m)
|
|
217
|
+
if (m) seen.set(`${m[1].toUpperCase()} ${m[2]}`, [m[1].toUpperCase(), m[2]]);
|
|
188
218
|
}
|
|
189
|
-
return
|
|
219
|
+
return [...seen.values()];
|
|
190
220
|
}
|
|
191
221
|
|
|
192
222
|
function operationExists(spec, method, path) {
|
|
@@ -194,10 +224,17 @@ function operationExists(spec, method, path) {
|
|
|
194
224
|
return Boolean(item && item[method.toLowerCase()]);
|
|
195
225
|
}
|
|
196
226
|
|
|
197
|
-
function
|
|
227
|
+
function findParam(baseNorm, method, path, name) {
|
|
198
228
|
const op = baseNorm && baseNorm.endpoints && baseNorm.endpoints[path] && baseNorm.endpoints[path][method.toLowerCase()];
|
|
199
|
-
|
|
200
|
-
|
|
229
|
+
return (op && (op.parameters || []).find((x) => x.name === name)) || null;
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
/** Unknown schemas count as objects: when in doubt, don't judge by the name. */
|
|
233
|
+
function isObjectSchema(schema) {
|
|
234
|
+
if (!schema) return true;
|
|
235
|
+
// Scalars and arrays serialize under the parameter's own name (?tag=a&tag=b).
|
|
236
|
+
if (['string', 'integer', 'number', 'boolean', 'array'].includes(schema.type)) return false;
|
|
237
|
+
return true;
|
|
201
238
|
}
|
|
202
239
|
|
|
203
240
|
/**
|
|
@@ -11,6 +11,8 @@
|
|
|
11
11
|
*
|
|
12
12
|
* method upper-case HTTP method
|
|
13
13
|
* path URL path only — no scheme/host, no query string
|
|
14
|
+
* host lower-case host without port, when the log records it (ALB,
|
|
15
|
+
* JSON with a host field, absolute request URLs), else null
|
|
14
16
|
* query raw query string without the leading '?' ('' when absent)
|
|
15
17
|
* status numeric status code, or null
|
|
16
18
|
* time epoch ms, or null when the line carries no parseable timestamp
|
|
@@ -23,7 +25,8 @@
|
|
|
23
25
|
* combined nginx / Apache "combined" and "common" log formats
|
|
24
26
|
* alb AWS Application Load Balancer access logs
|
|
25
27
|
* json one JSON object per line — nginx JSON log_format, Kong file-log,
|
|
26
|
-
* API Gateway JSON access logs, Envoy/Traefik JSON, ECS-style logs
|
|
28
|
+
* API Gateway JSON access logs, Envoy/Traefik JSON, ECS-style logs,
|
|
29
|
+
* and Kubernetes API-server audit logs (audit.k8s.io Events)
|
|
27
30
|
*/
|
|
28
31
|
|
|
29
32
|
const HTTP_METHODS = new Set(['GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS', 'TRACE', 'CONNECT']);
|
|
@@ -83,19 +86,27 @@ function parseAlb(line) {
|
|
|
83
86
|
|
|
84
87
|
// ─── JSON lines ─────────────────────────────────────────────────────────────
|
|
85
88
|
|
|
89
|
+
// Field names used by common producers: nginx/Apache JSON log_format, Kong
|
|
90
|
+
// file-log, Caddy, Traefik, Envoy, Google Cloud HTTP(S) LB, AWS API Gateway,
|
|
91
|
+
// ECS/OpenTelemetry, Elastic examples. Order matters — first non-empty wins,
|
|
92
|
+
// so nested keys come before same-named containers ("response.status" before
|
|
93
|
+
// "response").
|
|
86
94
|
const JSON_KEYS = {
|
|
87
|
-
method: ['method', 'request_method', 'httpMethod', 'http_method', 'verb',
|
|
88
|
-
'request.method', 'req.method', 'http.method', 'http.request.method'],
|
|
89
|
-
path: ['path', 'uri', 'request_uri', 'url', 'request.uri', 'request.url', 'req.url',
|
|
90
|
-
'req.path', 'http.url', 'http.target', 'url.original', 'url.path'],
|
|
95
|
+
method: ['method', 'request_method', 'httpMethod', 'http_method', 'verb', 'RequestMethod',
|
|
96
|
+
'request.method', 'req.method', 'http.method', 'http.request.method', 'httpRequest.requestMethod'],
|
|
97
|
+
path: ['path', 'uri', 'request_uri', 'url', 'RequestPath', 'request.uri', 'request.url', 'req.url',
|
|
98
|
+
'req.path', 'http.url', 'http.target', 'url.original', 'url.path', 'httpRequest.requestUrl'],
|
|
91
99
|
query: ['query', 'query_string', 'args', 'url.query'],
|
|
92
|
-
|
|
93
|
-
'
|
|
100
|
+
host: ['host', 'http_host', 'RequestHost', 'request.host', 'request.headers.host', 'request.headers.Host',
|
|
101
|
+
'req.headers.host', 'headers.host', 'url.domain', 'http.host', 'domainName'],
|
|
102
|
+
status: ['status', 'status_code', 'statusCode', 'response_code', 'DownstreamStatus', 'OriginStatus',
|
|
103
|
+
'response.status', 'res.statusCode', 'http.status_code', 'http.response.status_code',
|
|
104
|
+
'httpRequest.status', 'response'],
|
|
94
105
|
time: ['time', 'timestamp', '@timestamp', 'time_iso8601', 'time_local', 'requestTime',
|
|
95
|
-
'requestTimeEpoch', 'started_at', 'ts', 'start_time'],
|
|
96
|
-
userAgent: ['user_agent', 'http_user_agent', 'userAgent', 'user_agent.original',
|
|
97
|
-
'request.headers.user-agent', '
|
|
98
|
-
'http.user_agent', '
|
|
106
|
+
'requestTimeEpoch', 'started_at', 'ts', 'start_time', 'StartUTC'],
|
|
107
|
+
userAgent: ['user_agent', 'http_user_agent', 'userAgent', 'agent', 'user_agent.original',
|
|
108
|
+
'request.headers.user-agent', 'request.headers.User-Agent', 'req.headers.user-agent',
|
|
109
|
+
'headers.user-agent', 'http.user_agent', 'request_User-Agent', 'httpRequest.userAgent'],
|
|
99
110
|
};
|
|
100
111
|
|
|
101
112
|
function parseJsonLine(line) {
|
|
@@ -104,6 +115,9 @@ function parseJsonLine(line) {
|
|
|
104
115
|
let obj;
|
|
105
116
|
try { obj = JSON.parse(trimmed); } catch { return null; }
|
|
106
117
|
if (!obj || typeof obj !== 'object') return null;
|
|
118
|
+
if (obj.kind === 'Event' && typeof obj.apiVersion === 'string' && obj.apiVersion.startsWith('audit.k8s.io/')) {
|
|
119
|
+
return parseK8sAuditEvent(obj);
|
|
120
|
+
}
|
|
107
121
|
|
|
108
122
|
let method = pick(obj, JSON_KEYS.method);
|
|
109
123
|
let target = pick(obj, JSON_KEYS.path);
|
|
@@ -118,7 +132,7 @@ function parseJsonLine(line) {
|
|
|
118
132
|
method = method.toUpperCase();
|
|
119
133
|
if (!HTTP_METHODS.has(method)) return null;
|
|
120
134
|
|
|
121
|
-
const { path, query } = splitTarget(target);
|
|
135
|
+
const { path, query, host } = splitTarget(target);
|
|
122
136
|
if (!path) return null;
|
|
123
137
|
const explicitQuery = pick(obj, JSON_KEYS.query);
|
|
124
138
|
const status = Number(pick(obj, JSON_KEYS.status));
|
|
@@ -127,6 +141,7 @@ function parseJsonLine(line) {
|
|
|
127
141
|
method,
|
|
128
142
|
path,
|
|
129
143
|
query: query || (typeof explicitQuery === 'string' ? explicitQuery.replace(/^\?/, '') : ''),
|
|
144
|
+
host: host || normalizeHost(asString(pick(obj, JSON_KEYS.host))),
|
|
130
145
|
status: Number.isInteger(status) && status >= 100 && status <= 599 ? status : null,
|
|
131
146
|
time: parseAnyTime(pick(obj, JSON_KEYS.time)),
|
|
132
147
|
userAgent: nullIfDash(asString(pick(obj, JSON_KEYS.userAgent))),
|
|
@@ -135,6 +150,40 @@ function parseJsonLine(line) {
|
|
|
135
150
|
};
|
|
136
151
|
}
|
|
137
152
|
|
|
153
|
+
// ─── Kubernetes audit events ────────────────────────────────────────────────
|
|
154
|
+
|
|
155
|
+
// Kubernetes verbs → HTTP methods (kubernetes.io/docs/reference/access-authn-authz/authorization/#determine-the-request-verb)
|
|
156
|
+
const K8S_VERBS = {
|
|
157
|
+
get: 'GET', list: 'GET', watch: 'GET',
|
|
158
|
+
create: 'POST', update: 'PUT', patch: 'PATCH',
|
|
159
|
+
delete: 'DELETE', deletecollection: 'DELETE',
|
|
160
|
+
};
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* One request is logged once per audit stage (RequestReceived,
|
|
164
|
+
* ResponseStarted for long-running calls, ResponseComplete). Count only the
|
|
165
|
+
* final stage so each request is counted once.
|
|
166
|
+
*/
|
|
167
|
+
function parseK8sAuditEvent(obj) {
|
|
168
|
+
if (obj.stage !== 'ResponseComplete' && obj.stage !== 'Panic') return null;
|
|
169
|
+
const method = K8S_VERBS[obj.verb];
|
|
170
|
+
if (!method || typeof obj.requestURI !== 'string') return null;
|
|
171
|
+
const { path, query } = splitTarget(obj.requestURI);
|
|
172
|
+
if (!path) return null;
|
|
173
|
+
const code = obj.responseStatus && Number(obj.responseStatus.code);
|
|
174
|
+
return {
|
|
175
|
+
method,
|
|
176
|
+
path,
|
|
177
|
+
query,
|
|
178
|
+
host: null,
|
|
179
|
+
status: Number.isInteger(code) && code >= 100 && code <= 599 ? code : null,
|
|
180
|
+
time: parseAnyTime(obj.requestReceivedTimestamp || obj.stageTimestamp),
|
|
181
|
+
userAgent: nullIfDash(asString(obj.userAgent)),
|
|
182
|
+
extras: [],
|
|
183
|
+
json: obj,
|
|
184
|
+
};
|
|
185
|
+
}
|
|
186
|
+
|
|
138
187
|
/** Value at a dotted path. Tries the literal key first ("user_agent.original" as one key). */
|
|
139
188
|
function lookup(obj, dotted) {
|
|
140
189
|
if (obj == null) return undefined;
|
|
@@ -192,17 +241,21 @@ function splitRequestLine(s) {
|
|
|
192
241
|
if (parts.length < 2) return null;
|
|
193
242
|
const method = parts[0].toUpperCase();
|
|
194
243
|
if (!HTTP_METHODS.has(method)) return null;
|
|
195
|
-
const { path, query } = splitTarget(parts[1]);
|
|
244
|
+
const { path, query, host } = splitTarget(parts[1]);
|
|
196
245
|
if (!path) return null;
|
|
197
|
-
return { method, path, query };
|
|
246
|
+
return { method, path, query, host };
|
|
198
247
|
}
|
|
199
248
|
|
|
200
249
|
function splitTarget(target) {
|
|
201
250
|
let t = target;
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
if (
|
|
251
|
+
let host = null;
|
|
252
|
+
// Absolute URL (ALB, some proxies) → keep only the path + query, note the host.
|
|
253
|
+
const abs = /^[a-z][a-z0-9+.-]*:\/\/([^/?#]*)/i.exec(t);
|
|
254
|
+
if (abs) {
|
|
255
|
+
host = normalizeHost(abs[1]);
|
|
256
|
+
t = t.slice(abs[0].length) || '/';
|
|
257
|
+
}
|
|
258
|
+
if (!t.startsWith('/')) return { path: null, query: '', host };
|
|
206
259
|
const hash = t.indexOf('#');
|
|
207
260
|
if (hash >= 0) t = t.slice(0, hash);
|
|
208
261
|
const q = t.indexOf('?');
|
|
@@ -210,7 +263,15 @@ function splitTarget(target) {
|
|
|
210
263
|
const query = q >= 0 ? t.slice(q + 1) : '';
|
|
211
264
|
path = path.replace(/\/{2,}/g, '/');
|
|
212
265
|
if (path.length > 1 && path.endsWith('/')) path = path.slice(0, -1);
|
|
213
|
-
return { path, query };
|
|
266
|
+
return { path, query, host };
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
/** "API.Example.com:443" → "api.example.com"; "[::1]:80" → "[::1]". */
|
|
270
|
+
function normalizeHost(h) {
|
|
271
|
+
if (!h || h === '-') return null;
|
|
272
|
+
const s = String(h).trim().toLowerCase().replace(/^.*@/, '');
|
|
273
|
+
const m = /^(\[[^\]]+\]|[^:]+)(?::\d+)?$/.exec(s);
|
|
274
|
+
return m ? m[1] : null;
|
|
214
275
|
}
|
|
215
276
|
|
|
216
277
|
/** Query-string parameter NAMES only — values are never kept. */
|
|
@@ -27,18 +27,28 @@ class UsageAggregator {
|
|
|
27
27
|
* @param {object} [opts.matcher] from createRouteMatcher (import --spec)
|
|
28
28
|
* @param {number} [opts.sinceMs] drop records older than this epoch ms
|
|
29
29
|
*/
|
|
30
|
-
constructor({ clientSpec, matcher = null, sinceMs = null }) {
|
|
30
|
+
constructor({ clientSpec, matcher = null, sinceMs = null, hosts = [] }) {
|
|
31
31
|
this.clientSpec = clientSpec;
|
|
32
32
|
this.matcher = matcher;
|
|
33
33
|
this.sinceMs = sinceMs;
|
|
34
|
+
this.hostTest = compileHostFilter(hosts);
|
|
35
|
+
this.hostsSeen = new Map();
|
|
34
36
|
this.endpoints = new Map();
|
|
35
37
|
this.unmatchedCount = 0;
|
|
36
|
-
this.stats = { requests: 0, matched: 0, unmatched: 0, notFound: 0, tooOld: 0, overflow: 0 };
|
|
38
|
+
this.stats = { requests: 0, matched: 0, unmatched: 0, notFound: 0, tooOld: 0, otherHost: 0, overflow: 0 };
|
|
37
39
|
this.from = null;
|
|
38
40
|
this.to = null;
|
|
39
41
|
}
|
|
40
42
|
|
|
41
43
|
add(record) {
|
|
44
|
+
// One load balancer often fronts several APIs by host name. Requests for
|
|
45
|
+
// another host must not count as usage of this API.
|
|
46
|
+
if (record.host) {
|
|
47
|
+
if (this.hostTest && !this.hostTest(record.host)) { this.stats.otherHost++; return; }
|
|
48
|
+
if (this.hostsSeen.size < 50 || this.hostsSeen.has(record.host)) {
|
|
49
|
+
this.hostsSeen.set(record.host, (this.hostsSeen.get(record.host) || 0) + 1);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
42
52
|
if (this.sinceMs !== null && record.time !== null && record.time < this.sinceMs) {
|
|
43
53
|
this.stats.tooOld++;
|
|
44
54
|
return;
|
|
@@ -161,6 +171,18 @@ class UsageAggregator {
|
|
|
161
171
|
}
|
|
162
172
|
}
|
|
163
173
|
|
|
174
|
+
/** --host values: exact host names, or globs like "*.example.com". Null = no filter. */
|
|
175
|
+
function compileHostFilter(hosts) {
|
|
176
|
+
const list = (hosts || []).filter(Boolean).map((h) => String(h).toLowerCase());
|
|
177
|
+
if (list.length === 0) return null;
|
|
178
|
+
const tests = list.map((h) => {
|
|
179
|
+
if (!h.includes('*')) return (x) => x === h;
|
|
180
|
+
const re = new RegExp('^' + h.split('*').map((p) => p.replace(/[.+?^${}()|[\]\\]/g, '\\$&')).join('.*') + '$');
|
|
181
|
+
return (x) => re.test(x);
|
|
182
|
+
});
|
|
183
|
+
return (host) => tests.some((t) => t(host));
|
|
184
|
+
}
|
|
185
|
+
|
|
164
186
|
/** Reads and validates a usage.json. Throws with a readable message. */
|
|
165
187
|
async function loadUsage(file) {
|
|
166
188
|
if (!(await fsExtra.pathExists(file))) throw new Error(`Usage file not found: ${file}`);
|