specshield 3.5.0 → 3.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +61 -0
- package/README.md +114 -2
- package/package.json +1 -1
- package/src/cli.js +4 -0
- package/src/commands/gate.js +42 -5
- package/src/commands/prove.js +149 -0
- package/src/commands/replay.js +195 -0
- package/src/core/configLoader.js +6 -2
- package/src/core/gate/runGate.js +19 -1
- package/src/core/prove/contract.js +241 -0
- package/src/core/prove/execute.js +135 -0
- package/src/core/prove/http.js +104 -0
- package/src/core/prove/index.js +84 -0
- package/src/core/prove/instance.js +400 -0
- package/src/core/prove/relate.js +23 -0
- package/src/core/prove/render.js +169 -0
- package/src/core/prove/repair.js +109 -0
- package/src/core/prove/schema.js +261 -0
- package/src/core/prove/subschema.js +257 -0
- package/src/core/prove/witness.js +702 -0
- package/src/core/replay/runReplay.js +169 -0
- package/src/core/replay/shape.js +100 -0
- package/src/core/replay/traffic.js +168 -0
- package/src/core/security/diff.js +304 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,66 @@
|
|
|
1
1
|
# SpecShield CLI changelog
|
|
2
2
|
|
|
3
|
+
## Unreleased — security regressions in the gate
|
|
4
|
+
|
|
5
|
+
### Added
|
|
6
|
+
|
|
7
|
+
- `specshield gate --security` (or `gate.security: true`) blocks removed or weakened authentication, dropped OAuth scopes, new endpoints with forgotten auth, and newly exposed credentials or regulated data. Lower-risk changes are reported for review. Off by default. Findings use the existing shadow mode and waivers.
|
|
8
|
+
|
|
9
|
+
## Unreleased — proof, not opinion: `specshield prove` and `specshield replay`
|
|
10
|
+
|
|
11
|
+
### Added
|
|
12
|
+
|
|
13
|
+
- **`specshield prove <base> <target>`**: for every breaking change, the exact request that
|
|
14
|
+
worked before and fails now. For a response change, it gives a body the new spec allows that
|
|
15
|
+
clients of the old spec reject.
|
|
16
|
+
- **How a witness is found.** Candidate values for the changed location are enums, bounds just
|
|
17
|
+
past the new limits, pattern matches, every `oneOf` branch, and absence. Each is validated
|
|
18
|
+
with ajv against both full specs.
|
|
19
|
+
- **Isolation.** The rest of the witness is repaired to satisfy both specs. Each request
|
|
20
|
+
witness carries a control that differs only at the change, which proves the cause.
|
|
21
|
+
- **Results.** Each change is one of: proven, example (real but not rejectable: a removed
|
|
22
|
+
optional field), compatible (with a structural reason), or no proof (with the reason why).
|
|
23
|
+
- **A sound subschema check** behind the *compatible* verdicts. It recognizes added optional
|
|
24
|
+
properties, extended enums, relaxed bounds, widened types and numeric formats, and `oneOf`
|
|
25
|
+
branches the engine reports as changed only because they were reordered.
|
|
26
|
+
- Recursive schemas are compared coinductively.
|
|
27
|
+
- It assumes a closed world (clients send only documented fields).
|
|
28
|
+
- **`prove --against <url>`** sends every witness to a running build and reports what it does.
|
|
29
|
+
- *confirmed*: the witness is rejected while its control is accepted.
|
|
30
|
+
- *not reproduced*: the build still accepts it, so the spec and the code disagree.
|
|
31
|
+
- *inconclusive*: credentials or test data, not the change.
|
|
32
|
+
- Writes are sent only with `--include-mutating`; remote hosts only with `--allow-remote`.
|
|
33
|
+
- **`specshield replay`** replays a seeded sample of real traffic against the new build.
|
|
34
|
+
- **Sources:** HAR files, or access logs (nginx/Apache, ALB, JSON lines).
|
|
35
|
+
- **Sampling:** stratified per operation.
|
|
36
|
+
- **References:** the old build side by side (`--baseline`), the recorded responses, or the
|
|
37
|
+
old spec as the oracle (`--old-spec`, which works from status-only access logs).
|
|
38
|
+
- **Verdicts:** broken (worked before, fails now), changed (lost fields or changed types), or
|
|
39
|
+
inconclusive (auth, throttling, a 404 that missing test data could explain).
|
|
40
|
+
- **Safety:** captured credentials are never replayed, query values are hidden unless
|
|
41
|
+
`--show-values`, and writes are never rebuilt from body-less logs.
|
|
42
|
+
- **`gate --prove`** (also `gate.prove: true` in `.specshield.yml`) attaches the proof to every
|
|
43
|
+
blocking finding, in the terminal and as a collapsible *Proof* section in the GitHub job summary.
|
|
44
|
+
- Form-encoded request bodies (OpenAPI 3 `application/x-www-form-urlencoded`, Swagger 2.0
|
|
45
|
+
`formData`) are proven on their parsed values and sent bracket-encoded.
|
|
46
|
+
|
|
47
|
+
### Tested on real data
|
|
48
|
+
|
|
49
|
+
- **GitHub REST API**, Apr → Oct 2026, 112 findings: 35 proven, 11 by example, 41 compatible
|
|
50
|
+
(reordered `oneOf` rule branches, widened numeric formats, narrowed responses), 25 without
|
|
51
|
+
proof (removed status codes and shared schemas). 1.1 s.
|
|
52
|
+
- **Stripe**, Mar → Sep 2026, 112 findings: 89 proven, 19 by example, 4 without proof. Its
|
|
53
|
+
1,972 per-endpoint changes ran in 16 s, after two fixes:
|
|
54
|
+
- spec components are now registered with ajv once, instead of recompiled per schema;
|
|
55
|
+
- the subschema check now compares lazily with memoization, instead of inlining.
|
|
56
|
+
- **Kubernetes** (Swagger 2.0), Jul → Oct 2026, 103 findings: 58 proven. 384 per-endpoint
|
|
57
|
+
"field became required" findings on responses are shown compatible.
|
|
58
|
+
- Every proof on all three was isolated to its own change.
|
|
59
|
+
- **A Spring Boot service**, old and new builds side by side: all 6 proofs confirmed on the new
|
|
60
|
+
build, and all 6 *not reproduced* on a build whose code hadn't changed. Replay from a HAR,
|
|
61
|
+
and from an nginx log with `--old-spec`, found the removed field, the type change and the
|
|
62
|
+
rejected writes.
|
|
63
|
+
|
|
3
64
|
## 3.5.0 — 2026-10-29 — gate, fix, safe removal
|
|
4
65
|
|
|
5
66
|
### A gate teams can leave blocking: `specshield gate`
|
package/README.md
CHANGED
|
@@ -42,7 +42,7 @@ What the CLI does:
|
|
|
42
42
|
- [vs. Alternatives](#vs-alternatives) · [Pricing](#pricing)
|
|
43
43
|
|
|
44
44
|
**Commands — local & hosted compare**
|
|
45
|
-
- [Local Compare](#local-compare) · [**Who does a change break? (usage-aware compare)**](#who-does-a-change-break--usage-aware-compare) · [**Safe removal: deprecate → drain → remove**](#safe-removal-deprecate--watch-traffic-drain--remove) · [**Break → fix: `specshield fix`**](#break--fix-generate-the-non-breaking-version) · [**CI gate: `specshield gate`**](#a-gate-you-can-leave-blocking-specshield-gate) · [Remote Compare](#remote-compare) · [Comparison History](#comparison-history) · [Share a Comparison](#share-a-comparison) · [GitHub App PR Checks](#github-app--pr-checks)
|
|
45
|
+
- [Local Compare](#local-compare) · [**Who does a change break? (usage-aware compare)**](#who-does-a-change-break--usage-aware-compare) · [**Safe removal: deprecate → drain → remove**](#safe-removal-deprecate--watch-traffic-drain--remove) · [**Break → fix: `specshield fix`**](#break--fix-generate-the-non-breaking-version) · [**CI gate: `specshield gate`**](#a-gate-you-can-leave-blocking-specshield-gate) · [**Proof: `prove` & `replay`**](#proof-not-opinion-specshield-prove-and-specshield-replay) · [**Security regressions: `gate --security`**](#security-regressions-in-the-gate-specshield-gate---security) · [Remote Compare](#remote-compare) · [Comparison History](#comparison-history) · [Share a Comparison](#share-a-comparison) · [GitHub App PR Checks](#github-app--pr-checks)
|
|
46
46
|
|
|
47
47
|
**Governance — lint & gate**
|
|
48
48
|
- [`specshield govern`](#governance--lint-a-spec-and-gate-the-pr)
|
|
@@ -578,6 +578,118 @@ In GitHub Actions the gate also writes a job summary and annotations (`--no-summ
|
|
|
578
578
|
|
|
579
579
|
---
|
|
580
580
|
|
|
581
|
+
## Security regressions in the gate: `specshield gate --security`
|
|
582
|
+
|
|
583
|
+
The gate also blocks a PR that makes the API less safe. Turn it on with `--security`, or `gate.security: true` in `.specshield.yml`.
|
|
584
|
+
|
|
585
|
+
| Finding (waiver `change`) | Blocks? | Example |
|
|
586
|
+
|---|---|---|
|
|
587
|
+
| `auth-removed` | yes | An operation, or the API-wide `security`, now allows anonymous calls |
|
|
588
|
+
| `auth-scope-reduced` | yes | `[refunds:write, admin]` became `[refunds:write]` |
|
|
589
|
+
| `auth-weakened` | yes | Bearer became basic, an API key moved to the query string, or one required scheme was dropped |
|
|
590
|
+
| `endpoint-added-without-auth` | yes | A new endpoint declares no security while others in the API do |
|
|
591
|
+
| `sensitive-field-exposed` | credentials and regulated data; personal data on public endpoints | New response field `password`, `client_secret`, `cardNumber`, `ssn`; `writeOnly` dropped |
|
|
592
|
+
| `sensitive-field-exposed` | no, reported for review | New `email`, `phone` or token field on an authenticated endpoint |
|
|
593
|
+
| `auth-changed` | no, reported for review | Moved to a different scheme of equal strength |
|
|
594
|
+
| `public-endpoint-added` | no, reported for review | A new endpoint with an explicit `security: []` |
|
|
595
|
+
|
|
596
|
+
A new requirement counts as safe only when every credential that passes it would have passed the old one. Moving a requirement between global and operation level, or adding a scope, reports nothing.
|
|
597
|
+
|
|
598
|
+
Findings work like every other gate finding: `--shadow` reports without failing, and a waiver needs an owner, a reason and an expiry. Usage data never hides them. This reads what the spec declares; it doesn't test what the running service enforces. Sensitive fields are matched by name and `format`, so expect the occasional miss on unusual names.
|
|
599
|
+
|
|
600
|
+
## Proof, not opinion: `specshield prove` and `specshield replay`
|
|
601
|
+
|
|
602
|
+
A diff says *"breaking change"*. A reviewer asks *"does it actually break anything?"* — and a gate that can't answer gets overruled. `prove` answers with the exact request that worked before and fails now; `replay` answers with real traffic.
|
|
603
|
+
|
|
604
|
+
```bash
|
|
605
|
+
specshield prove openapi-v1.yaml openapi-v2.yaml
|
|
606
|
+
```
|
|
607
|
+
|
|
608
|
+
```
|
|
609
|
+
PROOF Field "requestBody.channel" was added to POST /payments (required)
|
|
610
|
+
worked before POST /payments {"amount":1.5,"currency":"USD","customerId":"cus_1"}
|
|
611
|
+
fails now body is missing required field "channel"
|
|
612
|
+
reproduce curl -X POST "$BASE_URL/payments" -H 'Content-Type: application/json' -d '{"amount":1.5,…}'
|
|
613
|
+
|
|
614
|
+
PROOF Response field "amount" changed type — 3 endpoints: GET /payments, POST /payments, GET /payments/{paymentId}
|
|
615
|
+
request GET /payments/PAY-1
|
|
616
|
+
new spec allows {"paymentId":"abc","amount":"abc","currency":"abc","status":"abc"} (200)
|
|
617
|
+
old clients reject it response.amount must be number
|
|
618
|
+
|
|
619
|
+
COMPATIBLE Field "responses.200.message" became required in PATCH /budgets/{id}
|
|
620
|
+
Everything the new schema lets the server send here, the old schema accepted too (checked schema against schema), so old clients can't break on this change.
|
|
621
|
+
```
|
|
622
|
+
|
|
623
|
+
**How a proof is made.** For each breaking change, `prove` generates values the old contract accepts at the changed location — enum values, boundaries just past the new limits, strings matching the old pattern, every `oneOf` branch, leaving the field out — builds a whole request around each, and validates it with ajv against **both** full specs. A *request* witness is accepted by the old spec and rejected by the new one; a *response* witness is a body the new spec lets the server send that a client built on the old spec rejects.
|
|
624
|
+
|
|
625
|
+
Two rules keep proofs honest:
|
|
626
|
+
|
|
627
|
+
- **Isolation.** Every other part of the witness is repaired until both specs accept it, so it fails for this change alone — not for the three other changes in the same release. Where no value can satisfy both (a field that went from integer to string next door), the conflict is listed separately as *also*.
|
|
628
|
+
- **A control.** Each request witness ships with its control: the same request with only the changed value swapped for one the new spec accepts. When the control passes, the change is provably the cause.
|
|
629
|
+
|
|
630
|
+
Every change gets one of four results:
|
|
631
|
+
|
|
632
|
+
| Result | Meaning |
|
|
633
|
+
|---|---|
|
|
634
|
+
| **PROOF** | A witness, validated against both specs as above |
|
|
635
|
+
| **EXAMPLE** | The change is real but schemas can't reject anything — a removed optional response field, a parameter the server will now ignore. Shown as a concrete before/after |
|
|
636
|
+
| **COMPATIBLE** | No witness exists, and a structural argument says why: the new schema accepts everything the old one did (a sound subschema check), only a numeric format changed, `oneOf` branches were reordered, the change only narrows what a server sends |
|
|
637
|
+
| **NO PROOF** | No witness found; the reason says why (a removed status code has no payload to show) |
|
|
638
|
+
|
|
639
|
+
**Against a running build.** `--against http://localhost:8080` sends every witness to the new build and reports what it does: *confirmed* (rejected, while the control got a 2xx), *not reproduced* (the build still accepts it — the spec and the code disagree), *inconclusive* (both rejected: credentials or test data, not the change). Only `GET`/`HEAD`/`OPTIONS` are sent unless you pass `--include-mutating`; only local and private hosts are allowed unless you pass `--allow-remote`; `--header "Authorization: Bearer $TOKEN"` adds credentials.
|
|
640
|
+
|
|
641
|
+
```bash
|
|
642
|
+
specshield prove v1.yaml v2.yaml --against http://localhost:8080 --include-mutating
|
|
643
|
+
# On http://localhost:8080: 6 confirmed
|
|
644
|
+
# … on the build ✔ confirmed — HTTP 400: rejected, while the same request valid under the new spec got HTTP 201
|
|
645
|
+
```
|
|
646
|
+
|
|
647
|
+
**In the gate.** `specshield gate --prove` (or `gate.prove: true`) attaches the proof to every blocking change — in the terminal, and as a collapsible *Proof* section with the request, the reason and a `curl` command in the GitHub job summary.
|
|
648
|
+
|
|
649
|
+
### Replay real traffic: `specshield replay`
|
|
650
|
+
|
|
651
|
+
Synthetic witnesses show what *can* break; replay shows what *does*, for the requests your clients actually send.
|
|
652
|
+
|
|
653
|
+
```bash
|
|
654
|
+
# Side by side: the old build and the new one, same data — the strongest comparison
|
|
655
|
+
specshield replay --from traffic.har --against http://localhost:8082 --baseline http://localhost:8081 --spec v2.yaml
|
|
656
|
+
|
|
657
|
+
# Access logs (nginx/Apache, ALB, JSON lines; .gz ok) record no bodies: use the old spec as the oracle
|
|
658
|
+
specshield replay --from access.log --against http://localhost:8082 --old-spec v1.yaml
|
|
659
|
+
```
|
|
660
|
+
|
|
661
|
+
```
|
|
662
|
+
✖ 10 requests that worked before break on the new build
|
|
663
|
+
✔ 1 ok ✖ 2 broken ▲ 8 changed
|
|
664
|
+
|
|
665
|
+
POST /payments 2 replayed · 2 broken
|
|
666
|
+
✖ POST /payments
|
|
667
|
+
worked before (HTTP 201), fails now (HTTP 400)
|
|
668
|
+
GET /payments/{paymentId} 5 replayed · 4 changed
|
|
669
|
+
▲ GET /payments/PAY-1
|
|
670
|
+
missing receiptUrl; amount number → string
|
|
671
|
+
```
|
|
672
|
+
|
|
673
|
+
- **Sampling.** Up to `--per-operation` requests (default 5) from every operation, `--sample` (default 200) in total, taken round-robin so rare operations survive the cut. Reservoir sampling with a `--seed`: the same seed replays the same requests.
|
|
674
|
+
- **What is compared.** With `--baseline <old build>`, each request goes to both builds — only the code differs. Without it, the recorded HAR response is the reference, and a `404` is *inconclusive* (missing test data explains it as well as a removed route). Response shapes are compared, not values: a field the old response had and the new one lacks, or a type that changed, is *changed*; different data is not. `--old-spec` checks every response against the old contract; `--spec` groups traffic by operation, spots removed routes and flags responses that don't match the new spec.
|
|
675
|
+
- **Safe by default.** Writes are replayed only with `--include-mutating`, and never from access logs (they don't record bodies). Captured credentials — `Authorization`, cookies, anything named like a key or token — are never replayed; pass your own with `--header`. Query values are hidden in the report unless `--show-values`. Only local and private hosts without `--allow-remote`.
|
|
676
|
+
|
|
677
|
+
Exit codes for both: `1` when something breaks (`prove`: breaking changes, as `compare`; `replay`: a request that worked before fails now), `0` otherwise, `2` on errors. `--json` prints everything, including each witness, its control and, with `--against`, what the build did.
|
|
678
|
+
|
|
679
|
+
**Tested on real data.**
|
|
680
|
+
|
|
681
|
+
| Spec pair | Findings | Proven | Example | Compatible | No proof | Time |
|
|
682
|
+
|---|---|---|---|---|---|---|
|
|
683
|
+
| GitHub REST API, Apr → Oct 2026 | 112 | 35 | 11 | 41 | 25 | 1.1 s |
|
|
684
|
+
| Stripe API, Mar → Sep 2026 (form bodies) | 112 | 89 | 19 | 0 | 4 | 16 s |
|
|
685
|
+
| Kubernetes (Swagger 2.0), Jul → Oct 2026 | 103 | 58 | 0 | 0 | 45 | 2.6 s |
|
|
686
|
+
|
|
687
|
+
Every proof was isolated to its own change. The *compatible* findings on GitHub are rule branches GitHub reordered inside a `oneOf` (the engine compares branches by position), numeric formats that only widened, and response fields that became required. *No proof* is almost entirely removed status codes and removed shared schemas, which are proven through the endpoints that used them. On Kubernetes, 384 of the per-endpoint "field became required" findings sit on responses and are shown compatible, while the same fields on requests are proven. Against a Spring Boot service whose old and new builds ran side by side, all 6 proofs were confirmed on the new build. All 6 were *not reproduced* on a build whose code hadn't changed.
|
|
688
|
+
|
|
689
|
+
**Limits.** Witnesses are built for JSON and form bodies. The subschema check assumes a closed world — clients send only fields the spec documents — which is what every compatibility check assumes; it is sound (a *compatible* verdict holds for every instance) but incomplete: regex inclusion, `not` and `if/then` are reported as *no proof* rather than guessed. Pattern strings are generated for the regex features API specs use; lookarounds and backreferences are skipped.
|
|
690
|
+
|
|
691
|
+
---
|
|
692
|
+
|
|
581
693
|
## Remote Compare
|
|
582
694
|
|
|
583
695
|
Sends your specs to SpecShield's hosted diff engine. Useful for keeping the
|
|
@@ -1706,7 +1818,7 @@ Per-subcommand flags:
|
|
|
1706
1818
|
| Code | Meaning |
|
|
1707
1819
|
|---|---|
|
|
1708
1820
|
| `0` | Clean — no breaking changes / deployable / all probes pass |
|
|
1709
|
-
| `1` | Breaking changes found / not deployable / one or more probes failed |
|
|
1821
|
+
| `1` | Breaking changes found / not deployable / one or more probes failed / a replayed request that worked before fails now |
|
|
1710
1822
|
| `2` | Config error, missing token, or runtime error |
|
|
1711
1823
|
|
|
1712
1824
|
---
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "specshield",
|
|
3
|
-
"version": "3.5.
|
|
3
|
+
"version": "3.5.2",
|
|
4
4
|
"description": "Contract compatibility testing for APIs — catch breaking OpenAPI changes before they reach your consumers, with can-i-deploy deploy gating and GitHub PR checks. (a.k.a. bidirectional contract testing.)",
|
|
5
5
|
"main": "src/cli.js",
|
|
6
6
|
"homepage": "https://specshield.io",
|
package/src/cli.js
CHANGED
|
@@ -16,6 +16,8 @@ const deprecationsCommand = require('./commands/deprecations');
|
|
|
16
16
|
const fixCommand = require('./commands/fix');
|
|
17
17
|
const gateCommand = require('./commands/gate');
|
|
18
18
|
const waiversCommand = require('./commands/waivers');
|
|
19
|
+
const proveCommand = require('./commands/prove');
|
|
20
|
+
const replayCommand = require('./commands/replay');
|
|
19
21
|
|
|
20
22
|
const program = new Command();
|
|
21
23
|
|
|
@@ -42,6 +44,8 @@ program.addCommand(deprecationsCommand);
|
|
|
42
44
|
program.addCommand(fixCommand);
|
|
43
45
|
program.addCommand(gateCommand);
|
|
44
46
|
program.addCommand(waiversCommand);
|
|
47
|
+
program.addCommand(proveCommand);
|
|
48
|
+
program.addCommand(replayCommand);
|
|
45
49
|
|
|
46
50
|
program.parseAsync(process.argv).catch((err) => {
|
|
47
51
|
const logger = require('./utils/logger');
|
package/src/commands/gate.js
CHANGED
|
@@ -10,6 +10,7 @@ const { usageOptions } = require('../core/usage/applyUsage');
|
|
|
10
10
|
const W = require('../core/gate/waivers');
|
|
11
11
|
const G = require('../core/gate/git');
|
|
12
12
|
const { plural } = require('../core/usage/format');
|
|
13
|
+
const R = require('../core/prove/render');
|
|
13
14
|
const { uploadGateRun } = require('../api/bdctClient');
|
|
14
15
|
const { getStoredApiKey } = require('../config/localConfig');
|
|
15
16
|
const logger = require('../utils/logger');
|
|
@@ -25,6 +26,8 @@ const gate = new Command('gate')
|
|
|
25
26
|
.option('--max-waiver-days <n>', `Longest a waiver may run before it must be renewed (default: ${W.DEFAULT_MAX_DAYS})`)
|
|
26
27
|
.option('--no-fetch', 'Never run git fetch, even when the baseline commit is missing from a shallow clone')
|
|
27
28
|
.option('--usage <file>', 'usage.json from "specshield usage import" — breaking changes no client uses stop blocking')
|
|
29
|
+
.option('--prove', 'Attach a proof to every breaking change: the request that worked at the baseline and fails now')
|
|
30
|
+
.option('--security', 'Also block security regressions: authentication removed or weakened, scopes dropped, sensitive fields newly exposed')
|
|
28
31
|
.option('--no-summary', 'Skip the GitHub Actions job summary and annotations')
|
|
29
32
|
.option('--upload', 'Record this run on your SpecShield dashboard, for the shadow-mode graduation report (Team plan)')
|
|
30
33
|
.option('--org <key>', 'Org key for --upload (default: gate.org or bdct.org in .specshield.yml)')
|
|
@@ -50,6 +53,8 @@ const gate = new Command('gate')
|
|
|
50
53
|
maxWaiverDays,
|
|
51
54
|
ignore: config.ignore || [],
|
|
52
55
|
usage: usage.usage ? usage : null,
|
|
56
|
+
prove: Boolean(opts.prove || g.prove),
|
|
57
|
+
security: Boolean(opts.security || g.security),
|
|
53
58
|
});
|
|
54
59
|
|
|
55
60
|
if (opts.upload || g.upload) r.upload = await upload(r, { ...opts, org: opts.org || g.org });
|
|
@@ -148,6 +153,9 @@ function label(c) {
|
|
|
148
153
|
return { where, what };
|
|
149
154
|
}
|
|
150
155
|
|
|
156
|
+
const isSecurity = (c) => c.category === 'security';
|
|
157
|
+
const noun = (list) => (list.every(isSecurity) ? 'security regression' : list.some(isSecurity) ? 'blocking change' : 'breaking change');
|
|
158
|
+
|
|
151
159
|
function render(r) {
|
|
152
160
|
const lines = [''];
|
|
153
161
|
lines.push(chalk.bold(` SpecShield™ Gate — ${r.spec}`));
|
|
@@ -169,8 +177,8 @@ function render(r) {
|
|
|
169
177
|
(r.nonBreaking ? chalk.gray(` · ${plural(r.nonBreaking, 'non-breaking change')}`) : ''));
|
|
170
178
|
} else {
|
|
171
179
|
const head = r.verdict === 'would-block'
|
|
172
|
-
? chalk.yellow.bold(` ▲ Shadow mode: would have blocked this change — ${plural(r.blocking.length,
|
|
173
|
-
: chalk.red.bold(` ✖ ${plural(r.blocking.length,
|
|
180
|
+
? chalk.yellow.bold(` ▲ Shadow mode: would have blocked this change — ${plural(r.blocking.length, noun(r.blocking))}`)
|
|
181
|
+
: chalk.red.bold(` ✖ ${plural(r.blocking.length, noun(r.blocking))} block${r.blocking.length === 1 ? 's' : ''} this change`);
|
|
174
182
|
lines.push(head);
|
|
175
183
|
for (const c of r.blocking) {
|
|
176
184
|
const { where, what } = label(c);
|
|
@@ -180,10 +188,20 @@ function render(r) {
|
|
|
180
188
|
if (c.expiredWaiver) {
|
|
181
189
|
lines.push(chalk.yellow(` waiver expired ${c.expiredWaiver.expires} (owner ${c.expiredWaiver.owner}) — renew it or fix the change`));
|
|
182
190
|
}
|
|
191
|
+
if (c.proof) lines.push(...R.lines(c.proof, { pad: ' ' }));
|
|
183
192
|
lines.push(chalk.gray(` intended? ${waiveCommand(c, r.waivers.file)}`));
|
|
184
193
|
}
|
|
185
194
|
}
|
|
186
195
|
|
|
196
|
+
if (r.security && r.security.warnings.length) {
|
|
197
|
+
lines.push('');
|
|
198
|
+
lines.push(chalk.bold(` Security review (${r.security.warnings.length}, not blocking)`));
|
|
199
|
+
for (const c of r.security.warnings) {
|
|
200
|
+
const { where, what } = label(c);
|
|
201
|
+
lines.push(` ${chalk.yellow('▲')} ${what}${where ? chalk.gray(` · ${where}`) : ''}`);
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
|
|
187
205
|
if (r.waived.length) {
|
|
188
206
|
lines.push('');
|
|
189
207
|
lines.push(chalk.bold(` Waived (${r.waived.length})`));
|
|
@@ -245,7 +263,8 @@ function annotate(r) {
|
|
|
245
263
|
const file = `file=${escProp(r.spec)}`;
|
|
246
264
|
for (const c of r.blocking) {
|
|
247
265
|
const { where, what } = label(c);
|
|
248
|
-
const
|
|
266
|
+
const kind = isSecurity(c) ? 'API security regression' : 'Breaking API change';
|
|
267
|
+
const title = r.verdict === 'would-block' ? `${kind} (shadow mode)` : kind;
|
|
249
268
|
const extra = c.expiredWaiver ? ` — waiver expired ${c.expiredWaiver.expires}` : '';
|
|
250
269
|
process.stdout.write(`::${level} ${file},title=${escProp(title)}::${esc(`${what}${where ? ` (${where})` : ''}${extra}`)}\n`);
|
|
251
270
|
}
|
|
@@ -269,8 +288,8 @@ function markdown(r) {
|
|
|
269
288
|
else if (r.blocking.length === 0) md.push(`No breaking changes block this change${r.waived.length ? ` (${r.waived.length} waived)` : ''}.`);
|
|
270
289
|
else {
|
|
271
290
|
md.push(r.verdict === 'would-block'
|
|
272
|
-
? `**Shadow mode:** would have blocked this change — ${plural(r.blocking.length,
|
|
273
|
-
: `**${plural(r.blocking.length,
|
|
291
|
+
? `**Shadow mode:** would have blocked this change — ${plural(r.blocking.length, noun(r.blocking))}.`
|
|
292
|
+
: `**${plural(r.blocking.length, noun(r.blocking))}** block this change.`);
|
|
274
293
|
md.push('');
|
|
275
294
|
md.push('| Change | Where | |');
|
|
276
295
|
md.push('|---|---|---|');
|
|
@@ -278,9 +297,27 @@ function markdown(r) {
|
|
|
278
297
|
const { where, what } = label(c);
|
|
279
298
|
md.push(`| ${cell(what)} | ${cell(where || '—')} | ${c.expiredWaiver ? `waiver expired ${c.expiredWaiver.expires}` : ''} |`);
|
|
280
299
|
}
|
|
300
|
+
const proven = r.blocking.filter((c) => c.proof && (c.proof.status === 'proven' || c.proof.status === 'example'));
|
|
301
|
+
if (proven.length) {
|
|
302
|
+
md.push('');
|
|
303
|
+
md.push(`<details><summary>Proof for ${plural(proven.length, 'change')}</summary>`);
|
|
304
|
+
for (const c of proven) {
|
|
305
|
+
md.push('', `#### ${cell(label(c).what)}`, '', R.markdown(c.proof));
|
|
306
|
+
}
|
|
307
|
+
md.push('', '</details>');
|
|
308
|
+
}
|
|
281
309
|
md.push('');
|
|
282
310
|
md.push(`Intended? Add a waiver with an owner, a reason and an expiry: \`${waiveCommand(r.blocking[0], r.waivers.file)}\``);
|
|
283
311
|
}
|
|
312
|
+
if (r.security && r.security.warnings.length) {
|
|
313
|
+
md.push('');
|
|
314
|
+
md.push(`**Security review (not blocking):**`);
|
|
315
|
+
md.push('');
|
|
316
|
+
for (const c of r.security.warnings) {
|
|
317
|
+
const { where, what } = label(c);
|
|
318
|
+
md.push(`- ${cell(what)}${where ? ` — ${cell(where)}` : ''}`);
|
|
319
|
+
}
|
|
320
|
+
}
|
|
284
321
|
if (r.waived.length) {
|
|
285
322
|
md.push('');
|
|
286
323
|
md.push('<details><summary>Waived</summary>');
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const { Command } = require('commander');
|
|
4
|
+
const chalk = require('chalk');
|
|
5
|
+
const { loadSpec } = require('../core/loadSpec');
|
|
6
|
+
const { parseSpec } = require('../core/parseSpec');
|
|
7
|
+
const { prove } = require('../core/prove');
|
|
8
|
+
const { confirmAll } = require('../core/prove/execute');
|
|
9
|
+
const R = require('../core/prove/render');
|
|
10
|
+
const H = require('../core/prove/http');
|
|
11
|
+
const { plural } = require('../core/usage/format');
|
|
12
|
+
const logger = require('../utils/logger');
|
|
13
|
+
const exitAfterFlush = require('../utils/exitAfterFlush');
|
|
14
|
+
|
|
15
|
+
const prove_ = new Command('prove')
|
|
16
|
+
.description('Proof, not opinion: for every breaking change, the exact request that worked before and fails now — checked against both specs, and optionally against a running build')
|
|
17
|
+
.argument('<base>', 'The spec your clients use today')
|
|
18
|
+
.argument('<target>', 'The new spec')
|
|
19
|
+
.option('--against <url>', 'Send each witness to a running build of the new API and report what it does')
|
|
20
|
+
.option('--header <header...>', 'Header for --against requests, e.g. "Authorization: Bearer $TOKEN" (repeatable)')
|
|
21
|
+
.option('--include-mutating', 'With --against, also send POST/PUT/PATCH/DELETE witnesses (they may change data)')
|
|
22
|
+
.option('--allow-remote', 'Allow --against to target a host that is not local or private')
|
|
23
|
+
.option('--timeout <ms>', 'Per-request timeout for --against', '10000')
|
|
24
|
+
.option('--base-url <url>', 'Base URL to print in the curl commands (default: $BASE_URL, or --against)')
|
|
25
|
+
.option('--warnings', 'Also prove constraint warnings (tightened min/max/length/pattern), which never fail the build')
|
|
26
|
+
.option('--json', 'Output machine-readable JSON')
|
|
27
|
+
.action(async (basePath, targetPath, opts) => {
|
|
28
|
+
try {
|
|
29
|
+
const baseDoc = parseSpec(await loadSpec(basePath), basePath);
|
|
30
|
+
const targetDoc = parseSpec(await loadSpec(targetPath), targetPath);
|
|
31
|
+
const r = prove(baseDoc, targetDoc);
|
|
32
|
+
const shown = opts.warnings ? [...r.breaking, ...r.warnings] : r.breaking;
|
|
33
|
+
|
|
34
|
+
let against = null;
|
|
35
|
+
if (opts.against) {
|
|
36
|
+
against = H.targetUrl(opts.against, { allowRemote: opts.allowRemote });
|
|
37
|
+
const items = shown.filter((c) => c.proof).map((c) => ({ diff: c.proof.diff, witness: c.proof }));
|
|
38
|
+
await confirmAll(items, {
|
|
39
|
+
baseDoc, targetDoc, against,
|
|
40
|
+
headers: H.parseHeaders(opts.header),
|
|
41
|
+
includeMutating: opts.includeMutating,
|
|
42
|
+
timeoutMs: Number(opts.timeout) || 10000,
|
|
43
|
+
});
|
|
44
|
+
const sent = items.filter((i) => i.witness.run.outcome !== 'skipped');
|
|
45
|
+
if (sent.length && sent.every((i) => i.witness.run.outcome === 'error')) {
|
|
46
|
+
throw new Error(`Can't reach ${against}: ${sent[0].witness.run.detail}`);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
if (opts.json) {
|
|
51
|
+
process.stdout.write(JSON.stringify(toJson(r, { baseDoc, targetDoc, against, warnings: opts.warnings }), null, 2) + '\n');
|
|
52
|
+
} else {
|
|
53
|
+
process.stdout.write(render(r, { baseDoc, targetDoc, against, base: opts.baseUrl || against, warnings: opts.warnings }) + '\n');
|
|
54
|
+
}
|
|
55
|
+
exitAfterFlush(r.breaking.length > 0 ? 1 : 0);
|
|
56
|
+
} catch (err) {
|
|
57
|
+
logger.error(`Error: ${err.message}`);
|
|
58
|
+
process.exit(2);
|
|
59
|
+
}
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
function title(doc) {
|
|
63
|
+
const t = (doc.info && doc.info.title) || 'API';
|
|
64
|
+
return { title: t, version: doc.info && doc.info.version ? String(doc.info.version) : null };
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function toJson(r, { baseDoc, targetDoc, against, warnings }) {
|
|
68
|
+
const item = (c) => {
|
|
69
|
+
const { proof, ...change } = c;
|
|
70
|
+
return { ...change, proof: proof ? strip(proof) : null };
|
|
71
|
+
};
|
|
72
|
+
return {
|
|
73
|
+
base: title(baseDoc),
|
|
74
|
+
target: title(targetDoc),
|
|
75
|
+
against,
|
|
76
|
+
summary: { ...r.summary, ...(against ? { onTheBuild: runCounts(r.breaking) } : {}) },
|
|
77
|
+
breakingChanges: r.breaking.map(item),
|
|
78
|
+
warnings: warnings ? r.warnings.map(item) : r.warnings.map(({ proof, ...c }) => c),
|
|
79
|
+
nonBreaking: r.nonBreaking,
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function strip(p) {
|
|
84
|
+
const { diff, ...rest } = p;
|
|
85
|
+
return { ...rest, at: diff };
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function runCounts(changes) {
|
|
89
|
+
const out = {};
|
|
90
|
+
for (const c of changes) if (c.proof && c.proof.run) out[c.proof.run.outcome] = (out[c.proof.run.outcome] || 0) + 1;
|
|
91
|
+
return out;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function render(r, { baseDoc, targetDoc, against, base, warnings }) {
|
|
95
|
+
const b = title(baseDoc);
|
|
96
|
+
const t = title(targetDoc);
|
|
97
|
+
const lines = [''];
|
|
98
|
+
lines.push(chalk.bold(` SpecShield™ Prove — ${t.title}${b.version ? ` ${b.version}` : ''} →${t.version ? ` ${t.version}` : ''}`));
|
|
99
|
+
lines.push(chalk.gray(' ─────────────────────────────────────────'));
|
|
100
|
+
if (r.breaking.length === 0) {
|
|
101
|
+
lines.push(chalk.green(' ✔ No breaking changes — nothing to prove.'));
|
|
102
|
+
} else {
|
|
103
|
+
const s = r.summary;
|
|
104
|
+
const parts = [`${chalk.green(`${s.proven} proven`)} with a failing request`];
|
|
105
|
+
if (s.example) parts.push(chalk.yellow(`${s.example} shown by example`));
|
|
106
|
+
if (s.compatible) parts.push(chalk.cyan(`${s.compatible} compatible on their own`));
|
|
107
|
+
if (s.unproven) parts.push(chalk.gray(`${s.unproven} without proof`));
|
|
108
|
+
lines.push(` ${plural(s.total, 'breaking change')} · ${parts.join(' · ')}`);
|
|
109
|
+
lines.push(chalk.gray(' Every proof is checked against both specs: the old one accepts it, the new one rejects it.'));
|
|
110
|
+
if (against) {
|
|
111
|
+
const counts = runCounts(r.breaking);
|
|
112
|
+
lines.push(` On ${against}: ` + Object.entries(counts).map(([k, n]) => `${n} ${k}`).join(' · '));
|
|
113
|
+
}
|
|
114
|
+
for (const c of r.breaking) block(lines, c, base);
|
|
115
|
+
}
|
|
116
|
+
const provable = r.warnings.filter((c) => c.proof && c.proof.status === 'proven');
|
|
117
|
+
if (warnings && r.warnings.length) {
|
|
118
|
+
lines.push('');
|
|
119
|
+
lines.push(chalk.yellow.bold(` Warnings (${r.warnings.length}) — reported, never fail the build`));
|
|
120
|
+
for (const c of r.warnings) block(lines, c, base);
|
|
121
|
+
} else if (provable.length) {
|
|
122
|
+
lines.push('');
|
|
123
|
+
lines.push(chalk.gray(` ${plural(provable.length, 'constraint warning')} also ${provable.length === 1 ? 'has' : 'have'} a failing request — see them with --warnings.`));
|
|
124
|
+
}
|
|
125
|
+
if (!base && r.breaking.some((c) => c.proof && c.proof.request)) {
|
|
126
|
+
lines.push('');
|
|
127
|
+
lines.push(chalk.gray(' Set BASE_URL to run the curl commands, e.g. export BASE_URL=http://localhost:8080'));
|
|
128
|
+
}
|
|
129
|
+
lines.push('');
|
|
130
|
+
return lines.join('\n');
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/** A merged change's description lists every endpoint (with repeats); name a few, count them all. */
|
|
134
|
+
function headline(c) {
|
|
135
|
+
const eps = [...new Set(c.affectedEndpoints || [])];
|
|
136
|
+
if (eps.length < 2) return c.description;
|
|
137
|
+
const head = c.description.replace(/ \(affects \d+ endpoints?:.*\)$/, '');
|
|
138
|
+
return `${head} — ${eps.length} endpoints: ${eps.slice(0, 3).join(', ')}${eps.length > 3 ? ` and ${eps.length - 3} more` : ''}`;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
function block(lines, c, base) {
|
|
142
|
+
lines.push('');
|
|
143
|
+
lines.push(` ${R.badge(c.proof)} ${headline(c)}`);
|
|
144
|
+
// A merged change already names its endpoints in the description.
|
|
145
|
+
if (c.method && !(c.affectedEndpoints && c.affectedEndpoints.length > 1)) lines.push(chalk.gray(` ${c.method.toUpperCase()} ${c.path}`));
|
|
146
|
+
lines.push(...R.lines(c.proof, { base }));
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
module.exports = prove_;
|