specshield 3.5.0 → 3.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +61 -0
- package/README.md +114 -2
- package/package.json +1 -1
- package/src/cli.js +4 -0
- package/src/commands/gate.js +42 -5
- package/src/commands/prove.js +149 -0
- package/src/commands/replay.js +195 -0
- package/src/core/configLoader.js +6 -2
- package/src/core/gate/runGate.js +19 -1
- package/src/core/prove/contract.js +241 -0
- package/src/core/prove/execute.js +135 -0
- package/src/core/prove/http.js +104 -0
- package/src/core/prove/index.js +84 -0
- package/src/core/prove/instance.js +400 -0
- package/src/core/prove/relate.js +23 -0
- package/src/core/prove/render.js +169 -0
- package/src/core/prove/repair.js +109 -0
- package/src/core/prove/schema.js +261 -0
- package/src/core/prove/subschema.js +257 -0
- package/src/core/prove/witness.js +702 -0
- package/src/core/replay/runReplay.js +169 -0
- package/src/core/replay/shape.js +100 -0
- package/src/core/replay/traffic.js +168 -0
- package/src/core/security/diff.js +304 -0
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Replays sampled real requests against a build of the new API and compares
|
|
5
|
+
* each response with a reference:
|
|
6
|
+
*
|
|
7
|
+
* --baseline <url> the same request sent to the old build, side by side —
|
|
8
|
+
* the strong comparison: same data, only the code differs
|
|
9
|
+
* recorded the response captured in the HAR (or the status in an
|
|
10
|
+
* access log) — weaker: data may differ between then and now
|
|
11
|
+
*
|
|
12
|
+
* Verdicts, per request:
|
|
13
|
+
* broken worked before (2xx/3xx), fails now (4xx/5xx)
|
|
14
|
+
* changed both succeed, but the new response lost fields or changed types
|
|
15
|
+
* ok same outcome, same shape
|
|
16
|
+
* improved failed before, works now
|
|
17
|
+
* inconclusive can't be attributed to the code: credentials, throttling, or a
|
|
18
|
+
* 404 that missing test data explains as well as a removed route
|
|
19
|
+
* error no response from the build
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
const H = require('../prove/http');
|
|
23
|
+
const C = require('../prove/contract');
|
|
24
|
+
const S = require('../prove/schema');
|
|
25
|
+
const { compare } = require('./shape');
|
|
26
|
+
const { createRouteMatcher } = require('../usage/routeMatcher');
|
|
27
|
+
|
|
28
|
+
async function runReplay(records, opts) {
|
|
29
|
+
const {
|
|
30
|
+
against, baseline = null, headers = {}, timeoutMs = 10000, concurrency = 4,
|
|
31
|
+
http = H.send, spec = null, oldSpec = null, basePath,
|
|
32
|
+
} = opts;
|
|
33
|
+
const matcher = spec ? createRouteMatcher(spec, { basePath }) : null;
|
|
34
|
+
const oldMatcher = oldSpec ? createRouteMatcher(oldSpec, { basePath }) : null;
|
|
35
|
+
|
|
36
|
+
const one = async (rec) => {
|
|
37
|
+
const target = rec.path + (rec.query ? `?${rec.query}` : '');
|
|
38
|
+
const req = (base) => ({
|
|
39
|
+
method: rec.method,
|
|
40
|
+
url: base + target,
|
|
41
|
+
headers: { ...rec.headers, ...headers },
|
|
42
|
+
body: rec.body === null ? undefined : rec.body,
|
|
43
|
+
timeoutMs,
|
|
44
|
+
});
|
|
45
|
+
let cand;
|
|
46
|
+
try { cand = await http(req(against)); } catch (err) { return { rec, verdict: 'error', detail: H.describeNetworkError(err) }; }
|
|
47
|
+
|
|
48
|
+
let ref;
|
|
49
|
+
if (baseline) {
|
|
50
|
+
try {
|
|
51
|
+
const r = await http(req(baseline));
|
|
52
|
+
ref = { status: r.status, json: r.json, source: 'baseline' };
|
|
53
|
+
} catch (err) {
|
|
54
|
+
return { rec, cand: brief(cand), verdict: 'error', detail: `baseline: ${H.describeNetworkError(err)}` };
|
|
55
|
+
}
|
|
56
|
+
} else {
|
|
57
|
+
ref = { status: rec.recorded.status, json: rec.recorded.json, source: rec.recorded.source };
|
|
58
|
+
}
|
|
59
|
+
const out = { rec, ref: brief(ref), cand: brief(cand), ...classify(ref, cand, { baseline: Boolean(baseline), removed: removedFromSpec(rec, matcher) }) };
|
|
60
|
+
// The old spec as the oracle: what old clients were promised, checked
|
|
61
|
+
// against what the build sends now. Works even when the reference has no
|
|
62
|
+
// body (access logs).
|
|
63
|
+
if (oldSpec && (out.verdict === 'ok' || out.verdict === 'changed') && cand.status < 400 && cand.json !== undefined) {
|
|
64
|
+
const broken = conformance(oldSpec, oldMatcher, rec, cand);
|
|
65
|
+
if (broken) {
|
|
66
|
+
out.verdict = 'changed';
|
|
67
|
+
out.detail = [out.detail && out.diff ? out.detail : null, `breaks the old contract: ${broken.join('; ')}`].filter(Boolean).join('; ');
|
|
68
|
+
out.oldContract = broken;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
if (spec && cand.status < 400 && cand.json !== undefined) out.specDrift = conformance(spec, matcher, rec, cand);
|
|
72
|
+
if (out.verdict === 'broken' && cand.text) out.excerpt = cand.text.slice(0, 160);
|
|
73
|
+
return out;
|
|
74
|
+
};
|
|
75
|
+
|
|
76
|
+
return H.pool(records, concurrency, one);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function classify(ref, cand, { baseline, removed }) {
|
|
80
|
+
const r = ref.status;
|
|
81
|
+
const c = cand.status;
|
|
82
|
+
if (r === null || r === undefined) {
|
|
83
|
+
return c < 400 ? { verdict: 'ok', detail: `HTTP ${c} (no recorded status to compare with)` } : { verdict: 'inconclusive', detail: `HTTP ${c} (no recorded status to compare with)` };
|
|
84
|
+
}
|
|
85
|
+
const refOk = r < 400;
|
|
86
|
+
const candOk = c < 400;
|
|
87
|
+
if ((c === 401 || c === 403) && r !== c) return { verdict: 'inconclusive', detail: `HTTP ${c} — the build wants credentials; pass them with --header` };
|
|
88
|
+
if (c === 429 || c === 503) return { verdict: 'inconclusive', detail: `HTTP ${c} — throttled or unavailable` };
|
|
89
|
+
|
|
90
|
+
if (refOk && !candOk) {
|
|
91
|
+
if (c === 404 && !baseline && !removed) {
|
|
92
|
+
return { verdict: 'inconclusive', detail: `HTTP ${r} → 404 — the record may not exist here; compare against the old build with --baseline` };
|
|
93
|
+
}
|
|
94
|
+
return { verdict: 'broken', detail: `worked before (HTTP ${r}), fails now (HTTP ${c})${removed ? ' — the operation is gone from the new spec' : ''}` };
|
|
95
|
+
}
|
|
96
|
+
if (!refOk && candOk) return { verdict: 'improved', detail: `failed before (HTTP ${r}), works now (HTTP ${c})` };
|
|
97
|
+
if (!refOk && !candOk) return { verdict: 'ok', detail: `fails as before (HTTP ${r} → ${c})` };
|
|
98
|
+
|
|
99
|
+
if (ref.json !== undefined && cand.json !== undefined) {
|
|
100
|
+
const d = compare(ref.json, cand.json);
|
|
101
|
+
if (d.missing.length || d.changed.length) {
|
|
102
|
+
return { verdict: 'changed', detail: describeDiff(d), diff: d };
|
|
103
|
+
}
|
|
104
|
+
} else if (ref.json !== undefined && cand.json === undefined) {
|
|
105
|
+
return { verdict: 'changed', detail: 'the response is no longer JSON' };
|
|
106
|
+
}
|
|
107
|
+
return { verdict: 'ok', detail: `HTTP ${r} → ${c}` };
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
function describeDiff(d) {
|
|
111
|
+
const parts = [];
|
|
112
|
+
if (d.missing.length) parts.push(`missing ${d.missing.slice(0, 4).map(show).join(', ')}${d.missing.length > 4 ? ` and ${d.missing.length - 4} more` : ''}`);
|
|
113
|
+
for (const x of d.changed.slice(0, 3)) parts.push(`${show(x.path)} ${x.from} → ${x.to}`);
|
|
114
|
+
if (d.changed.length > 3) parts.push(`${d.changed.length - 3} more type changes`);
|
|
115
|
+
return parts.join('; ');
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
function show(path) {
|
|
119
|
+
return path.replace(/^\./, '') || '(root)';
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
function brief(r) {
|
|
123
|
+
return r ? { status: r.status, source: r.source || 'build' } : null;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/** With --spec: is the request's operation gone from the new spec? */
|
|
127
|
+
function removedFromSpec(rec, matcher) {
|
|
128
|
+
if (!matcher) return false;
|
|
129
|
+
const m = matcher.match(rec.method, rec.path);
|
|
130
|
+
return !m || !m.operation;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/** Does the build's response match `spec` for this request's operation? Null when it does (or can't tell). */
|
|
134
|
+
function conformance(spec, matcher, rec, cand) {
|
|
135
|
+
const m = matcher.match(rec.method, rec.path);
|
|
136
|
+
if (!m || !m.operation) return null;
|
|
137
|
+
const contract = C.contractOf(spec, rec.method, m.template);
|
|
138
|
+
const check = C.checkResponse(contract, cand.status, cand.json);
|
|
139
|
+
if (check.ok) return null;
|
|
140
|
+
// One line per problem, not per list element: response[3].amount → response[].amount.
|
|
141
|
+
const msgs = [...new Set(check.errors.map((e) => S.describeError(e, 'response').replace(/\[\d+\]/g, '[]')))];
|
|
142
|
+
return msgs.slice(0, 3);
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/** Per-operation totals and the requests worth showing. */
|
|
146
|
+
function summarize(results) {
|
|
147
|
+
const totals = { ok: 0, broken: 0, changed: 0, improved: 0, inconclusive: 0, error: 0 };
|
|
148
|
+
const ops = new Map();
|
|
149
|
+
for (const r of results) {
|
|
150
|
+
totals[r.verdict]++;
|
|
151
|
+
const k = r.rec.operation;
|
|
152
|
+
if (!ops.has(k)) ops.set(k, { operation: k, total: 0, ok: 0, broken: 0, changed: 0, improved: 0, inconclusive: 0, error: 0, examples: [], drift: [] });
|
|
153
|
+
const o = ops.get(k);
|
|
154
|
+
o.total++;
|
|
155
|
+
o[r.verdict]++;
|
|
156
|
+
if (r.verdict !== 'ok' && o.examples.length < 3) o.examples.push(r);
|
|
157
|
+
if (r.specDrift && o.drift.length < 1) o.drift.push(r);
|
|
158
|
+
}
|
|
159
|
+
return { totals, operations: [...ops.values()].sort(order) };
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
const SEVERITY = ['broken', 'changed', 'error', 'inconclusive', 'improved', 'ok'];
|
|
163
|
+
|
|
164
|
+
function order(a, b) {
|
|
165
|
+
const rank = (o) => SEVERITY.findIndex((v) => o[v] > 0);
|
|
166
|
+
return rank(a) - rank(b) || a.operation.localeCompare(b.operation);
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
module.exports = { runReplay, classify, summarize };
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Response shapes, for telling "the data is different" (expected: another
|
|
5
|
+
* environment, another day) from "the contract is different" (a field is gone,
|
|
6
|
+
* a number became a string).
|
|
7
|
+
*
|
|
8
|
+
* A shape is the set of paths in a JSON value and the types seen at each:
|
|
9
|
+
* { ".items[].id": {number}, ".items[].tag": {string, null}, ... }
|
|
10
|
+
* Arrays fold their elements into "[]"; objects whose keys are data (ids,
|
|
11
|
+
* dates) rather than field names fold into "{*}".
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
const MAX_ELEMENTS = 50;
|
|
15
|
+
|
|
16
|
+
function shape(value) {
|
|
17
|
+
const out = new Map();
|
|
18
|
+
walk(value, '', out, 0);
|
|
19
|
+
return out;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function walk(v, path, out, depth) {
|
|
23
|
+
const t = typeOf(v);
|
|
24
|
+
if (!out.has(path)) out.set(path, new Set());
|
|
25
|
+
out.get(path).add(t);
|
|
26
|
+
if (depth > 20) return;
|
|
27
|
+
if (t === 'array') {
|
|
28
|
+
for (const x of v.slice(0, MAX_ELEMENTS)) walk(x, `${path}[]`, out, depth + 1);
|
|
29
|
+
} else if (t === 'object') {
|
|
30
|
+
const keys = Object.keys(v);
|
|
31
|
+
if (isDataKeyed(keys)) {
|
|
32
|
+
for (const k of keys.slice(0, MAX_ELEMENTS)) walk(v[k], `${path}{*}`, out, depth + 1);
|
|
33
|
+
} else {
|
|
34
|
+
for (const k of keys) walk(v[k], `${path}.${k}`, out, depth + 1);
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function isDataKeyed(keys) {
|
|
40
|
+
if (keys.length > 50) return true;
|
|
41
|
+
if (keys.length < 2) return false;
|
|
42
|
+
return keys.every((k) => /^\d+$/.test(k) || /^[0-9a-f]{8}-[0-9a-f]{4}-/i.test(k) || /^\d{4}-\d{2}-\d{2}/.test(k));
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function typeOf(v) {
|
|
46
|
+
if (v === null) return 'null';
|
|
47
|
+
if (Array.isArray(v)) return 'array';
|
|
48
|
+
return typeof v;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* What a client that read `ref` would miss in `cand`:
|
|
53
|
+
* missing fields present in the reference, absent in the candidate
|
|
54
|
+
* changed fields whose (non-null) type differs
|
|
55
|
+
* A path is only judged where the candidate has data around it — an empty
|
|
56
|
+
* list or a null parent says nothing about the contract.
|
|
57
|
+
*/
|
|
58
|
+
function compare(ref, cand) {
|
|
59
|
+
const a = shape(ref);
|
|
60
|
+
const b = shape(cand);
|
|
61
|
+
const missing = [];
|
|
62
|
+
const changed = [];
|
|
63
|
+
for (const [path, types] of a) {
|
|
64
|
+
if (path === '') continue;
|
|
65
|
+
const parent = parentOf(path);
|
|
66
|
+
const pTypes = b.get(parent);
|
|
67
|
+
if (!pTypes) continue; // reported at the parent, or no data there
|
|
68
|
+
if (!pTypes.has(path.endsWith('[]') ? 'array' : 'object')) continue;
|
|
69
|
+
const bt = b.get(path);
|
|
70
|
+
if (!bt && (path.endsWith('[]') || path.endsWith('{*}'))) continue; // an empty list or map: no data, not a missing field
|
|
71
|
+
if (!bt) {
|
|
72
|
+
if (!types.has('null') || types.size > 1) missing.push(path);
|
|
73
|
+
continue;
|
|
74
|
+
}
|
|
75
|
+
const x = nonNull(types);
|
|
76
|
+
const y = nonNull(bt);
|
|
77
|
+
if (x.length && y.length && !x.some((t) => y.includes(t))) changed.push({ path, from: x.join('|'), to: y.join('|') });
|
|
78
|
+
}
|
|
79
|
+
// A missing parent covers its children: report the top-most only.
|
|
80
|
+
const top = missing.filter((p) => !missing.some((q) => q !== p && (p.startsWith(q + '.') || p.startsWith(q + '[') || p.startsWith(q + '{'))));
|
|
81
|
+
return { missing: top, changed };
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function parentOf(path) {
|
|
85
|
+
if (path.endsWith('[]')) return path.slice(0, -2);
|
|
86
|
+
if (path.endsWith('{*}')) return path.slice(0, -3);
|
|
87
|
+
const i = path.lastIndexOf('.');
|
|
88
|
+
return i <= 0 ? '' : path.slice(0, i);
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function nonNull(types) {
|
|
92
|
+
return [...types].filter((t) => t !== 'null');
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/** ".items[].id" → "items[].id" for display. */
|
|
96
|
+
function display(path) {
|
|
97
|
+
return path.replace(/^\./, '') || '(root)';
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
module.exports = { shape, compare, display };
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Real traffic for `specshield replay`: requests read from HAR files (full
|
|
5
|
+
* requests with their recorded responses) or access logs (method, path, query
|
|
6
|
+
* and status), sampled so every operation is represented — not just the hot
|
|
7
|
+
* ones — and the same seed always picks the same requests.
|
|
8
|
+
*
|
|
9
|
+
* Record = {
|
|
10
|
+
* method, path, query, // query: raw string without "?"
|
|
11
|
+
* headers: { name: value }, // credentials and hop-by-hop headers removed
|
|
12
|
+
* body: string | null, // raw request body (HAR only)
|
|
13
|
+
* recorded: { status, json?, source: 'har' | 'log' },
|
|
14
|
+
* operation: 'GET /orders/{id}', // grouping key
|
|
15
|
+
* matched: boolean, // operation found in --spec
|
|
16
|
+
* }
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
const fs = require('fs');
|
|
20
|
+
const { importLogFile } = require('../usage/importLogs');
|
|
21
|
+
const { createRouteMatcher } = require('../usage/routeMatcher');
|
|
22
|
+
const { templatePath } = require('../har/pathTemplate');
|
|
23
|
+
|
|
24
|
+
const SAFE = new Set(['GET', 'HEAD', 'OPTIONS']);
|
|
25
|
+
|
|
26
|
+
// Never replayed from a capture: credentials (pass your own with --header),
|
|
27
|
+
// and headers that describe the original connection rather than the request.
|
|
28
|
+
const DROP = /^(:|host$|content-length$|connection$|keep-alive$|transfer-encoding$|te$|upgrade$|accept-encoding$|x-forwarded-|x-real-ip$|forwarded$|via$|x-amz|cf-|x-cloud-trace|traceparent$|tracestate$|x-request-id$|x-b3-|sec-|origin$|referer$|if-none-match$|if-modified-since$)/i;
|
|
29
|
+
const SECRET = /auth|token|key|secret|cookie|session|signature|csrf|xsrf|password/i;
|
|
30
|
+
|
|
31
|
+
function isHar(file) {
|
|
32
|
+
if (/\.har$/i.test(file)) return true;
|
|
33
|
+
if (/\.(log|gz|txt|jsonl)$/i.test(file) || file === '-') return false;
|
|
34
|
+
try {
|
|
35
|
+
const fd = fs.openSync(file, 'r');
|
|
36
|
+
const buf = Buffer.alloc(512);
|
|
37
|
+
const n = fs.readSync(fd, buf, 0, 512, 0);
|
|
38
|
+
fs.closeSync(fd);
|
|
39
|
+
return /^\s*\{\s*"log"\s*:/.test(buf.slice(0, n).toString('utf8'));
|
|
40
|
+
} catch { return false; }
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Reads every file, keeps up to `perOperation` requests per operation
|
|
45
|
+
* (reservoir sampling, seeded), then at most `limit` overall — taken round-robin
|
|
46
|
+
* across operations so rare ones survive the cut.
|
|
47
|
+
*/
|
|
48
|
+
async function loadTraffic(files, opts = {}) {
|
|
49
|
+
const {
|
|
50
|
+
format = 'auto', spec = null, basePath, perOperation = 5, limit = 200, seed = 1,
|
|
51
|
+
includeMutating = false, hosts = null,
|
|
52
|
+
} = opts;
|
|
53
|
+
const matcher = spec ? createRouteMatcher(spec, { basePath }) : null;
|
|
54
|
+
const rand = mulberry32(seed);
|
|
55
|
+
const groups = new Map();
|
|
56
|
+
const stats = { files: [], read: 0, considered: 0, mutatingSkipped: 0, bodilessWrites: 0, unmatched: 0, otherHosts: 0 };
|
|
57
|
+
|
|
58
|
+
const take = (rec) => {
|
|
59
|
+
stats.read++;
|
|
60
|
+
if (hosts && rec.host && !hosts.has(rec.host)) { stats.otherHosts++; return; }
|
|
61
|
+
if (!SAFE.has(rec.method)) {
|
|
62
|
+
// An access log has no request bodies: a write rebuilt from it would be a different request.
|
|
63
|
+
if (rec.recorded.source === 'log') { stats.bodilessWrites++; return; }
|
|
64
|
+
if (!includeMutating) { stats.mutatingSkipped++; return; }
|
|
65
|
+
}
|
|
66
|
+
const m = matcher ? matcher.match(rec.method, rec.path) : null;
|
|
67
|
+
if (matcher && (!m || !m.operation)) stats.unmatched++;
|
|
68
|
+
rec.matched = Boolean(m && m.operation);
|
|
69
|
+
rec.operation = rec.matched ? `${rec.method} ${m.template}` : `${rec.method} ${templatePath(rec.path).templated}`;
|
|
70
|
+
stats.considered++;
|
|
71
|
+
let g = groups.get(rec.operation);
|
|
72
|
+
if (!g) { g = { seen: 0, kept: [] }; groups.set(rec.operation, g); }
|
|
73
|
+
g.seen++;
|
|
74
|
+
if (g.kept.length < perOperation) g.kept.push(rec);
|
|
75
|
+
else {
|
|
76
|
+
const j = Math.floor(rand() * g.seen);
|
|
77
|
+
if (j < perOperation) g.kept[j] = rec;
|
|
78
|
+
}
|
|
79
|
+
};
|
|
80
|
+
|
|
81
|
+
for (const file of files) {
|
|
82
|
+
if (isHar(file)) {
|
|
83
|
+
const n = readHar(file, take);
|
|
84
|
+
stats.files.push({ file, kind: 'har', requests: n });
|
|
85
|
+
} else {
|
|
86
|
+
const res = await importLogFile(file, { format, aggregator: { add: (r) => take(fromLog(r)) } });
|
|
87
|
+
stats.files.push({ file, kind: `log (${res.format || 'empty'})`, requests: res.parsed });
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
const keys = [...groups.keys()].sort();
|
|
92
|
+
const records = [];
|
|
93
|
+
for (let round = 0; records.length < limit; round++) {
|
|
94
|
+
let added = false;
|
|
95
|
+
for (const k of keys) {
|
|
96
|
+
const rec = groups.get(k).kept[round];
|
|
97
|
+
if (!rec) continue;
|
|
98
|
+
records.push(rec);
|
|
99
|
+
added = true;
|
|
100
|
+
if (records.length >= limit) break;
|
|
101
|
+
}
|
|
102
|
+
if (!added) break;
|
|
103
|
+
}
|
|
104
|
+
stats.operations = keys.length;
|
|
105
|
+
stats.sampled = records.length;
|
|
106
|
+
return { records, stats, seen: new Map([...groups].map(([k, g]) => [k, g.seen])) };
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
function readHar(file, take) {
|
|
110
|
+
let doc;
|
|
111
|
+
try { doc = JSON.parse(fs.readFileSync(file, 'utf8')); } catch (e) { throw new Error(`${file} is not valid HAR JSON (${e.message})`); }
|
|
112
|
+
if (!doc || !doc.log || !Array.isArray(doc.log.entries)) throw new Error(`${file} is not a HAR file (no log.entries)`);
|
|
113
|
+
let n = 0;
|
|
114
|
+
for (const e of doc.log.entries) {
|
|
115
|
+
if (!e || !e.request || !e.request.url) continue;
|
|
116
|
+
let url;
|
|
117
|
+
try { url = new URL(e.request.url); } catch { continue; }
|
|
118
|
+
const headers = {};
|
|
119
|
+
for (const h of e.request.headers || []) {
|
|
120
|
+
if (!h || typeof h.name !== 'string' || DROP.test(h.name) || SECRET.test(h.name)) continue;
|
|
121
|
+
headers[h.name] = h.value;
|
|
122
|
+
}
|
|
123
|
+
const resText = e.response && e.response.content && typeof e.response.content.text === 'string' && e.response.content.encoding !== 'base64'
|
|
124
|
+
? e.response.content.text : null;
|
|
125
|
+
n++;
|
|
126
|
+
take({
|
|
127
|
+
method: String(e.request.method || 'GET').toUpperCase(),
|
|
128
|
+
path: url.pathname.replace(/\/{2,}/g, '/'),
|
|
129
|
+
query: url.search.replace(/^\?/, ''),
|
|
130
|
+
host: url.hostname.toLowerCase(),
|
|
131
|
+
headers,
|
|
132
|
+
body: e.request.postData && typeof e.request.postData.text === 'string' ? e.request.postData.text : null,
|
|
133
|
+
recorded: { status: e.response && e.response.status > 0 ? e.response.status : null, json: parseJson(resText), source: 'har' },
|
|
134
|
+
});
|
|
135
|
+
}
|
|
136
|
+
return n;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
function fromLog(r) {
|
|
140
|
+
return {
|
|
141
|
+
method: r.method,
|
|
142
|
+
path: r.path,
|
|
143
|
+
query: r.query || '',
|
|
144
|
+
host: r.host || null,
|
|
145
|
+
headers: {},
|
|
146
|
+
body: null,
|
|
147
|
+
recorded: { status: r.status, json: undefined, source: 'log' },
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
function parseJson(text) {
|
|
152
|
+
if (!text || !/^\s*[[{]/.test(text)) return undefined;
|
|
153
|
+
try { return JSON.parse(text); } catch { return undefined; }
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/** Small, fast, seedable PRNG — the same seed samples the same requests. */
|
|
157
|
+
function mulberry32(seed) {
|
|
158
|
+
let a = seed >>> 0;
|
|
159
|
+
return () => {
|
|
160
|
+
a = (a + 0x6D2B79F5) >>> 0;
|
|
161
|
+
let t = a;
|
|
162
|
+
t = Math.imul(t ^ (t >>> 15), t | 1);
|
|
163
|
+
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
|
|
164
|
+
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
module.exports = { loadTraffic, isHar, mulberry32, DROP, SECRET };
|