sparkforensics-cli 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/sparkforensics-analyze.mjs +288 -0
- package/package.json +29 -0
- package/vendor-core/analyzer.js +167 -0
- package/vendor-core/assert-never.js +3 -0
- package/vendor-core/cli/budgets.js +203 -0
- package/vendor-core/cli/collect-run.js +107 -0
- package/vendor-core/core-count.js +68 -0
- package/vendor-core/core-locality-ratio.js +54 -0
- package/vendor-core/core-time-series.js +92 -0
- package/vendor-core/core-usage-locality.js +70 -0
- package/vendor-core/detectors.js +1989 -0
- package/vendor-core/docs-config.js +72 -0
- package/vendor-core/docs-site-config.js +23 -0
- package/vendor-core/efficiency-model.js +62 -0
- package/vendor-core/etl-phases.js +28 -0
- package/vendor-core/event-handlers.js +906 -0
- package/vendor-core/event-schemas.js +405 -0
- package/vendor-core/evidence-availability.js +121 -0
- package/vendor-core/evidence-report.js +459 -0
- package/vendor-core/finding-action-label.js +97 -0
- package/vendor-core/finding-filter-predicate.js +38 -0
- package/vendor-core/format-utils.js +167 -0
- package/vendor-core/impact-band.js +50 -0
- package/vendor-core/impact-estimator.js +428 -0
- package/vendor-core/ingest.js +139 -0
- package/vendor-core/job-groups.js +30 -0
- package/vendor-core/load-vendored.js +24 -0
- package/vendor-core/lz4-block.js +135 -0
- package/vendor-core/mcp-error.js +3 -0
- package/vendor-core/mcp-server-factory.js +115 -0
- package/vendor-core/mcp-tools.js +331 -0
- package/vendor-core/model-assembler.js +76 -0
- package/vendor-core/occupancy.js +202 -0
- package/vendor-core/parser-worker.js +249 -0
- package/vendor-core/plan-dot.js +25 -0
- package/vendor-core/plan-duration-attribution.js +185 -0
- package/vendor-core/plan-graph-model.js +171 -0
- package/vendor-core/plan-node-detail.js +159 -0
- package/vendor-core/plan-summary.js +233 -0
- package/vendor-core/plan-tree-walk.js +29 -0
- package/vendor-core/proxy.js +157 -0
- package/vendor-core/recommendation-rollup.js +197 -0
- package/vendor-core/redact.js +175 -0
- package/vendor-core/rolling-log-reassembly.js +52 -0
- package/vendor-core/run-aggregates.js +44 -0
- package/vendor-core/run-comparison.js +458 -0
- package/vendor-core/scaling-sim.js +73 -0
- package/vendor-core/session-snapshot.js +79 -0
- package/vendor-core/shs-fetch.js +196 -0
- package/vendor-core/shs-load.js +121 -0
- package/vendor-core/shs-request.js +101 -0
- package/vendor-core/shs-schemas.js +13 -0
- package/vendor-core/snappy-block.js +140 -0
- package/vendor-core/stage-quantiles.js +199 -0
- package/vendor-core/threshold-summary.js +35 -0
- package/vendor-core/types.js +286 -0
- package/vendor-core/vendor/fflate.js +2695 -0
- package/vendor-core/vendor/fzstd.js +768 -0
- package/vendor-core/wall-clock.js +36 -0
- package/vendor-core/wasted-core-hours.js +68 -0
|
@@ -0,0 +1,459 @@
|
|
|
1
|
+
// Portable, redacted evidence report (ELV-002). Pure builder over an appModel:
|
|
2
|
+
// runs the detectors, then serializes a run summary + every finding into a
|
|
3
|
+
// deterministic, byte-stable JSON document plus a human-readable Markdown
|
|
4
|
+
// rendering. Raw task records are never included (privacy baseline); identifier
|
|
5
|
+
// redaction is opt-in via { redact: true }.
|
|
6
|
+
import { analyze, auditConfig } from './analyzer.js';
|
|
7
|
+
import { detectorCatalog } from './detectors.js';
|
|
8
|
+
import { typeTag, formatBytes, formatDuration, IMPACT_BAND_ORDER } from './format-utils.js';
|
|
9
|
+
import { redactReport } from './redact.js';
|
|
10
|
+
import { coreFindingActionLabel } from './finding-action-label.js';
|
|
11
|
+
import { matchesFindingFilterCriteria } from './finding-filter-predicate.js';
|
|
12
|
+
import { buildRecommendationRollup, isEligible, rankFindings, } from './recommendation-rollup.js';
|
|
13
|
+
import { getThresholdSummary } from './threshold-summary.js';
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
export const EVIDENCE_SCHEMA_VERSION = 3;
|
|
19
|
+
|
|
20
|
+
// NOTE on deviations from the plan's literal interface text: `findingRow`
|
|
21
|
+
// below always sets `id`/`metric`/`value`/`recommendation` via `?? null`
|
|
22
|
+
// (never omits the key), and `buildJson` always sets `evidenceAvailability`
|
|
23
|
+
// and `summary.app.{id,name,sparkVersion}` via `?? null` too (AppModel's own
|
|
24
|
+
// `evidenceAvailability` field is `EvidenceAvailability | null`, `app` is
|
|
25
|
+
// `SparkAppInfo | null`, and `SparkAppInfo.sparkVersion` documents `null` as
|
|
26
|
+
// a deliberate "unknown version" sentinel distinct from absence). So these
|
|
27
|
+
// can genuinely be `null` at runtime even though the plan pins them as
|
|
28
|
+
// plain/optional non-null types. Widened here to match reality rather than
|
|
29
|
+
// masking it with a cast, and kept as `?? null`, not `?? undefined`: since
|
|
30
|
+
// JSON.stringify drops `undefined` keys but keeps `null`, using `undefined`
|
|
31
|
+
// here would silently strip these fields from the serialized report.
|
|
32
|
+
//
|
|
33
|
+
// `value` is further widened to `number | string | null` (Task 22,
|
|
34
|
+
// src/detectors.ts): `stageFailed` and the four `configAudit` entries put
|
|
35
|
+
// human-readable text in `Finding.value` instead of a magnitude, and this
|
|
36
|
+
// row shape carries that value through unchanged.
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
// The `Fix these first` rollup row (ELV-002 extension): one entry per
|
|
52
|
+
// `buildRecommendationRollup` group, carrying the same impact-ranked
|
|
53
|
+
// aggregation the dashboard's `FixTheseFirst.tsx` widget renders, so the
|
|
54
|
+
// CLI/MCP/download paths get the "what's the highest-leverage fix" ranking
|
|
55
|
+
// too, not just the flat impact-sorted `findings` list above.
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
// One line per detector `type` that fired zero findings this run, so a flat
|
|
72
|
+
// evidence report can state "these were checked and came back clean" the
|
|
73
|
+
// same way the dashboard's clean-checks table does.
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
// Fields surfaced as first-class report columns. Everything else on a finding
|
|
95
|
+
// becomes its `evidence` payload (sorted for stable key order).
|
|
96
|
+
const CORE_KEYS = new Set([
|
|
97
|
+
'id', 'type', 'impactBand', 'stageId', 'metric', 'value',
|
|
98
|
+
'recommendation', 'detectorVersion', 'confidence', 'validationRequired', 'docAnchor', 'impactEstimate',
|
|
99
|
+
'actionLabel',
|
|
100
|
+
]);
|
|
101
|
+
|
|
102
|
+
function findingRow(f ) {
|
|
103
|
+
const evidence = {};
|
|
104
|
+
for (const k of Object.keys(f).sort()) {
|
|
105
|
+
if (!CORE_KEYS.has(k)) evidence[k] = (f )[k];
|
|
106
|
+
}
|
|
107
|
+
const row = {
|
|
108
|
+
id: f.id ?? null,
|
|
109
|
+
type: f.type,
|
|
110
|
+
tag: typeTag(f.type),
|
|
111
|
+
impactBand: f.impactBand,
|
|
112
|
+
stageId: f.stageId ?? null,
|
|
113
|
+
metric: f.metric ?? null,
|
|
114
|
+
value: f.value ?? null,
|
|
115
|
+
recommendation: f.recommendation ?? null,
|
|
116
|
+
detectorVersion: f.detectorVersion ?? 1,
|
|
117
|
+
evidence,
|
|
118
|
+
// Deliberate simplification vs. the view layer's `REGISTRY[type]?.findingLabel`
|
|
119
|
+
// fallback (src/view/finding-action-label.ts): there's no widget registry at
|
|
120
|
+
// this layer, and falling back to the finding's own `type` is reasonable
|
|
121
|
+
// since coreFindingActionLabel's switch already covers every type the real
|
|
122
|
+
// detectors emit; only obscure/future unmatched sub-variants hit this fallback.
|
|
123
|
+
actionLabel: coreFindingActionLabel(f) ?? f.type,
|
|
124
|
+
};
|
|
125
|
+
// Threshold/confidence provenance, only when the detector emitted it.
|
|
126
|
+
if (f.confidence != null) row.confidence = f.confidence;
|
|
127
|
+
if (f.validationRequired != null) row.validationRequired = f.validationRequired;
|
|
128
|
+
if (f.docAnchor != null) row.docAnchor = f.docAnchor;
|
|
129
|
+
if (f.impactEstimate != null) row.impactEstimate = f.impactEstimate;
|
|
130
|
+
return row;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
// Deterministic finding order: impact band, then type, then stage, then id, so a
|
|
134
|
+
// fixed appModel always serializes byte-for-byte identically.
|
|
135
|
+
function sortFindings(rows ) {
|
|
136
|
+
return [...rows].sort((a, b) => {
|
|
137
|
+
const s = (IMPACT_BAND_ORDER[a.impactBand] ?? 9) - (IMPACT_BAND_ORDER[b.impactBand] ?? 9);
|
|
138
|
+
if (s !== 0) return s;
|
|
139
|
+
if (a.type !== b.type) return a.type < b.type ? -1 : 1;
|
|
140
|
+
const sa = a.stageId ?? -1;
|
|
141
|
+
const sb = b.stageId ?? -1;
|
|
142
|
+
if (sa !== sb) return sa - sb;
|
|
143
|
+
return (a.id ?? '') < (b.id ?? '') ? -1 : (a.id ?? '') > (b.id ?? '') ? 1 : 0;
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
// `isEligible`/`rankFindings` are shared with FixTheseFirst.tsx via
|
|
148
|
+
// src/recommendation-rollup.ts (both files used to duplicate this logic
|
|
149
|
+
// near-verbatim); this file's own `isEligible` call site omits
|
|
150
|
+
// FixTheseFirst.tsx's extra `REGISTRY[finding.type] != null` check: every
|
|
151
|
+
// real finding reaching this file already came out of `analyze()`/
|
|
152
|
+
// `auditConfig()`, so it's always a known type, and REGISTRY (a `.tsx` file)
|
|
153
|
+
// isn't importable from this core module anyway.
|
|
154
|
+
|
|
155
|
+
// The impact-ranked "what's the highest-leverage fix" rollup, ported from
|
|
156
|
+
// FixTheseFirst.tsx so the CLI/MCP/download paths get the same ranking the
|
|
157
|
+
// dashboard shows. `buildRecommendationRollup` already returns groups in the
|
|
158
|
+
// correct cross-group order (time groups by recoverable ms descending, then
|
|
159
|
+
// resource/count groups by worst impact band), so this only maps each group to
|
|
160
|
+
// its JSON row shape without re-sorting.
|
|
161
|
+
function buildRecommendations(
|
|
162
|
+
findings ,
|
|
163
|
+
stages ,
|
|
164
|
+
) {
|
|
165
|
+
const eligible = findings.filter(isEligible);
|
|
166
|
+
const groups = buildRecommendationRollup(eligible, stages);
|
|
167
|
+
return groups.map((group ) => {
|
|
168
|
+
const ranked = rankFindings(group.findings);
|
|
169
|
+
const representative = ranked[0];
|
|
170
|
+
const findingIds = ranked.map((f) => f.id).filter((id) => id != null);
|
|
171
|
+
const base = {
|
|
172
|
+
type: group.type,
|
|
173
|
+
tag: typeTag(group.type),
|
|
174
|
+
actionLabel: coreFindingActionLabel(representative) ?? representative.type,
|
|
175
|
+
findingCount: group.findingCount,
|
|
176
|
+
findingIds,
|
|
177
|
+
};
|
|
178
|
+
if (group.kind === 'time') {
|
|
179
|
+
return {
|
|
180
|
+
...base,
|
|
181
|
+
kind: 'time',
|
|
182
|
+
stageCount: group.stageCount,
|
|
183
|
+
recoverableMsHigh: group.recoverableMsHigh,
|
|
184
|
+
// A point estimate, not a range: matches FixTheseFirst.tsx's own
|
|
185
|
+
// `trailingStat` for time groups, which prints the same figure twice
|
|
186
|
+
// rather than the finding-level low-high spread this group already
|
|
187
|
+
// collapsed away via computeStageUnionMs.
|
|
188
|
+
impact: formatWallClockRange(group.recoverableMsHigh, group.recoverableMsHigh),
|
|
189
|
+
};
|
|
190
|
+
}
|
|
191
|
+
if (group.kind === 'resource') {
|
|
192
|
+
return {
|
|
193
|
+
...base,
|
|
194
|
+
kind: 'resource',
|
|
195
|
+
unit: group.unit,
|
|
196
|
+
total: group.total,
|
|
197
|
+
impact: formatRawWaste({ value: group.total, unit: group.unit }),
|
|
198
|
+
};
|
|
199
|
+
}
|
|
200
|
+
return {
|
|
201
|
+
...base,
|
|
202
|
+
kind: 'count',
|
|
203
|
+
byImpactBand: group.byImpactBand,
|
|
204
|
+
// No single quantifiable figure for a count group; the impact-band tally
|
|
205
|
+
// (byImpactBand above) is the payload instead.
|
|
206
|
+
impact: null,
|
|
207
|
+
};
|
|
208
|
+
});
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
// Detector types that fired zero findings this run, so a flat evidence
|
|
212
|
+
// report can state "these were checked and came back clean" the same way
|
|
213
|
+
// the dashboard's clean-checks table does. Deliberately differs from the
|
|
214
|
+
// dashboard's Alerts.tsx "Clean checks" table, which excludes
|
|
215
|
+
// memoryUtilization/utilization/coreLocality (the "always-mounted reference"
|
|
216
|
+
// widgets; see isAlwaysMountedType in src/view/detector-registry.tsx)
|
|
217
|
+
// because those are already always shown as their own widget elsewhere on
|
|
218
|
+
// the board. A flat evidence report has no such separate always-visible
|
|
219
|
+
// surface for them, so this list intentionally includes them when they have
|
|
220
|
+
// zero findings, rather than mirroring the UI's display-consolidation
|
|
221
|
+
// exclusion.
|
|
222
|
+
function buildCleanChecks(findings ) {
|
|
223
|
+
const firedTypes = new Set(findings.map((f) => f.type));
|
|
224
|
+
const seen = new Set ();
|
|
225
|
+
const entries = [];
|
|
226
|
+
// detectorCatalog() can list the same `type` more than once (configAudit
|
|
227
|
+
// has 4 separate DETECTORS entries, all `type: 'configAudit'`); dedupe by
|
|
228
|
+
// type, keeping first occurrence, so a type with several sibling detector
|
|
229
|
+
// entries still contributes exactly one clean-check line.
|
|
230
|
+
for (const d of detectorCatalog() ) {
|
|
231
|
+
if (firedTypes.has(d.type) || seen.has(d.type)) continue;
|
|
232
|
+
seen.add(d.type);
|
|
233
|
+
entries.push({ type: d.type, tag: typeTag(d.type), thresholdSummary: getThresholdSummary(d.type) });
|
|
234
|
+
}
|
|
235
|
+
return entries;
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
// Keyed by appModel object identity, not runId: mcp-tools.ts caches one fixed
|
|
239
|
+
// appModel per runId for the run's whole cached lifetime (resolveOrCreateRun
|
|
240
|
+
// never mutates a cached entry's appModel), so re-running analyze()/
|
|
241
|
+
// auditConfig() for the same appModel always reproduces the same catalog.
|
|
242
|
+
// getFindingEvidence in particular calls buildEvidenceReport once per
|
|
243
|
+
// drill-down on an already-diagnosed run; without this, N findings looked up
|
|
244
|
+
// on one run meant N full detector re-runs. A WeakMap needs no explicit
|
|
245
|
+
// invalidation: once mcp-tools.ts evicts the appModel, this entry is
|
|
246
|
+
// unreachable and collectible too.
|
|
247
|
+
const jsonCache = new WeakMap ();
|
|
248
|
+
|
|
249
|
+
function buildJson(appModel ) {
|
|
250
|
+
const cached = jsonCache.get(appModel);
|
|
251
|
+
if (cached) return cached;
|
|
252
|
+
const { app, stages, executors, sql, jobs, runAggregates, evidenceAvailability } = appModel;
|
|
253
|
+
const catalog = analyze(
|
|
254
|
+
app, stages, executors?.added ?? [], executors?.removed ?? [],
|
|
255
|
+
jobs ?? new Map(), sql ?? new Map(),
|
|
256
|
+
runAggregates ?? null,
|
|
257
|
+
);
|
|
258
|
+
const config = auditConfig(app);
|
|
259
|
+
const allFindings = [...catalog, ...config];
|
|
260
|
+
const rows = sortFindings(allFindings.map(findingRow));
|
|
261
|
+
const recommendations = buildRecommendations(allFindings, stages ?? new Map());
|
|
262
|
+
const cleanChecks = buildCleanChecks(allFindings);
|
|
263
|
+
|
|
264
|
+
const impactBandCounts = { critical: 0, warning: 0, info: 0 };
|
|
265
|
+
for (const r of rows) if (r.impactBand in impactBandCounts) impactBandCounts[r.impactBand] += 1;
|
|
266
|
+
|
|
267
|
+
const result = {
|
|
268
|
+
schemaVersion: EVIDENCE_SCHEMA_VERSION,
|
|
269
|
+
summary: {
|
|
270
|
+
app: {
|
|
271
|
+
// `?? null`, not `?? undefined`: pre-migration behavior (and
|
|
272
|
+
// SparkAppInfo.sparkVersion's own doc comment) treats `null` as a
|
|
273
|
+
// deliberate "unknown/absent" sentinel distinct from an omitted key.
|
|
274
|
+
// JSON.stringify drops `undefined` keys but keeps `null`, so using
|
|
275
|
+
// `?? undefined` here would have silently dropped these fields from
|
|
276
|
+
// the serialized report whenever `app` is null or its fields unset.
|
|
277
|
+
id: app?.id ?? null,
|
|
278
|
+
name: app?.name ?? null,
|
|
279
|
+
sparkVersion: app?.sparkVersion ?? null,
|
|
280
|
+
},
|
|
281
|
+
stageCount: stages?.size ?? 0,
|
|
282
|
+
jobCount: jobs?.size ?? 0,
|
|
283
|
+
sqlExecutionCount: sql?.size ?? 0,
|
|
284
|
+
findingCount: rows.length,
|
|
285
|
+
impactBandCounts,
|
|
286
|
+
},
|
|
287
|
+
evidenceAvailability: evidenceAvailability ?? null,
|
|
288
|
+
// Detector metadata (type/version/scope/thresholds/docAnchor) so the
|
|
289
|
+
// threshold set that produced each finding travels with the evidence.
|
|
290
|
+
// Order follows DETECTORS, which is stable => byte-stable serialization.
|
|
291
|
+
detectors: detectorCatalog(),
|
|
292
|
+
findings: rows,
|
|
293
|
+
recommendations,
|
|
294
|
+
cleanChecks,
|
|
295
|
+
};
|
|
296
|
+
jsonCache.set(appModel, result);
|
|
297
|
+
return result;
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
// Human-readable rendering of an evidence value. Byte-magnitude keys are
|
|
301
|
+
// humanized (avgFileSizeBytes: 3.1 MB); objects/arrays serialize compactly so
|
|
302
|
+
// no payload is silently dropped from the Markdown.
|
|
303
|
+
function renderEvidenceValue(key , value ) {
|
|
304
|
+
if (typeof value === 'number' && /bytes$/i.test(key)) return formatBytes(value);
|
|
305
|
+
if (value !== null && typeof value === 'object') return JSON.stringify(value);
|
|
306
|
+
return String(value);
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
function formatWallClockRange(low , high ) {
|
|
310
|
+
const fmtMs = (ms ) => (ms === 0 ? '0s' : formatDuration(ms));
|
|
311
|
+
return low === high ? `Est. ${fmtMs(high)}` : `Est. ${fmtMs(low)}-${fmtMs(high)}`;
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
function formatRawWaste(rawWaste ) {
|
|
315
|
+
const rounded = Math.round(rawWaste.value * 10) / 10;
|
|
316
|
+
switch (rawWaste.unit) {
|
|
317
|
+
case 'bytes': return formatBytes(rawWaste.value);
|
|
318
|
+
case 'ms': return formatDuration(rawWaste.value);
|
|
319
|
+
case 'mbSeconds': return `${rounded} MB-s`;
|
|
320
|
+
case 'coreHours': return `${rounded.toFixed(1)} core-h`;
|
|
321
|
+
case 'coreMs': return `${rounded} core-ms`;
|
|
322
|
+
default: return String(rawWaste.value);
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
// `basis: 'informational'` findings carry no wallClock/rawWaste at all, so
|
|
327
|
+
// there's nothing quantifiable to print; the caller skips the line entirely.
|
|
328
|
+
function renderImpactEstimate(estimate ) {
|
|
329
|
+
const rangeText = estimate.wallClock ? formatWallClockRange(estimate.wallClock.low, estimate.wallClock.high) : null;
|
|
330
|
+
const wasteText = estimate.rawWaste ? formatRawWaste(estimate.rawWaste) : null;
|
|
331
|
+
if (!rangeText && !wasteText) return null;
|
|
332
|
+
const parts = [rangeText, wasteText].filter((p) => p != null).join(' · ');
|
|
333
|
+
return `${parts} (estimateMethod: ${estimate.estimateMethod})`;
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
function renderMarkdown(json ) {
|
|
337
|
+
const { summary, findings, evidenceAvailability, detectors, recommendations, cleanChecks } = json;
|
|
338
|
+
const lines = [];
|
|
339
|
+
lines.push('# Spark run evidence report');
|
|
340
|
+
lines.push('');
|
|
341
|
+
lines.push(`- Application: ${summary.app.name ?? '(unknown)'} (${summary.app.id ?? 'n/a'})`);
|
|
342
|
+
lines.push(`- Spark version: ${summary.app.sparkVersion ?? 'n/a'}`);
|
|
343
|
+
lines.push(`- Stages: ${summary.stageCount} · Jobs: ${summary.jobCount} · SQL executions: ${summary.sqlExecutionCount}`);
|
|
344
|
+
lines.push(`- Findings: ${summary.findingCount} (critical ${summary.impactBandCounts.critical}, warning ${summary.impactBandCounts.warning}, info ${summary.impactBandCounts.info})`);
|
|
345
|
+
lines.push('');
|
|
346
|
+
if (recommendations.length > 0) {
|
|
347
|
+
lines.push(`## Fix these first (${recommendations.length})`);
|
|
348
|
+
lines.push('');
|
|
349
|
+
recommendations.forEach((r, i) => {
|
|
350
|
+
lines.push(`${i + 1}. [${r.tag}] ${r.actionLabel}`);
|
|
351
|
+
const detail = r.kind === 'count'
|
|
352
|
+
? Object.entries(r.byImpactBand ?? {}).map(([impactBand, count]) => `${count} ${impactBand}`).join(', ')
|
|
353
|
+
: r.impact;
|
|
354
|
+
lines.push(` - ${detail} · ×${r.findingCount} finding(s)`);
|
|
355
|
+
});
|
|
356
|
+
lines.push('');
|
|
357
|
+
}
|
|
358
|
+
lines.push(`## Findings (${findings.length})`);
|
|
359
|
+
lines.push('');
|
|
360
|
+
for (const r of findings) {
|
|
361
|
+
const where = r.stageId != null ? ` (stage ${r.stageId})` : '';
|
|
362
|
+
lines.push(`### [${r.tag}] ${r.type} · ${r.impactBand}${where}`);
|
|
363
|
+
lines.push(`- action: ${r.actionLabel}`);
|
|
364
|
+
if (r.metric != null) lines.push(`- ${r.metric}: ${r.value}`);
|
|
365
|
+
if (r.recommendation) lines.push(`- ${r.recommendation}`);
|
|
366
|
+
if (r.confidence) lines.push(`- confidence: ${r.confidence}`);
|
|
367
|
+
if (r.validationRequired) lines.push(`- validation: ${r.validationRequired}`);
|
|
368
|
+
const impactText = r.impactEstimate ? renderImpactEstimate(r.impactEstimate) : null;
|
|
369
|
+
if (impactText) lines.push(`- impact: ${impactText}`);
|
|
370
|
+
lines.push(`- detector version: ${r.detectorVersion}`);
|
|
371
|
+
// Evidence payload (sorted for stable order) so two rows differing only by
|
|
372
|
+
// evidence (two smallFiles by direction, two partitionSizing by rule)
|
|
373
|
+
// render distinctly and carry the full AC3 field set into the Markdown.
|
|
374
|
+
const evidence = Object.entries(r.evidence ?? {}).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
|
|
375
|
+
if (evidence.length) {
|
|
376
|
+
lines.push('- evidence:');
|
|
377
|
+
for (const [k, v] of evidence) lines.push(` - ${k}: ${renderEvidenceValue(k, v)}`);
|
|
378
|
+
}
|
|
379
|
+
lines.push('');
|
|
380
|
+
}
|
|
381
|
+
if (evidenceAvailability?.entries?.length) {
|
|
382
|
+
lines.push('## Evidence availability');
|
|
383
|
+
lines.push('');
|
|
384
|
+
for (const e of evidenceAvailability.entries) {
|
|
385
|
+
lines.push(`- ${e.key}: ${e.state} (${e.summary})`);
|
|
386
|
+
}
|
|
387
|
+
lines.push('');
|
|
388
|
+
}
|
|
389
|
+
// Detector catalog: the version + threshold set that produced each finding,
|
|
390
|
+
// so the Markdown (the format pasted into a ticket) carries provenance too.
|
|
391
|
+
if (Array.isArray(detectors) && detectors.length) {
|
|
392
|
+
lines.push('## Detectors');
|
|
393
|
+
lines.push('');
|
|
394
|
+
for (const d of detectors) {
|
|
395
|
+
lines.push(`- ${d.type} (v${d.version}, ${d.scope}), thresholds: ${JSON.stringify(d.thresholds)}`);
|
|
396
|
+
}
|
|
397
|
+
lines.push('');
|
|
398
|
+
}
|
|
399
|
+
if (cleanChecks.length > 0) {
|
|
400
|
+
lines.push(`## Clean checks (${cleanChecks.length})`);
|
|
401
|
+
lines.push('');
|
|
402
|
+
for (const c of cleanChecks) {
|
|
403
|
+
lines.push(`- [${c.tag}] ${c.type}: ${c.thresholdSummary}`);
|
|
404
|
+
}
|
|
405
|
+
lines.push('');
|
|
406
|
+
}
|
|
407
|
+
return lines.join('\n');
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
// CLI/MCP-facing filter over FindingRow (post-row-transformation), delegating
|
|
417
|
+
// to the shared core predicate (src/finding-filter-predicate.ts) that also
|
|
418
|
+
// backs the dashboard's src/view/finding-filter.ts.
|
|
419
|
+
function matchesFindingsFilter(row , filter ) {
|
|
420
|
+
return matchesFindingFilterCriteria(row, filter);
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
/**
|
|
424
|
+
* Build a `FindingsFilter` from the three optional CLI/MCP filter dimensions,
|
|
425
|
+
* or `undefined` when none were passed (the "is anything actually set" gate
|
|
426
|
+
* both `bin/sparkforensics-analyze.mjs` and `diagnoseRun` need before calling
|
|
427
|
+
* `buildEvidenceReport`).
|
|
428
|
+
*/
|
|
429
|
+
export function toFindingsFilter(
|
|
430
|
+
impactBand , type , stageId ,
|
|
431
|
+
) {
|
|
432
|
+
return (impactBand || type || stageId !== undefined) ? { impactBand, type, stageId } : undefined;
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
/**
|
|
436
|
+
* Build a portable evidence report from an appModel.
|
|
437
|
+
* @param appModel
|
|
438
|
+
* @param opts redact=true pseudonymizes app ids / hosts; markdown=false skips
|
|
439
|
+
* rendering the Markdown string (returned as '' instead) for callers that
|
|
440
|
+
* only need `json`; findingsFilter narrows `json.findings` (and the
|
|
441
|
+
* rendered Markdown's Findings section) only — `summary`, `recommendations`,
|
|
442
|
+
* and `cleanChecks` stay computed from the full, unfiltered set, so a
|
|
443
|
+
* narrow filter never hides that other checks passed or other fixes exist.
|
|
444
|
+
*/
|
|
445
|
+
export function buildEvidenceReport(
|
|
446
|
+
appModel ,
|
|
447
|
+
{ redact = false, markdown: computeMarkdown = true, findingsFilter }
|
|
448
|
+
|
|
449
|
+
= {},
|
|
450
|
+
) {
|
|
451
|
+
let json = buildJson(appModel);
|
|
452
|
+
if (redact) json = redactReport(json);
|
|
453
|
+
// Filter after redact, not before: redaction only replaces string values
|
|
454
|
+
// on rows that survive (app id / host tokens), it never adds/removes rows,
|
|
455
|
+
// so the two orderings produce identical final content either way.
|
|
456
|
+
if (findingsFilter) json = { ...json, findings: json.findings.filter((row) => matchesFindingsFilter(row, findingsFilter)) };
|
|
457
|
+
const markdown = computeMarkdown ? renderMarkdown(json) : '';
|
|
458
|
+
return { markdown, json };
|
|
459
|
+
}
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
|
|
2
|
+
|
|
3
|
+
/** A short, imperative action label for a finding's row (e.g. "Reduce
|
|
4
|
+
* shuffle size"). Keyed off `finding.type` plus whichever discriminant field
|
|
5
|
+
* that detector uses for its sub-variants (`rule`, `direction`, `variant`, or
|
|
6
|
+
* `property`: there's no single field name shared across detectors; see
|
|
7
|
+
* detectors.ts). Core-safe: no `src/view/**` import, so both the dashboard
|
|
8
|
+
* (`src/view/finding-action-label.ts`, which layers the REGISTRY fallback on
|
|
9
|
+
* top of this) and `src/evidence-report.ts` (CLI/MCP path, which cannot
|
|
10
|
+
* import React-only code) share the exact same switch logic instead of
|
|
11
|
+
* forking it. Returns `undefined` for any (type, variant) combination this
|
|
12
|
+
* switch doesn't cover: the caller decides what to fall back to. */
|
|
13
|
+
export function coreFindingActionLabel(finding ) {
|
|
14
|
+
switch (finding.type) {
|
|
15
|
+
case 'skew':
|
|
16
|
+
return 'Fix task skew';
|
|
17
|
+
case 'stageShape':
|
|
18
|
+
switch (finding.rule) {
|
|
19
|
+
case 'lowParallelism': return 'Increase parallelism';
|
|
20
|
+
case 'dataExplosion': return 'Check for exploding join';
|
|
21
|
+
case 'taskStageSkew': return 'Fix straggler task';
|
|
22
|
+
}
|
|
23
|
+
break;
|
|
24
|
+
case 'shuffle':
|
|
25
|
+
return 'Reduce shuffle size';
|
|
26
|
+
case 'partitionSizing':
|
|
27
|
+
switch (finding.rule) {
|
|
28
|
+
case 'shufflePartitionSkew': return 'Fix skewed partition';
|
|
29
|
+
case 'lowShuffleParallelism': return 'Add shuffle partitions';
|
|
30
|
+
case 'maxPartitionTooBig': return 'Repartition oversized data';
|
|
31
|
+
}
|
|
32
|
+
break;
|
|
33
|
+
case 'spill':
|
|
34
|
+
return 'Reduce spill';
|
|
35
|
+
case 'gc':
|
|
36
|
+
return finding.direction === 'low' ? 'Right-size executor memory' : 'Reduce GC pressure';
|
|
37
|
+
case 'slowHost':
|
|
38
|
+
if (finding.variant === 'durationShare') return 'Fix data locality';
|
|
39
|
+
if (finding.variant === 'multiDim') return 'Investigate degraded executor';
|
|
40
|
+
return 'Check slow host';
|
|
41
|
+
case 'stageSlowness':
|
|
42
|
+
return 'Profile slow stage';
|
|
43
|
+
case 'stageFailed':
|
|
44
|
+
return 'Inspect stage failure';
|
|
45
|
+
case 'failures':
|
|
46
|
+
return 'Investigate task failures';
|
|
47
|
+
case 'straggler':
|
|
48
|
+
return 'Fix stragglers';
|
|
49
|
+
case 'speculationWaste':
|
|
50
|
+
return 'Tune speculation settings';
|
|
51
|
+
case 'retryWaste':
|
|
52
|
+
return 'Investigate retry cause';
|
|
53
|
+
case 'tinyTask':
|
|
54
|
+
return 'Coalesce small tasks';
|
|
55
|
+
case 'coldStart':
|
|
56
|
+
return 'Pre-warm cluster';
|
|
57
|
+
case 'utilization':
|
|
58
|
+
return 'Reduce cluster size';
|
|
59
|
+
case 'memoryUtilization':
|
|
60
|
+
switch (finding.variant) {
|
|
61
|
+
case 'idleCores': return 'Reduce idle cores';
|
|
62
|
+
case 'wasteModel': return 'Right-size executor memory';
|
|
63
|
+
case 'memoryBand':
|
|
64
|
+
if (finding.dataUnavailable) return 'Enable memory metrics';
|
|
65
|
+
return finding.rule === 'heapNearCapacity' ? 'Increase executor memory' : 'Reduce executor memory';
|
|
66
|
+
}
|
|
67
|
+
break;
|
|
68
|
+
case 'cacheUtilization':
|
|
69
|
+
return 'Increase cache memory';
|
|
70
|
+
case 'coreLocality':
|
|
71
|
+
return 'Fix data locality';
|
|
72
|
+
case 'autoscalingChurn':
|
|
73
|
+
return 'Reduce autoscaling churn';
|
|
74
|
+
case 'cachingOpportunity':
|
|
75
|
+
return finding.variant === 'composite' ? 'Cache repeated result' : 'Cache shared table';
|
|
76
|
+
case 'jobFailureRate':
|
|
77
|
+
return 'Investigate failed jobs';
|
|
78
|
+
case 'configAudit':
|
|
79
|
+
switch (finding.property) {
|
|
80
|
+
case 'spark.shuffle.service.enabled': return 'Enable shuffle service';
|
|
81
|
+
case 'spark.dynamicAllocation.minExecutors': return 'Fix autoscaling bounds';
|
|
82
|
+
case 'spark.dynamicAllocation.maxExecutors': return 'Set max executors';
|
|
83
|
+
case 'spark.serializer': return 'Switch to Kryo';
|
|
84
|
+
case 'spark.executor.memoryOverhead': return 'Raise memory overhead';
|
|
85
|
+
}
|
|
86
|
+
break;
|
|
87
|
+
case 'duplicatePlanSubtree':
|
|
88
|
+
return 'Dedupe repeated subtree';
|
|
89
|
+
case 'smallFiles':
|
|
90
|
+
return finding.direction === 'write' ? 'Coalesce output files' : 'Compact small files';
|
|
91
|
+
case 'underBroadcast':
|
|
92
|
+
return 'Use broadcast join';
|
|
93
|
+
case 'overBroadcast':
|
|
94
|
+
return 'Fix oversized broadcast';
|
|
95
|
+
}
|
|
96
|
+
return undefined;
|
|
97
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
// Core-owned finding-filter predicate, shared by the CLI/MCP evidence report
|
|
2
|
+
// (src/evidence-report.ts's FindingsFilter) and the dashboard's finding
|
|
3
|
+
// filter bar (src/view/finding-filter.ts's FilterSelection): same precedent
|
|
4
|
+
// as src/finding-action-label.ts, whose view-layer counterpart wraps it
|
|
5
|
+
// instead of reimplementing its switch. Each dimension is unconstrained when
|
|
6
|
+
// empty/absent; a `stageId` criterion (single value or a set, for the
|
|
7
|
+
// dashboard's multi-select) only matches a row whose own stageId is present.
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
function isNonEmpty (m ) {
|
|
11
|
+
if (m == null) return false;
|
|
12
|
+
return (Array.isArray(m) ? m.length : (m ).size) > 0;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function has (m , value ) {
|
|
16
|
+
return Array.isArray(m) ? m.includes(value) : (m ).has(value);
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
export function matchesFindingFilterCriteria(
|
|
26
|
+
row ,
|
|
27
|
+
criteria ,
|
|
28
|
+
) {
|
|
29
|
+
if (isNonEmpty(criteria.impactBand) && !has(criteria.impactBand, row.impactBand)) return false;
|
|
30
|
+
if (isNonEmpty(criteria.type) && !has(criteria.type, row.type)) return false;
|
|
31
|
+
const { stageId } = criteria;
|
|
32
|
+
if (typeof stageId === 'number') {
|
|
33
|
+
if (row.stageId !== stageId) return false;
|
|
34
|
+
} else if (isNonEmpty(stageId)) {
|
|
35
|
+
if (row.stageId == null || !has(stageId, row.stageId)) return false;
|
|
36
|
+
}
|
|
37
|
+
return true;
|
|
38
|
+
}
|