sparkforensics-cli 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/sparkforensics-analyze.mjs +288 -0
- package/package.json +29 -0
- package/vendor-core/analyzer.js +167 -0
- package/vendor-core/assert-never.js +3 -0
- package/vendor-core/cli/budgets.js +203 -0
- package/vendor-core/cli/collect-run.js +107 -0
- package/vendor-core/core-count.js +68 -0
- package/vendor-core/core-locality-ratio.js +54 -0
- package/vendor-core/core-time-series.js +92 -0
- package/vendor-core/core-usage-locality.js +70 -0
- package/vendor-core/detectors.js +1989 -0
- package/vendor-core/docs-config.js +72 -0
- package/vendor-core/docs-site-config.js +23 -0
- package/vendor-core/efficiency-model.js +62 -0
- package/vendor-core/etl-phases.js +28 -0
- package/vendor-core/event-handlers.js +906 -0
- package/vendor-core/event-schemas.js +405 -0
- package/vendor-core/evidence-availability.js +121 -0
- package/vendor-core/evidence-report.js +459 -0
- package/vendor-core/finding-action-label.js +97 -0
- package/vendor-core/finding-filter-predicate.js +38 -0
- package/vendor-core/format-utils.js +167 -0
- package/vendor-core/impact-band.js +50 -0
- package/vendor-core/impact-estimator.js +428 -0
- package/vendor-core/ingest.js +139 -0
- package/vendor-core/job-groups.js +30 -0
- package/vendor-core/load-vendored.js +24 -0
- package/vendor-core/lz4-block.js +135 -0
- package/vendor-core/mcp-error.js +3 -0
- package/vendor-core/mcp-server-factory.js +115 -0
- package/vendor-core/mcp-tools.js +331 -0
- package/vendor-core/model-assembler.js +76 -0
- package/vendor-core/occupancy.js +202 -0
- package/vendor-core/parser-worker.js +249 -0
- package/vendor-core/plan-dot.js +25 -0
- package/vendor-core/plan-duration-attribution.js +185 -0
- package/vendor-core/plan-graph-model.js +171 -0
- package/vendor-core/plan-node-detail.js +159 -0
- package/vendor-core/plan-summary.js +233 -0
- package/vendor-core/plan-tree-walk.js +29 -0
- package/vendor-core/proxy.js +157 -0
- package/vendor-core/recommendation-rollup.js +197 -0
- package/vendor-core/redact.js +175 -0
- package/vendor-core/rolling-log-reassembly.js +52 -0
- package/vendor-core/run-aggregates.js +44 -0
- package/vendor-core/run-comparison.js +458 -0
- package/vendor-core/scaling-sim.js +73 -0
- package/vendor-core/session-snapshot.js +79 -0
- package/vendor-core/shs-fetch.js +196 -0
- package/vendor-core/shs-load.js +121 -0
- package/vendor-core/shs-request.js +101 -0
- package/vendor-core/shs-schemas.js +13 -0
- package/vendor-core/snappy-block.js +140 -0
- package/vendor-core/stage-quantiles.js +199 -0
- package/vendor-core/threshold-summary.js +35 -0
- package/vendor-core/types.js +286 -0
- package/vendor-core/vendor/fflate.js +2695 -0
- package/vendor-core/vendor/fzstd.js +768 -0
- package/vendor-core/wall-clock.js +36 -0
- package/vendor-core/wasted-core-hours.js +68 -0
|
@@ -0,0 +1,331 @@
|
|
|
1
|
+
import { resolve as resolvePath } from 'node:path';
|
|
2
|
+
import { existsSync, statSync } from 'node:fs';
|
|
3
|
+
import { randomUUID } from 'node:crypto';
|
|
4
|
+
import { collectRun } from './cli/collect-run.js';
|
|
5
|
+
import { deriveEvidenceAvailability } from './evidence-availability.js';
|
|
6
|
+
import { resolveFromShs, DEFAULT_MAX_ARCHIVE_BYTES, DEFAULT_IDLE_TIMEOUT_MS } from './shs-load.js';
|
|
7
|
+
import { mcpError } from './mcp-error.js';
|
|
8
|
+
import { buildEvidenceReport, toFindingsFilter, } from './evidence-report.js';
|
|
9
|
+
import { redactAppIdentity, redactComparison } from './redact.js';
|
|
10
|
+
import { computeWallClock } from './wall-clock.js';
|
|
11
|
+
import { analyze } from './analyzer.js';
|
|
12
|
+
import { buildComparison, renderComparisonMarkdown, } from './run-comparison.js';
|
|
13
|
+
import { evaluateBudgets, } from './cli/budgets.js';
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
// NOTE on a deviation from the plan's literal text: the plan defines RunRef as
|
|
18
|
+
// `{ runId?: string } & Partial<RunSource>` (path/shsBaseUrl/appId/attemptId
|
|
19
|
+
// flattened directly onto the object). Every real call site disagrees with
|
|
20
|
+
// that shape: resolveOrCreateRun's own destructuring (`{ source, runId } = {}`
|
|
21
|
+
// below), both mcp-server-factory.ts call sites (`compareRuns({ runId: runIdA,
|
|
22
|
+
// source: sourceA }, ...)`), and every resolveOrCreateRun/compareRuns call in
|
|
23
|
+
// tests/mcp-tools.test.js all pass a *nested* `source` key, never a flattened
|
|
24
|
+
// path/shsBaseUrl directly on the ref object. Widened to match that real,
|
|
25
|
+
// long-standing shape instead of reshaping every working call site (or every
|
|
26
|
+
// test) to fit the literal text.
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
// The plan's Produces list types compareRuns's return as `CompareRunsResult`
|
|
46
|
+
// (run-comparison.ts's raw comparison shape: baselineLabel/candidateLabel/
|
|
47
|
+
// stageSkew/baseStages/candStages/...). compareRuns below actually returns a
|
|
48
|
+
// distinct, smaller MCP-facing projection of that (runIdA/runIdB/
|
|
49
|
+
// findingsDelta/metricDeltas/confidence/reason/matchedCoverage); it never
|
|
50
|
+
// had baselineLabel, stageSkew, baseStages, or candStages. Introduced this
|
|
51
|
+
// interface to match the real returned shape rather than casting a literal
|
|
52
|
+
// that's actually missing several required CompareRunsResult properties.
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
function envInt(name , fallback ) {
|
|
64
|
+
const v = Number(process.env[name]);
|
|
65
|
+
return Number.isFinite(v) && v > 0 ? v : fallback;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// `touch()` below does a full Map delete+re-insert per cache hit to maintain
|
|
69
|
+
// LRU order: O(n) per touch, fine at this small cap but worth re-checking
|
|
70
|
+
// the cost if this cap is ever raised significantly.
|
|
71
|
+
const CACHE_CAP = envInt('SPARKFORENSICS_MCP_CACHE_CAP', 8);
|
|
72
|
+
const CACHE_TTL_MS = envInt('SPARKFORENSICS_MCP_CACHE_TTL_MS', 15 * 60 * 1000);
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
const byRunId = new Map (); // runId -> { appModel, cacheKey, lastAccess }
|
|
77
|
+
const byCacheKey = new Map (); // cacheKey -> runId
|
|
78
|
+
const pendingByCacheKey = new Map (); // cacheKey -> in-flight Promise<{runId, appModel}>
|
|
79
|
+
|
|
80
|
+
export function pathCacheKey(path ) {
|
|
81
|
+
const resolved = resolvePath(path);
|
|
82
|
+
if (!existsSync(resolved)) throw mcpError('invalid-event-log', `No such file: ${resolved}`);
|
|
83
|
+
// Size narrows a same-millisecond mtime collision; ctime narrows the case
|
|
84
|
+
// where a copy tool (rsync --preserve-times, tar) restores an identical
|
|
85
|
+
// mtime and size for different content: ctime can't be set by the copying
|
|
86
|
+
// tool, so it still reflects the real time the file landed on disk.
|
|
87
|
+
const { mtimeMs, ctimeMs, size } = statSync(resolved);
|
|
88
|
+
return `path:${resolved}:${mtimeMs}:${ctimeMs}:${size}`;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function shsCacheKey({ shsBaseUrl, appId, attemptId } ) {
|
|
92
|
+
return `shs:${shsBaseUrl}:${appId}:${attemptId ?? ''}`;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function touch(runId ) {
|
|
96
|
+
const entry = byRunId.get(runId);
|
|
97
|
+
// Defensive only for type-narrowing: every call site below touches a runId
|
|
98
|
+
// it (or getCachedAppModel) has already confirmed present.
|
|
99
|
+
if (!entry) return;
|
|
100
|
+
byRunId.delete(runId);
|
|
101
|
+
entry.lastAccess = Date.now();
|
|
102
|
+
byRunId.set(runId, entry);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function deleteRunEntry(id , entry ) {
|
|
106
|
+
byRunId.delete(id);
|
|
107
|
+
if (byCacheKey.get(entry.cacheKey) === id) byCacheKey.delete(entry.cacheKey);
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
function evictStale() {
|
|
111
|
+
const now = Date.now();
|
|
112
|
+
for (const [id, entry] of byRunId) {
|
|
113
|
+
if (now - entry.lastAccess > CACHE_TTL_MS) {
|
|
114
|
+
deleteRunEntry(id, entry);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
function evictOverflow() {
|
|
120
|
+
while (byRunId.size > CACHE_CAP) {
|
|
121
|
+
const oldestId = byRunId.keys().next().value;
|
|
122
|
+
// Defensive only for type-narrowing: the while condition guarantees
|
|
123
|
+
// byRunId is non-empty here, so .keys().next() always has a value.
|
|
124
|
+
if (oldestId === undefined) break;
|
|
125
|
+
const entry = byRunId.get(oldestId);
|
|
126
|
+
if (entry) deleteRunEntry(oldestId, entry);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
export function getCachedAppModel(runId ) {
|
|
131
|
+
evictStale();
|
|
132
|
+
const entry = byRunId.get(runId);
|
|
133
|
+
if (!entry) throw mcpError('run-not-found', `No cached run for runId ${runId}.`);
|
|
134
|
+
touch(runId);
|
|
135
|
+
return entry.appModel;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
async function resolveFromPath(path ) {
|
|
139
|
+
const { appModel, skippedLines } = await collectRun(path);
|
|
140
|
+
appModel.evidenceAvailability = deriveEvidenceAvailability(appModel, { skippedLines });
|
|
141
|
+
return appModel;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
export async function resolveOrCreateRun(
|
|
145
|
+
{ source, runId } = {},
|
|
146
|
+
{ fetchImpl = fetch, maxArchiveBytes = DEFAULT_MAX_ARCHIVE_BYTES, idleTimeoutMs = DEFAULT_IDLE_TIMEOUT_MS }
|
|
147
|
+
= {},
|
|
148
|
+
) {
|
|
149
|
+
evictStale();
|
|
150
|
+
if (runId) {
|
|
151
|
+
return { runId, appModel: getCachedAppModel(runId) };
|
|
152
|
+
}
|
|
153
|
+
if (!source) throw mcpError('access-or-upstream-failure', 'Provide either source or runId.');
|
|
154
|
+
|
|
155
|
+
// `'path' in source`, not `source.path`: RunSource's two variants share no
|
|
156
|
+
// common property, so a plain `.path` access doesn't type-check on the
|
|
157
|
+
// union. Equivalent at runtime to the original truthy check for every real
|
|
158
|
+
// caller (a path source's `path` is always a non-empty resolved filesystem
|
|
159
|
+
// path; it's never the empty string).
|
|
160
|
+
const cacheKey = 'path' in source ? pathCacheKey(source.path) : shsCacheKey(source);
|
|
161
|
+
const cached = byCacheKey.get(cacheKey);
|
|
162
|
+
if (cached && byRunId.has(cached)) {
|
|
163
|
+
touch(cached);
|
|
164
|
+
return { runId: cached, appModel: byRunId.get(cached) .appModel };
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
// Concurrent calls for the same not-yet-cached source (e.g. two tool calls
|
|
168
|
+
// in flight for the same path) must await one shared parse rather than each
|
|
169
|
+
// racing to insert their own cache entry: otherwise every racer's insert
|
|
170
|
+
// after the first is an orphaned entry (unreachable via cacheKey, wasting a
|
|
171
|
+
// cache slot until TTL/overflow).
|
|
172
|
+
const pending = pendingByCacheKey.get(cacheKey);
|
|
173
|
+
if (pending) return pending;
|
|
174
|
+
|
|
175
|
+
const resolution = (async () => {
|
|
176
|
+
try {
|
|
177
|
+
const appModel = 'path' in source
|
|
178
|
+
? await resolveFromPath(source.path)
|
|
179
|
+
: await resolveFromShs(source.shsBaseUrl, source.appId, source.attemptId, { fetchImpl, maxArchiveBytes, idleTimeoutMs });
|
|
180
|
+
|
|
181
|
+
const newRunId = randomUUID();
|
|
182
|
+
byRunId.set(newRunId, { appModel, cacheKey, lastAccess: Date.now() });
|
|
183
|
+
byCacheKey.set(cacheKey, newRunId);
|
|
184
|
+
evictOverflow();
|
|
185
|
+
return { runId: newRunId, appModel };
|
|
186
|
+
} finally {
|
|
187
|
+
pendingByCacheKey.delete(cacheKey);
|
|
188
|
+
}
|
|
189
|
+
})();
|
|
190
|
+
pendingByCacheKey.set(cacheKey, resolution);
|
|
191
|
+
return resolution;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
export function diagnoseRun(runId , opts
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
)
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
{
|
|
201
|
+
const appModel = getCachedAppModel(runId);
|
|
202
|
+
const findingsFilter = toFindingsFilter(opts?.impactBand, opts?.type, opts?.stageId);
|
|
203
|
+
const { json, markdown } = buildEvidenceReport(appModel, { redact: opts?.redact, markdown: opts?.markdown, findingsFilter });
|
|
204
|
+
const include = opts?.include ?? [];
|
|
205
|
+
return {
|
|
206
|
+
runId, findings: json.findings, recommendations: json.recommendations, cleanChecks: json.cleanChecks,
|
|
207
|
+
runComplete: appModel.app?.endTime != null,
|
|
208
|
+
...(include.includes('summary') ? { summary: json.summary } : {}),
|
|
209
|
+
...(include.includes('evidenceAvailability') ? { evidenceAvailability: json.evidenceAvailability } : {}),
|
|
210
|
+
...(include.includes('detectors') ? { detectors: json.detectors } : {}),
|
|
211
|
+
...(opts?.markdown ? { markdown } : {}),
|
|
212
|
+
};
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
export function getFindingEvidence(
|
|
216
|
+
runId , findingId , opts ,
|
|
217
|
+
) {
|
|
218
|
+
const appModel = getCachedAppModel(runId);
|
|
219
|
+
const { json } = buildEvidenceReport(appModel, { redact: opts?.redact, markdown: false });
|
|
220
|
+
const finding = json.findings.find((f) => f.id === findingId);
|
|
221
|
+
if (!finding) throw mcpError('finding-not-found', `No finding ${findingId} on run ${runId}.`);
|
|
222
|
+
return { runId, finding };
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
function hasCompleteInterval(app ) {
|
|
226
|
+
return Number.isFinite(app?.startTime) && Number.isFinite(app?.endTime) && (app?.endTime ?? 0) > (app?.startTime ?? 0);
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
export function getRunSummary(runId , opts ) {
|
|
230
|
+
const appModel = getCachedAppModel(runId);
|
|
231
|
+
const { app, stages, jobs, sql, executors } = appModel;
|
|
232
|
+
const durationMs = hasCompleteInterval(app) ? computeWallClock(app, stages).total : null;
|
|
233
|
+
// No buildEvidenceReport call here to redact (see the type comment above),
|
|
234
|
+
// so reuse redact.ts's app-id + host-token pseudonymization directly via
|
|
235
|
+
// redactAppIdentity(). Passing name/sparkVersion through it (not just id)
|
|
236
|
+
// matters because app.name is free text and can itself carry a host/IP token.
|
|
237
|
+
const rawApp = { id: app?.id ?? null, name: app?.name ?? null, sparkVersion: app?.sparkVersion ?? null };
|
|
238
|
+
const redactedApp = opts?.redact ? redactAppIdentity(rawApp) : rawApp;
|
|
239
|
+
return {
|
|
240
|
+
runId,
|
|
241
|
+
app: redactedApp,
|
|
242
|
+
stageCount: stages.size,
|
|
243
|
+
jobCount: jobs.size,
|
|
244
|
+
sqlExecutionCount: sql.size,
|
|
245
|
+
executorCount: { added: executors.added.length, removed: executors.removed.length },
|
|
246
|
+
durationMs,
|
|
247
|
+
runComplete: app?.endTime != null,
|
|
248
|
+
};
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
// Shared by compareRuns and evaluateBudgetsForRun: both need a resolved run's
|
|
252
|
+
// finding catalog (same analyze() call shape) before doing anything else with it.
|
|
253
|
+
async function resolveAndAnalyze(ref ) {
|
|
254
|
+
const { runId, appModel } = await resolveOrCreateRun(ref);
|
|
255
|
+
const catalog = analyze(
|
|
256
|
+
appModel.app, appModel.stages, appModel.executors.added, appModel.executors.removed,
|
|
257
|
+
appModel.jobs, appModel.sql, appModel.runAggregates,
|
|
258
|
+
);
|
|
259
|
+
return { runId, appModel, catalog };
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
export async function compareRuns(
|
|
263
|
+
a , b , opts ,
|
|
264
|
+
) {
|
|
265
|
+
const [
|
|
266
|
+
{ runId: runIdA, appModel: appModelA, catalog: catalogA },
|
|
267
|
+
{ runId: runIdB, appModel: appModelB, catalog: catalogB },
|
|
268
|
+
] = await Promise.all([resolveAndAnalyze(a), resolveAndAnalyze(b)]);
|
|
269
|
+
|
|
270
|
+
// buildComparison's captureSnapshot step uses an empty taskDataCache: that
|
|
271
|
+
// arg only feeds the interactive stage-detail drill-down modal, which none
|
|
272
|
+
// of compareRuns/matchStages/metricDeltas/findingsDelta/stageSkewDeltas/
|
|
273
|
+
// stageList read. Findings/metrics come from `catalog` (already computed
|
|
274
|
+
// via `analyze()` above) and from `appModel.stages`/`sql` (which already
|
|
275
|
+
// carry plan-tree data), both fully populated regardless. Produces
|
|
276
|
+
// identical comparison output to the dashboard's session-cache snapshots;
|
|
277
|
+
// the MCP tool just never exposes per-task drill-down, so there's nothing
|
|
278
|
+
// to prefetch into it.
|
|
279
|
+
const built = buildComparison(
|
|
280
|
+
{ label: runIdA, appModel: appModelA, catalog: catalogA },
|
|
281
|
+
{ label: runIdB, appModel: appModelB, catalog: catalogB },
|
|
282
|
+
);
|
|
283
|
+
// Stage names throughout `built` (findings.introduced/resolved[].stages,
|
|
284
|
+
// baseStages/candStages[].name) carry raw Spark stage text, which can embed
|
|
285
|
+
// a host/IP token as free text — the same residual redactReport() already
|
|
286
|
+
// scrubs from the evidence report.
|
|
287
|
+
const result = opts?.redact ? redactComparison(built) : built;
|
|
288
|
+
|
|
289
|
+
return {
|
|
290
|
+
runIdA,
|
|
291
|
+
runIdB,
|
|
292
|
+
findingsDelta: result.findings,
|
|
293
|
+
metricDeltas: result.metrics,
|
|
294
|
+
confidence: result.confidence,
|
|
295
|
+
reason: result.reason,
|
|
296
|
+
matchedCoverage: result.matchedCoverage,
|
|
297
|
+
...(opts?.markdown ? { markdown: renderComparisonMarkdown(result) } : {}),
|
|
298
|
+
};
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
export async function evaluateBudgetsForRun(
|
|
302
|
+
primary ,
|
|
303
|
+
budgets ,
|
|
304
|
+
secondary ,
|
|
305
|
+
) {
|
|
306
|
+
// Mirrors the CLI's own --regression-metric/--max-regression-pct pairing
|
|
307
|
+
// guard (bin/sparkforensics-analyze.mjs): unlike the CLI, this tool never
|
|
308
|
+
// defaults regressionMetric on the caller's behalf, so seeing it set here
|
|
309
|
+
// unambiguously means the caller asked for a regression check and forgot
|
|
310
|
+
// the threshold — evaluateBudgets() would otherwise skip the check with no
|
|
311
|
+
// signal at all.
|
|
312
|
+
if (budgets.regressionMetric !== undefined && budgets.maxRegressionPct === undefined) {
|
|
313
|
+
throw mcpError('access-or-upstream-failure', 'regressionMetric requires maxRegressionPct.');
|
|
314
|
+
}
|
|
315
|
+
const [{ runId, appModel, catalog }, second] = await Promise.all([
|
|
316
|
+
resolveAndAnalyze(primary),
|
|
317
|
+
secondary ? resolveAndAnalyze(secondary) : Promise.resolve(undefined),
|
|
318
|
+
]);
|
|
319
|
+
|
|
320
|
+
let comparison ;
|
|
321
|
+
if (second) {
|
|
322
|
+
comparison = buildComparison(
|
|
323
|
+
{ label: runId, appModel, catalog },
|
|
324
|
+
{ label: second.runId, appModel: second.appModel, catalog: second.catalog },
|
|
325
|
+
);
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
const { results, violated, inconclusive } = evaluateBudgets({ appModel, catalog, budgets, comparison });
|
|
329
|
+
|
|
330
|
+
return { runId, results, violated, inconclusive };
|
|
331
|
+
}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
// Worker-message → appModel assembly. Shared by file-load and SHS-URL-load paths,
|
|
13
|
+
// which previously carried byte-identical callback blocks. Pure model mutation:
|
|
14
|
+
// analysis/render/persist stay in the caller via the onDone/onProgress/onError hooks.
|
|
15
|
+
export function createModelCallbacks(
|
|
16
|
+
appModel ,
|
|
17
|
+
{ onProgress, onDone, onError }
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
,
|
|
22
|
+
) {
|
|
23
|
+
return {
|
|
24
|
+
onProgress,
|
|
25
|
+
onApp(data ) { appModel.app = data ; },
|
|
26
|
+
onStage(data ) {
|
|
27
|
+
const stage = data ;
|
|
28
|
+
appModel.stages.set(stage.id, stage);
|
|
29
|
+
},
|
|
30
|
+
onSql(data ) {
|
|
31
|
+
// Full overwrite, not a merge: safe only because `planTree` (set by
|
|
32
|
+
// onSqlPlan below, from a separate 'sqlPlan' message) is never present
|
|
33
|
+
// on a 'sql' message's data. That in turn relies on
|
|
34
|
+
// SparkListenerSQLAdaptiveExecutionUpdate always preceding
|
|
35
|
+
// SparkListenerSQLExecutionEnd for the same execution (Spark emits a
|
|
36
|
+
// re-plan mid-run, never after the query finishes); see
|
|
37
|
+
// applyAdaptiveExecutionUpdate in event-handlers.ts. A future producer
|
|
38
|
+
// of 'sql' messages that could arrive after onSqlPlan would need this
|
|
39
|
+
// to merge instead of overwrite.
|
|
40
|
+
const event = data ;
|
|
41
|
+
appModel.sql.set(event.id, data );
|
|
42
|
+
for (const stageId of (event.stageIds ?? [])) {
|
|
43
|
+
const stage = appModel.stages.get(stageId);
|
|
44
|
+
if (stage) stage.sqlExecutionId = event.id;
|
|
45
|
+
}
|
|
46
|
+
},
|
|
47
|
+
onSqlPlan(data ) {
|
|
48
|
+
const event = data ;
|
|
49
|
+
const exec = appModel.sql.get(event.executionId);
|
|
50
|
+
if (exec) exec.planTree = event.planTree;
|
|
51
|
+
},
|
|
52
|
+
onExecutor(data ) {
|
|
53
|
+
const event = data ;
|
|
54
|
+
if (event.kind === 'added') appModel.executors.added.push(event);
|
|
55
|
+
else appModel.executors.removed.push(event);
|
|
56
|
+
},
|
|
57
|
+
onJob(data ) {
|
|
58
|
+
const job = data ;
|
|
59
|
+
appModel.jobs.set(job.id, job);
|
|
60
|
+
},
|
|
61
|
+
onRunAggregates(data ) { appModel.runAggregates = data ; },
|
|
62
|
+
// Patch per-stage executorMetrics posted once before `done`: see the
|
|
63
|
+
// parser worker's SparkListenerStageExecutorMetrics handler (those events
|
|
64
|
+
// arrive after StageCompleted, so the stage message itself carried an
|
|
65
|
+
// empty map). `data` is a Map<stageId, Map<execId, metrics>>.
|
|
66
|
+
onStageExecutorMetrics(data ) {
|
|
67
|
+
if (!(data instanceof Map)) return;
|
|
68
|
+
const metricsByStage = data ;
|
|
69
|
+
for (const [stageId, execMetrics] of metricsByStage) {
|
|
70
|
+
const stage = appModel.stages.get(stageId);
|
|
71
|
+
if (stage) stage.executorMetrics = execMetrics;
|
|
72
|
+
}
|
|
73
|
+
},
|
|
74
|
+
onDone, onError,
|
|
75
|
+
};
|
|
76
|
+
}
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
// src/occupancy.ts
|
|
2
|
+
// Occupancy-weighted wall-clock attribution (N1 redesign). Replaces the old
|
|
3
|
+
// CPM pass over parentIds for per-finding impact estimation. No graph, no
|
|
4
|
+
// parentIds traversal: every stage's wall-clock claim is apportioned purely
|
|
5
|
+
// from its own observed window and how much it overlapped with other
|
|
6
|
+
// stages.
|
|
7
|
+
import { mergeIntervals } from './wall-clock.js';
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
function durationMs(s ) {
|
|
18
|
+
return (s.completedAt ?? 0) - (s.submittedAt ?? 0);
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
// Average-concurrency proxy, held constant across the stage's whole active
|
|
22
|
+
// window: no per-task timestamps exist outside the parser worker to do
|
|
23
|
+
// better (see the redesign spec's "Explicitly rejected" section).
|
|
24
|
+
function coreWeight(s ) {
|
|
25
|
+
const d = durationMs(s);
|
|
26
|
+
return d > 0 ? (s.executorRunTime ?? 0) / d : 0;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Sweeps every stage's observed [submittedAt, completedAt) window and splits
|
|
31
|
+
* each instant's wall-clock among the stages active at that instant,
|
|
32
|
+
* proportional to coreWeight(S). Stages with duration(S) <= 0 are excluded
|
|
33
|
+
* entirely: they get no key in the returned map. When every active stage in
|
|
34
|
+
* an interval has coreWeight 0 (no executorRunTime data), the interval is
|
|
35
|
+
* split equally among them instead of everyone getting 0: a lone active
|
|
36
|
+
* stage must still resolve to occupying its own whole window regardless of
|
|
37
|
+
* whether core-time data exists for it.
|
|
38
|
+
*/
|
|
39
|
+
export function computeOccupancyMs(stages ) {
|
|
40
|
+
const valid = [...stages.values()].filter((s) => durationMs(s) > 0);
|
|
41
|
+
const occupancy = new Map ();
|
|
42
|
+
for (const s of valid) occupancy.set(s.id, 0);
|
|
43
|
+
if (valid.length === 0) return occupancy;
|
|
44
|
+
|
|
45
|
+
const events = [];
|
|
46
|
+
for (const s of valid) {
|
|
47
|
+
const w = coreWeight(s);
|
|
48
|
+
events.push({ time: s.submittedAt ?? 0, id: s.id, weight: w, isStart: true });
|
|
49
|
+
events.push({ time: s.completedAt ?? 0, id: s.id, weight: w, isStart: false });
|
|
50
|
+
}
|
|
51
|
+
events.sort((a, b) => a.time - b.time);
|
|
52
|
+
|
|
53
|
+
const active = new Map (); // id -> coreWeight
|
|
54
|
+
let prevTime = events[0].time;
|
|
55
|
+
let i = 0;
|
|
56
|
+
while (i < events.length) {
|
|
57
|
+
const time = events[i].time;
|
|
58
|
+
if (time > prevTime && active.size > 0) {
|
|
59
|
+
const span = time - prevTime;
|
|
60
|
+
let totalWeight = 0;
|
|
61
|
+
for (const w of active.values()) totalWeight += w;
|
|
62
|
+
for (const [id, w] of active) {
|
|
63
|
+
const share = totalWeight > 0 ? w / totalWeight : 1 / active.size;
|
|
64
|
+
occupancy.set(id, (occupancy.get(id) ?? 0) + span * share);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
while (i < events.length && events[i].time === time) {
|
|
68
|
+
const e = events[i];
|
|
69
|
+
if (e.isStart) active.set(e.id, e.weight);
|
|
70
|
+
else active.delete(e.id);
|
|
71
|
+
i++;
|
|
72
|
+
}
|
|
73
|
+
prevTime = time;
|
|
74
|
+
}
|
|
75
|
+
return occupancy;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Occupancy-share of a stage's own observed duration, gate(S) in [0, 1];
|
|
80
|
+
* 1.0 means the stage ran completely alone, 0 means it had zero executorRunTime
|
|
81
|
+
* while overlapping other, positive-weight stages (so it got no share of the
|
|
82
|
+
* shared window). Excludes stages with duration(S) <= 0 (no key in the result).
|
|
83
|
+
*/
|
|
84
|
+
export function computeGate(stages ) {
|
|
85
|
+
const occupancyMs = computeOccupancyMs(stages);
|
|
86
|
+
const gate = new Map ();
|
|
87
|
+
for (const s of stages.values()) {
|
|
88
|
+
const d = durationMs(s);
|
|
89
|
+
if (d <= 0) continue;
|
|
90
|
+
const occ = occupancyMs.get(s.id) ?? 0;
|
|
91
|
+
// Clamp for float round-off in the sweep; occupancy can never truly exceed duration.
|
|
92
|
+
gate.set(s.id, Math.min(1, occ / d));
|
|
93
|
+
}
|
|
94
|
+
return gate;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* A physical floor on a stage's own duration: bounded below by its single
|
|
99
|
+
* longest task (unsplittable no matter how much parallelism exists) or by
|
|
100
|
+
* its total core-work spread across every core in the cluster, whichever is
|
|
101
|
+
* larger. totalCores <= 0 (no executor data) falls back to taskDurationMax
|
|
102
|
+
* alone.
|
|
103
|
+
*/
|
|
104
|
+
export function computeCeiling(stage , totalCores ) {
|
|
105
|
+
const taskDurationMax = stage.taskDurationMax ?? 0;
|
|
106
|
+
if (totalCores <= 0) return taskDurationMax;
|
|
107
|
+
const coreWork = stage.executorRunTime ?? 0;
|
|
108
|
+
return Math.max(taskDurationMax, coreWork / totalCores);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* Caps a waste formula's raw claim at the portion of the stage's own
|
|
113
|
+
* observed duration that sits above its physical floor: a finding can never
|
|
114
|
+
* claim to recover more than that.
|
|
115
|
+
*/
|
|
116
|
+
export function clipToCeiling(wasteMsClaimed , stage , ceiling ) {
|
|
117
|
+
const room = Math.max(0, durationMs(stage) - ceiling);
|
|
118
|
+
return Math.min(wasteMsClaimed, room);
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
export function computeOccupancy(
|
|
127
|
+
stages ,
|
|
128
|
+
totalCores ,
|
|
129
|
+
) {
|
|
130
|
+
const gate = computeGate(stages);
|
|
131
|
+
const info = new Map ();
|
|
132
|
+
for (const s of stages.values()) {
|
|
133
|
+
const g = gate.get(s.id);
|
|
134
|
+
if (g === undefined) continue; // excluded from the sweep (duration <= 0)
|
|
135
|
+
info.set(s.id, { gate: g, ceiling: computeCeiling(s, totalCores) });
|
|
136
|
+
}
|
|
137
|
+
return info;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
const SERIAL_GATE_THRESHOLD = 0.999;
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* Per-finding estimate for a single-stage waste claim. Returns null when the
|
|
149
|
+
* stage was excluded from the sweep (duration <= 0): callers must fall back
|
|
150
|
+
* to a resourceOnly/informational basis with wallClock: null, never a fake
|
|
151
|
+
* {0, 0}.
|
|
152
|
+
*/
|
|
153
|
+
export function estimateSingleStage(
|
|
154
|
+
wasteMsClaimed ,
|
|
155
|
+
stageId ,
|
|
156
|
+
stages ,
|
|
157
|
+
info ,
|
|
158
|
+
) {
|
|
159
|
+
const stage = stages.get(stageId);
|
|
160
|
+
const stageInfo = info.get(stageId);
|
|
161
|
+
if (!stage || !stageInfo) return null;
|
|
162
|
+
const clipped = clipToCeiling(wasteMsClaimed, stage, stageInfo.ceiling);
|
|
163
|
+
if (stageInfo.gate >= SERIAL_GATE_THRESHOLD) {
|
|
164
|
+
return { basis: 'serial', wallClock: { low: clipped, high: clipped } };
|
|
165
|
+
}
|
|
166
|
+
return { basis: 'contended', wallClock: { low: clipped * stageInfo.gate, high: clipped } };
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/**
|
|
170
|
+
* Sum-then-cap over a finding's own stages: the union of the finding's own
|
|
171
|
+
* stage windows bounds the joint claim, so two overlapping stages' waste
|
|
172
|
+
* can't be double-counted. Returns null only when every one of the
|
|
173
|
+
* finding's stages was excluded from the sweep.
|
|
174
|
+
*/
|
|
175
|
+
export function estimateMultiStage(
|
|
176
|
+
stageIds ,
|
|
177
|
+
wasteMsByStage ,
|
|
178
|
+
stages ,
|
|
179
|
+
info ,
|
|
180
|
+
) {
|
|
181
|
+
const perStage = [];
|
|
182
|
+
const intervals = [];
|
|
183
|
+
for (const id of stageIds) {
|
|
184
|
+
const est = estimateSingleStage(wasteMsByStage.get(id) ?? 0, id, stages, info);
|
|
185
|
+
if (!est) continue;
|
|
186
|
+
perStage.push(est);
|
|
187
|
+
const stage = stages.get(id) ;
|
|
188
|
+
intervals.push([stage.submittedAt ?? 0, stage.completedAt ?? 0]);
|
|
189
|
+
}
|
|
190
|
+
if (perStage.length === 0) return null;
|
|
191
|
+
const sumHigh = perStage.reduce((sum, e) => sum + e.wallClock.high, 0);
|
|
192
|
+
const sumLow = perStage.reduce((sum, e) => sum + e.wallClock.low, 0);
|
|
193
|
+
const unionMs = mergeIntervals(intervals).reduce((sum, [a, b]) => sum + (b - a), 0);
|
|
194
|
+
const high = Math.min(sumHigh, unionMs);
|
|
195
|
+
const low = Math.min(sumLow, unionMs);
|
|
196
|
+
// Numeric low === high isn't enough: the union cap can force that equality
|
|
197
|
+
// even when the constituent stages were individually contended (e.g. two
|
|
198
|
+
// fully-overlapping stages each at gate 0.5). Only claim 'serial' when
|
|
199
|
+
// every contributing per-stage estimate was itself serial.
|
|
200
|
+
const basis = perStage.every((e) => e.basis === 'serial') ? 'serial' : 'contended';
|
|
201
|
+
return { basis, wallClock: { low, high } };
|
|
202
|
+
}
|