sparkforensics-cli 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/bin/sparkforensics-analyze.mjs +288 -0
  2. package/package.json +29 -0
  3. package/vendor-core/analyzer.js +167 -0
  4. package/vendor-core/assert-never.js +3 -0
  5. package/vendor-core/cli/budgets.js +203 -0
  6. package/vendor-core/cli/collect-run.js +107 -0
  7. package/vendor-core/core-count.js +68 -0
  8. package/vendor-core/core-locality-ratio.js +54 -0
  9. package/vendor-core/core-time-series.js +92 -0
  10. package/vendor-core/core-usage-locality.js +70 -0
  11. package/vendor-core/detectors.js +1989 -0
  12. package/vendor-core/docs-config.js +72 -0
  13. package/vendor-core/docs-site-config.js +23 -0
  14. package/vendor-core/efficiency-model.js +62 -0
  15. package/vendor-core/etl-phases.js +28 -0
  16. package/vendor-core/event-handlers.js +906 -0
  17. package/vendor-core/event-schemas.js +405 -0
  18. package/vendor-core/evidence-availability.js +121 -0
  19. package/vendor-core/evidence-report.js +459 -0
  20. package/vendor-core/finding-action-label.js +97 -0
  21. package/vendor-core/finding-filter-predicate.js +38 -0
  22. package/vendor-core/format-utils.js +167 -0
  23. package/vendor-core/impact-band.js +50 -0
  24. package/vendor-core/impact-estimator.js +428 -0
  25. package/vendor-core/ingest.js +139 -0
  26. package/vendor-core/job-groups.js +30 -0
  27. package/vendor-core/load-vendored.js +24 -0
  28. package/vendor-core/lz4-block.js +135 -0
  29. package/vendor-core/mcp-error.js +3 -0
  30. package/vendor-core/mcp-server-factory.js +115 -0
  31. package/vendor-core/mcp-tools.js +331 -0
  32. package/vendor-core/model-assembler.js +76 -0
  33. package/vendor-core/occupancy.js +202 -0
  34. package/vendor-core/parser-worker.js +249 -0
  35. package/vendor-core/plan-dot.js +25 -0
  36. package/vendor-core/plan-duration-attribution.js +185 -0
  37. package/vendor-core/plan-graph-model.js +171 -0
  38. package/vendor-core/plan-node-detail.js +159 -0
  39. package/vendor-core/plan-summary.js +233 -0
  40. package/vendor-core/plan-tree-walk.js +29 -0
  41. package/vendor-core/proxy.js +157 -0
  42. package/vendor-core/recommendation-rollup.js +197 -0
  43. package/vendor-core/redact.js +175 -0
  44. package/vendor-core/rolling-log-reassembly.js +52 -0
  45. package/vendor-core/run-aggregates.js +44 -0
  46. package/vendor-core/run-comparison.js +458 -0
  47. package/vendor-core/scaling-sim.js +73 -0
  48. package/vendor-core/session-snapshot.js +79 -0
  49. package/vendor-core/shs-fetch.js +196 -0
  50. package/vendor-core/shs-load.js +121 -0
  51. package/vendor-core/shs-request.js +101 -0
  52. package/vendor-core/shs-schemas.js +13 -0
  53. package/vendor-core/snappy-block.js +140 -0
  54. package/vendor-core/stage-quantiles.js +199 -0
  55. package/vendor-core/threshold-summary.js +35 -0
  56. package/vendor-core/types.js +286 -0
  57. package/vendor-core/vendor/fflate.js +2695 -0
  58. package/vendor-core/vendor/fzstd.js +768 -0
  59. package/vendor-core/wall-clock.js +36 -0
  60. package/vendor-core/wasted-core-hours.js +68 -0
@@ -0,0 +1,331 @@
1
+ import { resolve as resolvePath } from 'node:path';
2
+ import { existsSync, statSync } from 'node:fs';
3
+ import { randomUUID } from 'node:crypto';
4
+ import { collectRun } from './cli/collect-run.js';
5
+ import { deriveEvidenceAvailability } from './evidence-availability.js';
6
+ import { resolveFromShs, DEFAULT_MAX_ARCHIVE_BYTES, DEFAULT_IDLE_TIMEOUT_MS } from './shs-load.js';
7
+ import { mcpError } from './mcp-error.js';
8
+ import { buildEvidenceReport, toFindingsFilter, } from './evidence-report.js';
9
+ import { redactAppIdentity, redactComparison } from './redact.js';
10
+ import { computeWallClock } from './wall-clock.js';
11
+ import { analyze } from './analyzer.js';
12
+ import { buildComparison, renderComparisonMarkdown, } from './run-comparison.js';
13
+ import { evaluateBudgets, } from './cli/budgets.js';
14
+
15
+
16
+
17
+ // NOTE on a deviation from the plan's literal text: the plan defines RunRef as
18
+ // `{ runId?: string } & Partial<RunSource>` (path/shsBaseUrl/appId/attemptId
19
+ // flattened directly onto the object). Every real call site disagrees with
20
+ // that shape: resolveOrCreateRun's own destructuring (`{ source, runId } = {}`
21
+ // below), both mcp-server-factory.ts call sites (`compareRuns({ runId: runIdA,
22
+ // source: sourceA }, ...)`), and every resolveOrCreateRun/compareRuns call in
23
+ // tests/mcp-tools.test.js all pass a *nested* `source` key, never a flattened
24
+ // path/shsBaseUrl directly on the ref object. Widened to match that real,
25
+ // long-standing shape instead of reshaping every working call site (or every
26
+ // test) to fit the literal text.
27
+
28
+
29
+
30
+
31
+
32
+
33
+
34
+
35
+
36
+
37
+
38
+
39
+
40
+
41
+
42
+
43
+
44
+
45
+ // The plan's Produces list types compareRuns's return as `CompareRunsResult`
46
+ // (run-comparison.ts's raw comparison shape: baselineLabel/candidateLabel/
47
+ // stageSkew/baseStages/candStages/...). compareRuns below actually returns a
48
+ // distinct, smaller MCP-facing projection of that (runIdA/runIdB/
49
+ // findingsDelta/metricDeltas/confidence/reason/matchedCoverage); it never
50
+ // had baselineLabel, stageSkew, baseStages, or candStages. Introduced this
51
+ // interface to match the real returned shape rather than casting a literal
52
+ // that's actually missing several required CompareRunsResult properties.
53
+
54
+
55
+
56
+
57
+
58
+
59
+
60
+
61
+
62
+
63
+ function envInt(name , fallback ) {
64
+ const v = Number(process.env[name]);
65
+ return Number.isFinite(v) && v > 0 ? v : fallback;
66
+ }
67
+
68
+ // `touch()` below does a full Map delete+re-insert per cache hit to maintain
69
+ // LRU order: O(n) per touch, fine at this small cap but worth re-checking
70
+ // the cost if this cap is ever raised significantly.
71
+ const CACHE_CAP = envInt('SPARKFORENSICS_MCP_CACHE_CAP', 8);
72
+ const CACHE_TTL_MS = envInt('SPARKFORENSICS_MCP_CACHE_TTL_MS', 15 * 60 * 1000);
73
+
74
+
75
+
76
+ const byRunId = new Map (); // runId -> { appModel, cacheKey, lastAccess }
77
+ const byCacheKey = new Map (); // cacheKey -> runId
78
+ const pendingByCacheKey = new Map (); // cacheKey -> in-flight Promise<{runId, appModel}>
79
+
80
+ export function pathCacheKey(path ) {
81
+ const resolved = resolvePath(path);
82
+ if (!existsSync(resolved)) throw mcpError('invalid-event-log', `No such file: ${resolved}`);
83
+ // Size narrows a same-millisecond mtime collision; ctime narrows the case
84
+ // where a copy tool (rsync --preserve-times, tar) restores an identical
85
+ // mtime and size for different content: ctime can't be set by the copying
86
+ // tool, so it still reflects the real time the file landed on disk.
87
+ const { mtimeMs, ctimeMs, size } = statSync(resolved);
88
+ return `path:${resolved}:${mtimeMs}:${ctimeMs}:${size}`;
89
+ }
90
+
91
+ function shsCacheKey({ shsBaseUrl, appId, attemptId } ) {
92
+ return `shs:${shsBaseUrl}:${appId}:${attemptId ?? ''}`;
93
+ }
94
+
95
+ function touch(runId ) {
96
+ const entry = byRunId.get(runId);
97
+ // Defensive only for type-narrowing: every call site below touches a runId
98
+ // it (or getCachedAppModel) has already confirmed present.
99
+ if (!entry) return;
100
+ byRunId.delete(runId);
101
+ entry.lastAccess = Date.now();
102
+ byRunId.set(runId, entry);
103
+ }
104
+
105
+ function deleteRunEntry(id , entry ) {
106
+ byRunId.delete(id);
107
+ if (byCacheKey.get(entry.cacheKey) === id) byCacheKey.delete(entry.cacheKey);
108
+ }
109
+
110
+ function evictStale() {
111
+ const now = Date.now();
112
+ for (const [id, entry] of byRunId) {
113
+ if (now - entry.lastAccess > CACHE_TTL_MS) {
114
+ deleteRunEntry(id, entry);
115
+ }
116
+ }
117
+ }
118
+
119
+ function evictOverflow() {
120
+ while (byRunId.size > CACHE_CAP) {
121
+ const oldestId = byRunId.keys().next().value;
122
+ // Defensive only for type-narrowing: the while condition guarantees
123
+ // byRunId is non-empty here, so .keys().next() always has a value.
124
+ if (oldestId === undefined) break;
125
+ const entry = byRunId.get(oldestId);
126
+ if (entry) deleteRunEntry(oldestId, entry);
127
+ }
128
+ }
129
+
130
+ export function getCachedAppModel(runId ) {
131
+ evictStale();
132
+ const entry = byRunId.get(runId);
133
+ if (!entry) throw mcpError('run-not-found', `No cached run for runId ${runId}.`);
134
+ touch(runId);
135
+ return entry.appModel;
136
+ }
137
+
138
+ async function resolveFromPath(path ) {
139
+ const { appModel, skippedLines } = await collectRun(path);
140
+ appModel.evidenceAvailability = deriveEvidenceAvailability(appModel, { skippedLines });
141
+ return appModel;
142
+ }
143
+
144
+ export async function resolveOrCreateRun(
145
+ { source, runId } = {},
146
+ { fetchImpl = fetch, maxArchiveBytes = DEFAULT_MAX_ARCHIVE_BYTES, idleTimeoutMs = DEFAULT_IDLE_TIMEOUT_MS }
147
+ = {},
148
+ ) {
149
+ evictStale();
150
+ if (runId) {
151
+ return { runId, appModel: getCachedAppModel(runId) };
152
+ }
153
+ if (!source) throw mcpError('access-or-upstream-failure', 'Provide either source or runId.');
154
+
155
+ // `'path' in source`, not `source.path`: RunSource's two variants share no
156
+ // common property, so a plain `.path` access doesn't type-check on the
157
+ // union. Equivalent at runtime to the original truthy check for every real
158
+ // caller (a path source's `path` is always a non-empty resolved filesystem
159
+ // path; it's never the empty string).
160
+ const cacheKey = 'path' in source ? pathCacheKey(source.path) : shsCacheKey(source);
161
+ const cached = byCacheKey.get(cacheKey);
162
+ if (cached && byRunId.has(cached)) {
163
+ touch(cached);
164
+ return { runId: cached, appModel: byRunId.get(cached) .appModel };
165
+ }
166
+
167
+ // Concurrent calls for the same not-yet-cached source (e.g. two tool calls
168
+ // in flight for the same path) must await one shared parse rather than each
169
+ // racing to insert their own cache entry: otherwise every racer's insert
170
+ // after the first is an orphaned entry (unreachable via cacheKey, wasting a
171
+ // cache slot until TTL/overflow).
172
+ const pending = pendingByCacheKey.get(cacheKey);
173
+ if (pending) return pending;
174
+
175
+ const resolution = (async () => {
176
+ try {
177
+ const appModel = 'path' in source
178
+ ? await resolveFromPath(source.path)
179
+ : await resolveFromShs(source.shsBaseUrl, source.appId, source.attemptId, { fetchImpl, maxArchiveBytes, idleTimeoutMs });
180
+
181
+ const newRunId = randomUUID();
182
+ byRunId.set(newRunId, { appModel, cacheKey, lastAccess: Date.now() });
183
+ byCacheKey.set(cacheKey, newRunId);
184
+ evictOverflow();
185
+ return { runId: newRunId, appModel };
186
+ } finally {
187
+ pendingByCacheKey.delete(cacheKey);
188
+ }
189
+ })();
190
+ pendingByCacheKey.set(cacheKey, resolution);
191
+ return resolution;
192
+ }
193
+
194
+ export function diagnoseRun(runId , opts
195
+
196
+
197
+ )
198
+
199
+
200
+ {
201
+ const appModel = getCachedAppModel(runId);
202
+ const findingsFilter = toFindingsFilter(opts?.impactBand, opts?.type, opts?.stageId);
203
+ const { json, markdown } = buildEvidenceReport(appModel, { redact: opts?.redact, markdown: opts?.markdown, findingsFilter });
204
+ const include = opts?.include ?? [];
205
+ return {
206
+ runId, findings: json.findings, recommendations: json.recommendations, cleanChecks: json.cleanChecks,
207
+ runComplete: appModel.app?.endTime != null,
208
+ ...(include.includes('summary') ? { summary: json.summary } : {}),
209
+ ...(include.includes('evidenceAvailability') ? { evidenceAvailability: json.evidenceAvailability } : {}),
210
+ ...(include.includes('detectors') ? { detectors: json.detectors } : {}),
211
+ ...(opts?.markdown ? { markdown } : {}),
212
+ };
213
+ }
214
+
215
+ export function getFindingEvidence(
216
+ runId , findingId , opts ,
217
+ ) {
218
+ const appModel = getCachedAppModel(runId);
219
+ const { json } = buildEvidenceReport(appModel, { redact: opts?.redact, markdown: false });
220
+ const finding = json.findings.find((f) => f.id === findingId);
221
+ if (!finding) throw mcpError('finding-not-found', `No finding ${findingId} on run ${runId}.`);
222
+ return { runId, finding };
223
+ }
224
+
225
+ function hasCompleteInterval(app ) {
226
+ return Number.isFinite(app?.startTime) && Number.isFinite(app?.endTime) && (app?.endTime ?? 0) > (app?.startTime ?? 0);
227
+ }
228
+
229
+ export function getRunSummary(runId , opts ) {
230
+ const appModel = getCachedAppModel(runId);
231
+ const { app, stages, jobs, sql, executors } = appModel;
232
+ const durationMs = hasCompleteInterval(app) ? computeWallClock(app, stages).total : null;
233
+ // No buildEvidenceReport call here to redact (see the type comment above),
234
+ // so reuse redact.ts's app-id + host-token pseudonymization directly via
235
+ // redactAppIdentity(). Passing name/sparkVersion through it (not just id)
236
+ // matters because app.name is free text and can itself carry a host/IP token.
237
+ const rawApp = { id: app?.id ?? null, name: app?.name ?? null, sparkVersion: app?.sparkVersion ?? null };
238
+ const redactedApp = opts?.redact ? redactAppIdentity(rawApp) : rawApp;
239
+ return {
240
+ runId,
241
+ app: redactedApp,
242
+ stageCount: stages.size,
243
+ jobCount: jobs.size,
244
+ sqlExecutionCount: sql.size,
245
+ executorCount: { added: executors.added.length, removed: executors.removed.length },
246
+ durationMs,
247
+ runComplete: app?.endTime != null,
248
+ };
249
+ }
250
+
251
+ // Shared by compareRuns and evaluateBudgetsForRun: both need a resolved run's
252
+ // finding catalog (same analyze() call shape) before doing anything else with it.
253
+ async function resolveAndAnalyze(ref ) {
254
+ const { runId, appModel } = await resolveOrCreateRun(ref);
255
+ const catalog = analyze(
256
+ appModel.app, appModel.stages, appModel.executors.added, appModel.executors.removed,
257
+ appModel.jobs, appModel.sql, appModel.runAggregates,
258
+ );
259
+ return { runId, appModel, catalog };
260
+ }
261
+
262
+ export async function compareRuns(
263
+ a , b , opts ,
264
+ ) {
265
+ const [
266
+ { runId: runIdA, appModel: appModelA, catalog: catalogA },
267
+ { runId: runIdB, appModel: appModelB, catalog: catalogB },
268
+ ] = await Promise.all([resolveAndAnalyze(a), resolveAndAnalyze(b)]);
269
+
270
+ // buildComparison's captureSnapshot step uses an empty taskDataCache: that
271
+ // arg only feeds the interactive stage-detail drill-down modal, which none
272
+ // of compareRuns/matchStages/metricDeltas/findingsDelta/stageSkewDeltas/
273
+ // stageList read. Findings/metrics come from `catalog` (already computed
274
+ // via `analyze()` above) and from `appModel.stages`/`sql` (which already
275
+ // carry plan-tree data), both fully populated regardless. Produces
276
+ // identical comparison output to the dashboard's session-cache snapshots;
277
+ // the MCP tool just never exposes per-task drill-down, so there's nothing
278
+ // to prefetch into it.
279
+ const built = buildComparison(
280
+ { label: runIdA, appModel: appModelA, catalog: catalogA },
281
+ { label: runIdB, appModel: appModelB, catalog: catalogB },
282
+ );
283
+ // Stage names throughout `built` (findings.introduced/resolved[].stages,
284
+ // baseStages/candStages[].name) carry raw Spark stage text, which can embed
285
+ // a host/IP token as free text — the same residual redactReport() already
286
+ // scrubs from the evidence report.
287
+ const result = opts?.redact ? redactComparison(built) : built;
288
+
289
+ return {
290
+ runIdA,
291
+ runIdB,
292
+ findingsDelta: result.findings,
293
+ metricDeltas: result.metrics,
294
+ confidence: result.confidence,
295
+ reason: result.reason,
296
+ matchedCoverage: result.matchedCoverage,
297
+ ...(opts?.markdown ? { markdown: renderComparisonMarkdown(result) } : {}),
298
+ };
299
+ }
300
+
301
+ export async function evaluateBudgetsForRun(
302
+ primary ,
303
+ budgets ,
304
+ secondary ,
305
+ ) {
306
+ // Mirrors the CLI's own --regression-metric/--max-regression-pct pairing
307
+ // guard (bin/sparkforensics-analyze.mjs): unlike the CLI, this tool never
308
+ // defaults regressionMetric on the caller's behalf, so seeing it set here
309
+ // unambiguously means the caller asked for a regression check and forgot
310
+ // the threshold — evaluateBudgets() would otherwise skip the check with no
311
+ // signal at all.
312
+ if (budgets.regressionMetric !== undefined && budgets.maxRegressionPct === undefined) {
313
+ throw mcpError('access-or-upstream-failure', 'regressionMetric requires maxRegressionPct.');
314
+ }
315
+ const [{ runId, appModel, catalog }, second] = await Promise.all([
316
+ resolveAndAnalyze(primary),
317
+ secondary ? resolveAndAnalyze(secondary) : Promise.resolve(undefined),
318
+ ]);
319
+
320
+ let comparison ;
321
+ if (second) {
322
+ comparison = buildComparison(
323
+ { label: runId, appModel, catalog },
324
+ { label: second.runId, appModel: second.appModel, catalog: second.catalog },
325
+ );
326
+ }
327
+
328
+ const { results, violated, inconclusive } = evaluateBudgets({ appModel, catalog, budgets, comparison });
329
+
330
+ return { runId, results, violated, inconclusive };
331
+ }
@@ -0,0 +1,76 @@
1
+
2
+
3
+
4
+
5
+
6
+
7
+
8
+
9
+
10
+
11
+
12
+ // Worker-message → appModel assembly. Shared by file-load and SHS-URL-load paths,
13
+ // which previously carried byte-identical callback blocks. Pure model mutation:
14
+ // analysis/render/persist stay in the caller via the onDone/onProgress/onError hooks.
15
+ export function createModelCallbacks(
16
+ appModel ,
17
+ { onProgress, onDone, onError }
18
+
19
+
20
+
21
+ ,
22
+ ) {
23
+ return {
24
+ onProgress,
25
+ onApp(data ) { appModel.app = data ; },
26
+ onStage(data ) {
27
+ const stage = data ;
28
+ appModel.stages.set(stage.id, stage);
29
+ },
30
+ onSql(data ) {
31
+ // Full overwrite, not a merge: safe only because `planTree` (set by
32
+ // onSqlPlan below, from a separate 'sqlPlan' message) is never present
33
+ // on a 'sql' message's data. That in turn relies on
34
+ // SparkListenerSQLAdaptiveExecutionUpdate always preceding
35
+ // SparkListenerSQLExecutionEnd for the same execution (Spark emits a
36
+ // re-plan mid-run, never after the query finishes); see
37
+ // applyAdaptiveExecutionUpdate in event-handlers.ts. A future producer
38
+ // of 'sql' messages that could arrive after onSqlPlan would need this
39
+ // to merge instead of overwrite.
40
+ const event = data ;
41
+ appModel.sql.set(event.id, data );
42
+ for (const stageId of (event.stageIds ?? [])) {
43
+ const stage = appModel.stages.get(stageId);
44
+ if (stage) stage.sqlExecutionId = event.id;
45
+ }
46
+ },
47
+ onSqlPlan(data ) {
48
+ const event = data ;
49
+ const exec = appModel.sql.get(event.executionId);
50
+ if (exec) exec.planTree = event.planTree;
51
+ },
52
+ onExecutor(data ) {
53
+ const event = data ;
54
+ if (event.kind === 'added') appModel.executors.added.push(event);
55
+ else appModel.executors.removed.push(event);
56
+ },
57
+ onJob(data ) {
58
+ const job = data ;
59
+ appModel.jobs.set(job.id, job);
60
+ },
61
+ onRunAggregates(data ) { appModel.runAggregates = data ; },
62
+ // Patch per-stage executorMetrics posted once before `done`: see the
63
+ // parser worker's SparkListenerStageExecutorMetrics handler (those events
64
+ // arrive after StageCompleted, so the stage message itself carried an
65
+ // empty map). `data` is a Map<stageId, Map<execId, metrics>>.
66
+ onStageExecutorMetrics(data ) {
67
+ if (!(data instanceof Map)) return;
68
+ const metricsByStage = data ;
69
+ for (const [stageId, execMetrics] of metricsByStage) {
70
+ const stage = appModel.stages.get(stageId);
71
+ if (stage) stage.executorMetrics = execMetrics;
72
+ }
73
+ },
74
+ onDone, onError,
75
+ };
76
+ }
@@ -0,0 +1,202 @@
1
+ // src/occupancy.ts
2
+ // Occupancy-weighted wall-clock attribution (N1 redesign). Replaces the old
3
+ // CPM pass over parentIds for per-finding impact estimation. No graph, no
4
+ // parentIds traversal: every stage's wall-clock claim is apportioned purely
5
+ // from its own observed window and how much it overlapped with other
6
+ // stages.
7
+ import { mergeIntervals } from './wall-clock.js';
8
+
9
+
10
+
11
+
12
+
13
+
14
+
15
+
16
+
17
+ function durationMs(s ) {
18
+ return (s.completedAt ?? 0) - (s.submittedAt ?? 0);
19
+ }
20
+
21
+ // Average-concurrency proxy, held constant across the stage's whole active
22
+ // window: no per-task timestamps exist outside the parser worker to do
23
+ // better (see the redesign spec's "Explicitly rejected" section).
24
+ function coreWeight(s ) {
25
+ const d = durationMs(s);
26
+ return d > 0 ? (s.executorRunTime ?? 0) / d : 0;
27
+ }
28
+
29
+ /**
30
+ * Sweeps every stage's observed [submittedAt, completedAt) window and splits
31
+ * each instant's wall-clock among the stages active at that instant,
32
+ * proportional to coreWeight(S). Stages with duration(S) <= 0 are excluded
33
+ * entirely: they get no key in the returned map. When every active stage in
34
+ * an interval has coreWeight 0 (no executorRunTime data), the interval is
35
+ * split equally among them instead of everyone getting 0: a lone active
36
+ * stage must still resolve to occupying its own whole window regardless of
37
+ * whether core-time data exists for it.
38
+ */
39
+ export function computeOccupancyMs(stages ) {
40
+ const valid = [...stages.values()].filter((s) => durationMs(s) > 0);
41
+ const occupancy = new Map ();
42
+ for (const s of valid) occupancy.set(s.id, 0);
43
+ if (valid.length === 0) return occupancy;
44
+
45
+ const events = [];
46
+ for (const s of valid) {
47
+ const w = coreWeight(s);
48
+ events.push({ time: s.submittedAt ?? 0, id: s.id, weight: w, isStart: true });
49
+ events.push({ time: s.completedAt ?? 0, id: s.id, weight: w, isStart: false });
50
+ }
51
+ events.sort((a, b) => a.time - b.time);
52
+
53
+ const active = new Map (); // id -> coreWeight
54
+ let prevTime = events[0].time;
55
+ let i = 0;
56
+ while (i < events.length) {
57
+ const time = events[i].time;
58
+ if (time > prevTime && active.size > 0) {
59
+ const span = time - prevTime;
60
+ let totalWeight = 0;
61
+ for (const w of active.values()) totalWeight += w;
62
+ for (const [id, w] of active) {
63
+ const share = totalWeight > 0 ? w / totalWeight : 1 / active.size;
64
+ occupancy.set(id, (occupancy.get(id) ?? 0) + span * share);
65
+ }
66
+ }
67
+ while (i < events.length && events[i].time === time) {
68
+ const e = events[i];
69
+ if (e.isStart) active.set(e.id, e.weight);
70
+ else active.delete(e.id);
71
+ i++;
72
+ }
73
+ prevTime = time;
74
+ }
75
+ return occupancy;
76
+ }
77
+
78
+ /**
79
+ * Occupancy-share of a stage's own observed duration, gate(S) in [0, 1];
80
+ * 1.0 means the stage ran completely alone, 0 means it had zero executorRunTime
81
+ * while overlapping other, positive-weight stages (so it got no share of the
82
+ * shared window). Excludes stages with duration(S) <= 0 (no key in the result).
83
+ */
84
+ export function computeGate(stages ) {
85
+ const occupancyMs = computeOccupancyMs(stages);
86
+ const gate = new Map ();
87
+ for (const s of stages.values()) {
88
+ const d = durationMs(s);
89
+ if (d <= 0) continue;
90
+ const occ = occupancyMs.get(s.id) ?? 0;
91
+ // Clamp for float round-off in the sweep; occupancy can never truly exceed duration.
92
+ gate.set(s.id, Math.min(1, occ / d));
93
+ }
94
+ return gate;
95
+ }
96
+
97
+ /**
98
+ * A physical floor on a stage's own duration: bounded below by its single
99
+ * longest task (unsplittable no matter how much parallelism exists) or by
100
+ * its total core-work spread across every core in the cluster, whichever is
101
+ * larger. totalCores <= 0 (no executor data) falls back to taskDurationMax
102
+ * alone.
103
+ */
104
+ export function computeCeiling(stage , totalCores ) {
105
+ const taskDurationMax = stage.taskDurationMax ?? 0;
106
+ if (totalCores <= 0) return taskDurationMax;
107
+ const coreWork = stage.executorRunTime ?? 0;
108
+ return Math.max(taskDurationMax, coreWork / totalCores);
109
+ }
110
+
111
+ /**
112
+ * Caps a waste formula's raw claim at the portion of the stage's own
113
+ * observed duration that sits above its physical floor: a finding can never
114
+ * claim to recover more than that.
115
+ */
116
+ export function clipToCeiling(wasteMsClaimed , stage , ceiling ) {
117
+ const room = Math.max(0, durationMs(stage) - ceiling);
118
+ return Math.min(wasteMsClaimed, room);
119
+ }
120
+
121
+
122
+
123
+
124
+
125
+
126
+ export function computeOccupancy(
127
+ stages ,
128
+ totalCores ,
129
+ ) {
130
+ const gate = computeGate(stages);
131
+ const info = new Map ();
132
+ for (const s of stages.values()) {
133
+ const g = gate.get(s.id);
134
+ if (g === undefined) continue; // excluded from the sweep (duration <= 0)
135
+ info.set(s.id, { gate: g, ceiling: computeCeiling(s, totalCores) });
136
+ }
137
+ return info;
138
+ }
139
+
140
+ const SERIAL_GATE_THRESHOLD = 0.999;
141
+
142
+
143
+
144
+
145
+
146
+
147
+ /**
148
+ * Per-finding estimate for a single-stage waste claim. Returns null when the
149
+ * stage was excluded from the sweep (duration <= 0): callers must fall back
150
+ * to a resourceOnly/informational basis with wallClock: null, never a fake
151
+ * {0, 0}.
152
+ */
153
+ export function estimateSingleStage(
154
+ wasteMsClaimed ,
155
+ stageId ,
156
+ stages ,
157
+ info ,
158
+ ) {
159
+ const stage = stages.get(stageId);
160
+ const stageInfo = info.get(stageId);
161
+ if (!stage || !stageInfo) return null;
162
+ const clipped = clipToCeiling(wasteMsClaimed, stage, stageInfo.ceiling);
163
+ if (stageInfo.gate >= SERIAL_GATE_THRESHOLD) {
164
+ return { basis: 'serial', wallClock: { low: clipped, high: clipped } };
165
+ }
166
+ return { basis: 'contended', wallClock: { low: clipped * stageInfo.gate, high: clipped } };
167
+ }
168
+
169
+ /**
170
+ * Sum-then-cap over a finding's own stages: the union of the finding's own
171
+ * stage windows bounds the joint claim, so two overlapping stages' waste
172
+ * can't be double-counted. Returns null only when every one of the
173
+ * finding's stages was excluded from the sweep.
174
+ */
175
+ export function estimateMultiStage(
176
+ stageIds ,
177
+ wasteMsByStage ,
178
+ stages ,
179
+ info ,
180
+ ) {
181
+ const perStage = [];
182
+ const intervals = [];
183
+ for (const id of stageIds) {
184
+ const est = estimateSingleStage(wasteMsByStage.get(id) ?? 0, id, stages, info);
185
+ if (!est) continue;
186
+ perStage.push(est);
187
+ const stage = stages.get(id) ;
188
+ intervals.push([stage.submittedAt ?? 0, stage.completedAt ?? 0]);
189
+ }
190
+ if (perStage.length === 0) return null;
191
+ const sumHigh = perStage.reduce((sum, e) => sum + e.wallClock.high, 0);
192
+ const sumLow = perStage.reduce((sum, e) => sum + e.wallClock.low, 0);
193
+ const unionMs = mergeIntervals(intervals).reduce((sum, [a, b]) => sum + (b - a), 0);
194
+ const high = Math.min(sumHigh, unionMs);
195
+ const low = Math.min(sumLow, unionMs);
196
+ // Numeric low === high isn't enough: the union cap can force that equality
197
+ // even when the constituent stages were individually contended (e.g. two
198
+ // fully-overlapping stages each at gate 0.5). Only claim 'serial' when
199
+ // every contributing per-stage estimate was itself serial.
200
+ const basis = perStage.every((e) => e.basis === 'serial') ? 'serial' : 'contended';
201
+ return { basis, wallClock: { low, high } };
202
+ }