sparkforensics-cli 0.1.0 → 0.2.0

package/vendor-core/mcp-tools.js

@@ -1,5 +1,6 @@
-import { resolve as resolvePath } from 'node:path';
-import { existsSync, statSync } from 'node:fs';
+import { resolve as resolvePath, join, dirname } from 'node:path';
+import { existsSync, statSync, readFileSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
 import { randomUUID } from 'node:crypto';
 import { collectRun } from './cli/collect-run.js';
 import { deriveEvidenceAvailability } from './evidence-availability.js';
@@ -11,45 +12,31 @@ import { computeWallClock } from './wall-clock.js';
 import { analyze } from './analyzer.js';
 import { buildComparison, renderComparisonMarkdown,                        } from './run-comparison.js';
 import { evaluateBudgets,                                       } from './cli/budgets.js';
+import { FINDING_NAMES, titleCase } from './finding-names.js';
+import { docAnchorForType, tuningDocSlugForAnchor, pageForAnchor } from './docs-config.js';
+import { typeTag } from './format-utils.js';
                                                                   
 
                                                                                                      
-// NOTE on a deviation from the plan's literal text: the plan defines RunRef as
-// `{ runId?: string } & Partial<RunSource>` (path/shsBaseUrl/appId/attemptId
-// flattened directly onto the object). Every real call site disagrees with
-// that shape: resolveOrCreateRun's own destructuring (`{ source, runId } = {}`
-// below), both mcp-server-factory.ts call sites (`compareRuns({ runId: runIdA,
-// source: sourceA }, ...)`), and every resolveOrCreateRun/compareRuns call in
-// tests/mcp-tools.test.js all pass a *nested* `source` key, never a flattened
-// path/shsBaseUrl directly on the ref object. Widened to match that real,
-// long-standing shape instead of reshaping every working call site (or every
-// test) to fit the literal text.
+// RunRef uses a nested `source` key, matching every real call site (resolveOrCreateRun's
+// destructuring, both mcp-server-factory.ts sites, and the tests all pass a nested `source`,
+// never flattened path/shsBaseUrl).
                                                             
                              
                 
-                                                                                     
-                                                                          
-                                                                                 
-                                                                          
+                                                                                                    
                                                                                
                                                                   
                                                     
                             
-                                                                        
-                                                                           
-                                                                         
-                                                                         
-                                                                       
+                                                                                                  
+                                                                                                   
+                                                                
                        
  
-// The plan's Produces list types compareRuns's return as `CompareRunsResult`
-// (run-comparison.ts's raw comparison shape: baselineLabel/candidateLabel/
-// stageSkew/baseStages/candStages/...). compareRuns below actually returns a
-// distinct, smaller MCP-facing projection of that (runIdA/runIdB/
-// findingsDelta/metricDeltas/confidence/reason/matchedCoverage); it never
-// had baselineLabel, stageSkew, baseStages, or candStages. Introduced this
-// interface to match the real returned shape rather than casting a literal
-// that's actually missing several required CompareRunsResult properties.
+// compareRuns returns a smaller MCP-facing projection of CompareRunsResult
+// (runIdA/runIdB/findingsDelta/metricDeltas/confidence/reason/matchedCoverage), not the full raw
+// shape (no baselineLabel/stageSkew/baseStages/candStages).
                                        
                  
                  
@@ -65,9 +52,8 @@ function envInt(name        , fallback        )         {
   return Number.isFinite(v) && v > 0 ? v : fallback;
 }
 
-// `touch()` below does a full Map delete+re-insert per cache hit to maintain
-// LRU order: O(n) per touch, fine at this small cap but worth re-checking
-// the cost if this cap is ever raised significantly.
+// touch() below does a full Map delete+re-insert per cache hit to maintain LRU order: O(n) per
+// touch, fine at this small cap but worth re-checking if the cap is raised significantly.
 const CACHE_CAP = envInt('SPARKFORENSICS_MCP_CACHE_CAP', 8);
 const CACHE_TTL_MS = envInt('SPARKFORENSICS_MCP_CACHE_TTL_MS', 15 * 60 * 1000);
 
@@ -80,10 +66,9 @@ const pendingByCacheKey = new Map
 export function pathCacheKey(path        )         {
   const resolved = resolvePath(path);
   if (!existsSync(resolved)) throw mcpError('invalid-event-log', `No such file: ${resolved}`);
-  // Size narrows a same-millisecond mtime collision; ctime narrows the case
-  // where a copy tool (rsync --preserve-times, tar) restores an identical
-  // mtime and size for different content: ctime can't be set by the copying
-  // tool, so it still reflects the real time the file landed on disk.
+  // Size narrows a same-millisecond mtime collision; ctime narrows the case where a copy tool
+  // (rsync --preserve-times, tar) restores an identical mtime+size for different content: ctime
+  // can't be set by the copying tool, so it still reflects when the file landed on disk.
   const { mtimeMs, ctimeMs, size } = statSync(resolved);
   return `path:${resolved}:${mtimeMs}:${ctimeMs}:${size}`;
 }
@@ -94,8 +79,7 @@ function shsCacheKey({ shsBaseUrl, appId, attemptId }
 
 function touch(runId        )       {
   const entry = byRunId.get(runId);
-  // Defensive only for type-narrowing: every call site below touches a runId
-  // it (or getCachedAppModel) has already confirmed present.
+  // Defensive only for type-narrowing: every call site touches a runId already confirmed present.
   if (!entry) return;
   byRunId.delete(runId);
   entry.lastAccess = Date.now();
@@ -119,8 +103,7 @@ function evictStale()       {
 function evictOverflow()       {
   while (byRunId.size > CACHE_CAP) {
     const oldestId = byRunId.keys().next().value;
-    // Defensive only for type-narrowing: the while condition guarantees
-    // byRunId is non-empty here, so .keys().next() always has a value.
+    // Defensive only for type-narrowing: the while condition guarantees byRunId is non-empty here.
     if (oldestId === undefined) break;
     const entry = byRunId.get(oldestId);
     if (entry) deleteRunEntry(oldestId, entry);
@@ -152,11 +135,8 @@ export async function resolveOrCreateRun(
   }
   if (!source) throw mcpError('access-or-upstream-failure', 'Provide either source or runId.');
 
-  // `'path' in source`, not `source.path`: RunSource's two variants share no
-  // common property, so a plain `.path` access doesn't type-check on the
-  // union. Equivalent at runtime to the original truthy check for every real
-  // caller (a path source's `path` is always a non-empty resolved filesystem
-  // path; it's never the empty string).
+  // `'path' in source`, not `source.path`: RunSource's two variants share no common property, so a
+  // plain `.path` access doesn't type-check on the union. Equivalent at runtime for every real caller.
   const cacheKey = 'path' in source ? pathCacheKey(source.path) : shsCacheKey(source);
   const cached = byCacheKey.get(cacheKey);
   if (cached && byRunId.has(cached)) {
@@ -164,11 +144,9 @@ export async function resolveOrCreateRun(
     return { runId: cached, appModel: byRunId.get(cached) .appModel };
   }
 
-  // Concurrent calls for the same not-yet-cached source (e.g. two tool calls
-  // in flight for the same path) must await one shared parse rather than each
-  // racing to insert their own cache entry: otherwise every racer's insert
-  // after the first is an orphaned entry (unreachable via cacheKey, wasting a
-  // cache slot until TTL/overflow).
+  // Concurrent calls for the same not-yet-cached source must await one shared parse, not each race
+  // to insert its own entry: otherwise every racer's insert after the first is orphaned (unreachable
+  // via cacheKey, wasting a cache slot until TTL/overflow).
   const pending = pendingByCacheKey.get(cacheKey);
   if (pending) return pending;
 
@@ -176,7 +154,7 @@ export async function resolveOrCreateRun(
     try {
       const appModel = 'path' in source
         ? await resolveFromPath(source.path)
-        : await resolveFromShs(source.shsBaseUrl, source.appId, source.attemptId, { fetchImpl, maxArchiveBytes, idleTimeoutMs });
+        : (await resolveFromShs(source.shsBaseUrl, source.appId, source.attemptId, { fetchImpl, maxArchiveBytes, idleTimeoutMs })).appModel;
 
       const newRunId = randomUUID();
       byRunId.set(newRunId, { appModel, cacheKey, lastAccess: Date.now() });
@@ -212,6 +190,68 @@ export function diagnoseRun(runId        , opts
   };
 }
 
+const DOCS_CONTENT_DIR = join(dirname(fileURLToPath(import.meta.url)), 'docs-content');
+
+// Both vendored corpora lead with their own heading line; strips the leading '#'/'###' markers, an
+// optional `` `TAG`: `` prefix (the detection doc's heading shape), and a trailing `{#anchor}`,
+// leaving just the human title. The tuning doc's plain `# Title` heading passes through as no-ops.
+function extractDocTitle(markdown        )         {
+  const firstLine = markdown.split('\n', 1)[0] ?? '';
+  return firstLine
+    .replace(/^#+\s*/, '')
+    .replace(/`[A-Z]+`:\s*/, '')
+    .replace(/\s*\{#[a-z0-9-]+\}\s*$/, '')
+    .trim();
+}
+
+                                       
+               
+               
+                                                                
+                                                                       
+ 
+
+/** Detection + tuning reference documentation for one finding `type`, independent of any run
+ * (documentation is a property of the type: a client fetches it once per type and caches it). */
+export function getFindingDocumentation(type        )                       {
+  const label = FINDING_NAMES[type];
+  if (label === undefined) throw mcpError('invalid-type', `Unknown finding type: ${type}`);
+
+  const tag = typeTag(type);
+  const detectionContent = readFileSync(join(DOCS_CONTENT_DIR, 'detection', `${tag.toLowerCase()}.md`), 'utf8');
+  const detectionDoc = { tag, title: extractDocTitle(detectionContent), content: detectionContent };
+
+  const anchor = docAnchorForType(type);
+  const slug = anchor ? tuningDocSlugForAnchor(anchor) : null;
+  const tuningPath = slug ? join(DOCS_CONTENT_DIR, 'tuning', `${slug}.md`) : null;
+  let tuningDoc                                    = null;
+  if (tuningPath && anchor && existsSync(tuningPath)) {
+    const tuningContent = readFileSync(tuningPath, 'utf8');
+    tuningDoc = { anchor, title: extractDocTitle(tuningContent), content: tuningContent };
+  }
+
+  return { type, name: titleCase(label), detectionDoc, tuningDoc };
+}
+
+const CHAPTERS_NAV_FILE = join(DOCS_CONTENT_DIR, 'chapters', 'nav-index.json');
+
+                                                                                 
+
+/** Full tuning-reference markdown for one doc anchor, run-independent. Resolves the anchor to its
+ * owning page via pageForAnchor (so '#metric-task-duration' returns the 'metrics' page), looks it up
+ * in the committed nav-index, and reads the markdown from the same docs-content store the website
+ * renders from. The general-chapter counterpart to getFindingDocumentation (keyed by finding type). */
+export function getReferenceDoc(anchor        )               {
+  const page = pageForAnchor(String(anchor).replace(/^#/, ''));
+  const nav = JSON.parse(readFileSync(CHAPTERS_NAV_FILE, 'utf8'))   
+                                                                         ;
+  const entry = nav.find((e) => e.anchor === page);
+  if (!entry) throw mcpError('invalid-anchor', `Unknown reference anchor: ${anchor}`);
+  const dir = entry.store === 'tuning' ? 'tuning' : 'chapters';
+  const content = readFileSync(join(DOCS_CONTENT_DIR, dir, `${entry.slug}.md`), 'utf8');
+  return { anchor: page, title: entry.title, content };
+}
+
 export function getFindingEvidence(
   runId        , findingId        , opts                       ,
 )                                         {
@@ -230,10 +270,9 @@ export function getRunSummary(runId        , opts                       )
   const appModel = getCachedAppModel(runId);
   const { app, stages, jobs, sql, executors } = appModel;
   const durationMs = hasCompleteInterval(app) ? computeWallClock(app, stages).total : null;
-  // No buildEvidenceReport call here to redact (see the type comment above),
-  // so reuse redact.ts's app-id + host-token pseudonymization directly via
-  // redactAppIdentity(). Passing name/sparkVersion through it (not just id)
-  // matters because app.name is free text and can itself carry a host/IP token.
+  // No buildEvidenceReport call here to redact, so reuse redact.ts's app-id + host-token
+  // pseudonymization directly. Passing name/sparkVersion (not just id) matters: app.name is free
+  // text and can itself carry a host/IP token.
   const rawApp = { id: app?.id ?? null, name: app?.name ?? null, sparkVersion: app?.sparkVersion ?? null };
   const redactedApp = opts?.redact ? redactAppIdentity(rawApp) : rawApp;
   return {
@@ -248,8 +287,8 @@ export function getRunSummary(runId        , opts                       )
   };
 }
 
-// Shared by compareRuns and evaluateBudgetsForRun: both need a resolved run's
-// finding catalog (same analyze() call shape) before doing anything else with it.
+// Shared by compareRuns and evaluateBudgetsForRun: both need a resolved run's finding catalog
+// (same analyze() call shape) first.
 async function resolveAndAnalyze(ref        )                                                                     {
   const { runId, appModel } = await resolveOrCreateRun(ref);
   const catalog = analyze(
@@ -267,23 +306,16 @@ export async function compareRuns(
     { runId: runIdB, appModel: appModelB, catalog: catalogB },
   ] = await Promise.all([resolveAndAnalyze(a), resolveAndAnalyze(b)]);
 
-  // buildComparison's captureSnapshot step uses an empty taskDataCache: that
-  // arg only feeds the interactive stage-detail drill-down modal, which none
-  // of compareRuns/matchStages/metricDeltas/findingsDelta/stageSkewDeltas/
-  // stageList read. Findings/metrics come from `catalog` (already computed
-  // via `analyze()` above) and from `appModel.stages`/`sql` (which already
-  // carry plan-tree data), both fully populated regardless. Produces
-  // identical comparison output to the dashboard's session-cache snapshots;
-  // the MCP tool just never exposes per-task drill-down, so there's nothing
-  // to prefetch into it.
+  // buildComparison's captureSnapshot uses an empty taskDataCache: that arg only feeds the
+  // interactive stage-detail drill-down, which none of compare/matchStages/metricDeltas/findingsDelta
+  // read. Findings/metrics come from `catalog` and appModel.stages/sql, both fully populated. Produces
+  // identical output to the dashboard; the MCP tool just never exposes per-task drill-down.
   const built = buildComparison(
     { label: runIdA, appModel: appModelA, catalog: catalogA },
     { label: runIdB, appModel: appModelB, catalog: catalogB },
   );
-  // Stage names throughout `built` (findings.introduced/resolved[].stages,
-  // baseStages/candStages[].name) carry raw Spark stage text, which can embed
-  // a host/IP token as free text — the same residual redactReport() already
-  // scrubs from the evidence report.
+  // Stage names throughout `built` carry raw Spark stage text, which can embed a host/IP token as
+  // free text, the same residual redactReport() already scrubs from the evidence report.
   const result = opts?.redact ? redactComparison(built) : built;
 
   return {
@@ -303,12 +335,9 @@ export async function evaluateBudgetsForRun(
   budgets               ,
   secondary         ,
 )                                                                                                {
-  // Mirrors the CLI's own --regression-metric/--max-regression-pct pairing
-  // guard (bin/sparkforensics-analyze.mjs): unlike the CLI, this tool never
-  // defaults regressionMetric on the caller's behalf, so seeing it set here
-  // unambiguously means the caller asked for a regression check and forgot
-  // the threshold — evaluateBudgets() would otherwise skip the check with no
-  // signal at all.
+  // Mirrors the CLI's --regression-metric/--max-regression-pct pairing guard: unlike the CLI, this
+  // tool never defaults regressionMetric, so seeing it set here means the caller asked for a
+  // regression check and forgot the threshold, evaluateBudgets() would otherwise skip it silently.
   if (budgets.regressionMetric !== undefined && budgets.maxRegressionPct === undefined) {
     throw mcpError('access-or-upstream-failure', 'regressionMetric requires maxRegressionPct.');
   }

package/vendor-core/model-assembler.js

@@ -9,9 +9,8 @@
                 
                     
 
-// Worker-message → appModel assembly. Shared by file-load and SHS-URL-load paths,
-// which previously carried byte-identical callback blocks. Pure model mutation:
-// analysis/render/persist stay in the caller via the onDone/onProgress/onError hooks.
+// Worker-message -> appModel assembly. Shared by the file-load and SHS-URL-load paths. Pure model
+// mutation: analysis/render/persist stay in the caller via the onDone/onProgress/onError hooks.
 export function createModelCallbacks(
   appModel          ,
   { onProgress, onDone, onError }   
@@ -28,15 +27,10 @@ export function createModelCallbacks(
       appModel.stages.set(stage.id, stage);
     },
     onSql(data         ) {
-      // Full overwrite, not a merge: safe only because `planTree` (set by
-      // onSqlPlan below, from a separate 'sqlPlan' message) is never present
-      // on a 'sql' message's data. That in turn relies on
-      // SparkListenerSQLAdaptiveExecutionUpdate always preceding
-      // SparkListenerSQLExecutionEnd for the same execution (Spark emits a
-      // re-plan mid-run, never after the query finishes); see
-      // applyAdaptiveExecutionUpdate in event-handlers.ts. A future producer
-      // of 'sql' messages that could arrive after onSqlPlan would need this
-      // to merge instead of overwrite.
+      // Full overwrite, not a merge: safe only because planTree (set by onSqlPlan from a separate
+      // 'sqlPlan' message) is never present on a 'sql' message's data. That relies on
+      // SQLAdaptiveExecutionUpdate always preceding SQLExecutionEnd (Spark re-plans mid-run, never
+      // after the query finishes). A 'sql' producer that could arrive after onSqlPlan would need to merge.
       const event = data                                       ;
       appModel.sql.set(event.id, data                );
       for (const stageId of (event.stageIds ?? [])) {
@@ -59,10 +53,8 @@ export function createModelCallbacks(
       appModel.jobs.set(job.id, job);
     },
     onRunAggregates(data         ) { appModel.runAggregates = data                 ; },
-    // Patch per-stage executorMetrics posted once before `done`: see the
-    // parser worker's SparkListenerStageExecutorMetrics handler (those events
-    // arrive after StageCompleted, so the stage message itself carried an
-    // empty map). `data` is a Map<stageId, Map<execId, metrics>>.
+    // Patch per-stage executorMetrics posted once before `done`: those events arrive after
+    // StageCompleted, so the stage message itself carried an empty map. `data` is Map<stageId, Map<execId, metrics>>.
     onStageExecutorMetrics(data         ) {
       if (!(data instanceof Map)) return;
       const metricsByStage = data                                     ;

package/vendor-core/occupancy.js

@@ -1,9 +1,6 @@
-// src/occupancy.ts
-// Occupancy-weighted wall-clock attribution (N1 redesign). Replaces the old
-// CPM pass over parentIds for per-finding impact estimation. No graph, no
-// parentIds traversal: every stage's wall-clock claim is apportioned purely
-// from its own observed window and how much it overlapped with other
-// stages.
+// Occupancy-weighted wall-clock attribution for per-finding impact: every
+// stage's wall-clock claim is apportioned purely from its own observed window
+// and how much it overlapped with other stages. No graph, no parentIds traversal.
 import { mergeIntervals } from './wall-clock.js';
 
                                  
@@ -18,9 +15,8 @@ function durationMs(s                )         {
   return (s.completedAt ?? 0) - (s.submittedAt ?? 0);
 }
 
-// Average-concurrency proxy, held constant across the stage's whole active
-// window: no per-task timestamps exist outside the parser worker to do
-// better (see the redesign spec's "Explicitly rejected" section).
+// Average-concurrency proxy, held constant across the stage's active window:
+// no per-task timestamps exist outside the parser worker to do better.
 function coreWeight(s                )         {
   const d = durationMs(s);
   return d > 0 ? (s.executorRunTime ?? 0) / d : 0;

package/vendor-core/parser-worker.js

@@ -24,29 +24,23 @@ export {
 
 export { naturalCompare, reassembleRollingEntries } from './rolling-log-reassembly.js';
 
-// 512 KB rather than a larger round number: Spark event logs commonly
-// compress ~15-20x, so a coarser chunk decompresses into a burst of tens of
-// thousands of lines reported at one unmoving pct before jumping: smaller
-// reads give the progress bar more checkpoints to advance through instead.
+// 512 KB, not a larger round number: Spark logs commonly compress ~15-20x, so a
+// coarser chunk decompresses into a burst of tens of thousands of lines at one
+// unmoving pct; smaller reads give the progress bar more checkpoints.
 const CHUNK_SIZE = 512 * 1024;
 const MIN_PROGRESS_STEPS = 100;
-// Emitting every 2000 lines was the other half of the stair-step: a highly
-// compressed chunk can decode thousands of lines in one go, so 2000 was
-// rarely a mid-chunk boundary: it just meant a handful of progress
-// messages for the whole file, each landing on a different currentPct.
+// Small emit interval: a highly compressed chunk decodes thousands of lines at
+// once, so a large interval yields only a handful of progress messages per file.
 const PROGRESS_EMIT_LINES = 300;
 
                                      
                                                      
 
-// Minimal shape streamFile/runParse/runParseFiles actually read off `file`
-// (name, size, slice(start,end).arrayBuffer()): narrower than the full DOM
-// `File` interface. A real `File` (the browser Worker path, via
-// WorkerIncomingMessage below) satisfies this structurally, but so does the
-// plain object src/cli/collect-run.ts's nodeFileFromPath builds for Node
-// (which has no DOM `File` constructor to build a real one from). Widened
-// from the plan's literal `File` to match that real second caller rather
-// than forcing a cast at its call site.
+// Minimal shape streamFile/runParse/runParseFiles read off `file` (name, size,
+// slice(start,end).arrayBuffer()): narrower than the full DOM `File`. A real
+// `File` (the browser Worker path) satisfies it structurally, but so does the
+// plain object src/cli/collect-run.ts's nodeFileFromPath builds for Node, which
+// has no DOM `File` constructor.
                    
                
                
@@ -60,11 +54,9 @@ const PROGRESS_EMIT_LINES = 300;
                                          
                                                                      
 
-// fflate's Gunzip and fzstd's Decompress are untyped vendor JS (plain
-// prototype classes, not `class` declarations), so TS can't infer a
-// construct signature for them; this local shape is just enough to type
-// the two call sites below without touching the vendored files (mirrors
-// shs-fetch.ts's identical StreamingDecoder shim).
+// fflate's Gunzip and fzstd's Decompress are untyped vendor JS, so TS can't
+// infer a construct signature for them; this local shape types the call sites
+// without touching the vendored files (mirrors shs-fetch.ts's shim).
                                                                            
                                                                                            
 
@@ -73,7 +65,7 @@ const PROGRESS_EMIT_LINES = 300;
 // it becomes available. Shared by runParse (single file) and runParseFiles
 // (rolling multi-file directories): the codec is sniffed per-file since
 // Spark's file-rolling never spans a compressor's own framing.
-async function streamFile(
+export async function streamFile(
   file            ,
   onChunk                                           ,
   chunkSize        ,
@@ -116,11 +108,10 @@ async function streamFile(
 }
 
 // Parse a dropped `File`, always streaming to honor the core invariant: the
-// decompressed task-event stream must never be buffered whole. The file is read
-// in `chunkSize` slices; a compressed log (gzip, Zstandard, Spark LZ4Block, or
-// Spark's Snappy framing) is inflated incrementally so only one decompressed
-// chunk is live at a time, exactly like the uncompressed path.
-// `chunkSize` is injectable purely so tests can force multi-chunk streaming.
+// decompressed task-event stream must never be buffered whole. Read in
+// `chunkSize` slices; a compressed log is inflated incrementally so only one
+// decompressed chunk is live at a time, like the uncompressed path.
+// `chunkSize` is injectable so tests can force multi-chunk streaming.
 export async function runParse(
   file            ,
   state             ,

package/vendor-core/plan-dot.js

@@ -1,6 +1,4 @@
-// Graphviz DOT export of a resolved planTree (sparkDoctor SqlPlanDotWriter).
-// Plain string building, no dependency, no metric annotation yet (deferred,
-// matching sparkDoctor's own unimplemented roadmap item).
+// Graphviz DOT export of a resolved planTree. No metric annotation.
                                            
 import { walkPlanTree } from './plan-tree-walk.js';
 
@@ -17,8 +15,7 @@ export function planTreeToDot(planTree                 , { title = 'plan' }
     idOf.set(node, id);
     const label = node.detail && node.detail !== node.name ? `${node.name}\\n${node.detail}` : node.name;
     lines.push(`  ${id} [label="${esc(label)}"];`);
-    // Pre-order guarantee: parent's id is already assigned by the time a
-    // child is visited, so a single walk covers both node labels and edges.
+    // Pre-order: parent's id is assigned before any child, so one walk covers labels and edges.
     if (parent) edges.push(`  ${idOf.get(parent)} -> ${id};`);
   }, { dedupe: true });
   return [...lines, ...edges, '}'].join('\n');

package/vendor-core/plan-duration-attribution.js

@@ -1,14 +1,12 @@
 // Approximate stage wall-time back to plan operators, splitting the tree at
 // Exchange boundaries into segments and zipping segments (deepest-first) to
 // submission-ordered stage IDs. No exact ground truth exists in Spark's event
-// model: this is inference (dataflint's GraphDurationAttribution approach).
+// model: this is inference.
 import { walkPlanTree } from './plan-tree-walk.js';
                                            
 
                                                                                                           
 
-function isExchange(node          )          { return /exchange/i.test(node.name); }
-
 function timingMs(node          )                {
   for (const m of (node.metrics ?? [])) {
     if (m.metricType === 'timing') return m.value;
@@ -42,7 +40,7 @@ export function computeSegments(planTree                 )
     if (!parent) {
       seg = 0;
       segmentTopology.set(seg, { parentIndex: null, depth: 0, traversalOrder: nextTraversalOrder++ });
-    } else if (isExchange(parent)) {
+    } else if (parent.exchangeRole === 'read') {
       const parentIndex = segOf.get(parent) ;
       seg = nextSegment++;
       segmentTopology.set(seg, {
@@ -116,20 +114,16 @@ function componentDistance(fromIndex        , toIndex        , segmentTopology
   return Number.POSITIVE_INFINITY;
 }
 
-// For *display* purposes only (the "Stage N" label on a segment's group box
-// in the plan graph): a segment past zipSegmentsToStages' Math.min(...)
-// cutoff has no timing data of its own to pair on, but the physical plan
-// still places it structurally near a component that DID win a stage slot
-// (most often a small broadcast-builder component immediately upstream of the
-// join component that consumes it). Inherit the structurally nearest strictly
-// paired component's stage id rather than leaving its box an unexplained
-// "Stage —". Distance is the number of parent/child edges in the component
-// tree, never numeric id distance. Neighbor search is always against the
-// ORIGINAL strict pairs, never against other already-filled components, so
-// labels don't drift through a chain of inherited-from-inherited guesses.
-// zipSegmentsToStages/attributeStageDurationToPlan are untouched by this and
-// keep their own strict pairing: duration attribution must never double-
-// count one stage's wall time across two segments.
+// Display-only: the "Stage N" label on a segment's group box. A segment past
+// zipSegmentsToStages' Math.min cutoff has no timing to pair on, but the plan
+// still places it structurally near a component that won a stage slot (often a
+// broadcast-builder just upstream of the join consuming it). Inherit the
+// nearest strictly-paired component's stage id rather than showing "Stage —".
+// Distance counts parent/child edges in the component tree, not numeric id
+// distance. Neighbor search is always against the ORIGINAL strict pairs, not
+// other filled components, so labels don't drift through inherited-from-
+// inherited guesses. Duration attribution keeps its own strict pairing and
+// must never double-count one stage's wall time across two segments.
 export function mapSegmentsToStagesForDisplay(
   segments              ,
   stagesById                      ,
@@ -156,19 +150,10 @@ export function mapSegmentsToStagesForDisplay(
   return map;
 }
 
-export function attributeStageDurationToPlan(
-  planTree                 ,
-  stagesById                      ,
-  sqlExec                         ,
+function computeExclusiveSharesForPairs(
+  pairs                                              ,
 )                        {
   const out = new Map                  ();
-  const stageIds = sqlExec?.stageIds ?? [];
-  if (!planTree || stageIds.length === 0) return out;
-
-  const { segments, segmentTopology } = computeSegments(planTree);
-  const typedStagesById = stagesById                                                               ;
-  const { pairs } = zipSegmentsToStages(segments, typedStagesById, stageIds, segmentTopology);
-
   for (const { nodes, stage } of pairs) {
     const { submittedAt, completedAt } = stage                                                ;
     const wall = Math.max(0, completedAt - submittedAt);
@@ -183,3 +168,67 @@ export function attributeStageDurationToPlan(
   }
   return out;
 }
+
+export function attributeStageDurationToPlan(
+  planTree                 ,
+  stagesById                      ,
+  sqlExec                         ,
+)                        {
+  const stageIds = sqlExec?.stageIds ?? [];
+  if (!planTree || stageIds.length === 0) return new Map();
+  const { segments, segmentTopology } = computeSegments(planTree);
+  const typedStagesById = stagesById                                                               ;
+  const { pairs } = zipSegmentsToStages(segments, typedStagesById, stageIds, segmentTopology);
+  return computeExclusiveSharesForPairs(pairs);
+}
+
+// Exclusive share plus every descendant present in the same segment's node
+// list. "Same segment" is what keeps this from leaking across an Exchange
+// boundary: computeSegments already cuts a new segment at each write half,
+// so a node's descendants outside its own segment are simply not in
+// `inSegment` and get excluded by construction.
+export function attributeStageDurationToPlanInclusive(
+  planTree                 ,
+  stagesById                      ,
+  sqlExec                         ,
+)                        {
+  const stageIds = sqlExec?.stageIds ?? [];
+  const out = new Map                  ();
+  if (!planTree || stageIds.length === 0) return out;
+  const { segments, segmentTopology } = computeSegments(planTree);
+  const typedStagesById = stagesById                                                               ;
+  const { pairs } = zipSegmentsToStages(segments, typedStagesById, stageIds, segmentTopology);
+  const exclusive = computeExclusiveSharesForPairs(pairs);
+
+  for (const { nodes } of pairs) {
+    const inSegment = new Set(nodes);
+    for (const node of nodes) {
+      // `visited` is fresh per top-level node, not shared across the `nodes`
+      // loop: a diamond-shared descendant's value, when computed while
+      // suppressed by ANOTHER node's own traversal (to avoid double-counting
+      // within THAT node's rollup), is not valid to reuse for the shared
+      // descendant's own independent top-level entry, those are different
+      // questions ("what does this contribute to node X's total" vs. "what
+      // is this node's own total in isolation"). This also rules out a
+      // node->total memo shared across top-level calls: a cached total
+      // computed unsuppressed (as its own top-level entry) would, if reused
+      // while this node is a descendant of a DIFFERENT ancestor being
+      // suppressed for double-counting, smuggle a shared descendant's share
+      // back in twice - see the diamond-shape regression test. Losing
+      // memoization across different top-level calls is fine: these trees
+      // are small and this was never a performance-critical path.
+      const visited = new Set          ();
+      function inclusiveOf(n          )         {
+        if (visited.has(n)) return 0;
+        visited.add(n);
+        let total = exclusive.get(n) ?? 0;
+        for (const child of n.children ?? []) {
+          if (inSegment.has(child)) total += inclusiveOf(child);
+        }
+        return total;
+      }
+      out.set(node, inclusiveOf(node));
+    }
+  }
+  return out;
+}