sparkforensics-server 0.6.0 → 0.8.1

package/vendor-core/stage-quantiles.js

@@ -1,6 +1,7 @@
                                                             
 import { medianOfSorted } from './median.js';
                                                                              
+                                                  
 
 // Single source of truth for the packed per-task numeric array: FIELDS (offset
 // constants), TASK_FIELD_NAMES (display labels), and the finalizeStage hot-loop
@@ -43,9 +44,8 @@ export const TASK_FIELD_NAMES           = TASK_FIELD_DESCRIPTORS.map((d) => d.na
 const TASK_FIELD_PROPS = TASK_FIELD_DESCRIPTORS.map((d) => d.prop);
 
 // Numeric accumulator fields on `stage` that this function reads/increments.
-// `stage`'s public parameter type stays the loose Record shape per this
-// module's Produces contract; this narrow view is only for the arithmetic
-// below, never a restatement of the object's real shape.
+// `stage`'s public parameter type stays the loose Record shape; this narrow view is only for the
+// arithmetic below, never a restatement of the object's real shape.
                                    
                            
                             
@@ -82,7 +82,8 @@ export function finalizeStage(
   const localityStats = new Map();
   let peakExecutionMemoryMax = 0;
 
-  for (const t of stage.taskAttempts.values()) {
+  const attempts = stage.taskAttempts;
+  for (const t of attempts.values()) {
     taskCount++;
     if (t.failed) {
       failedTasks++;
@@ -105,7 +106,15 @@ export function finalizeStage(
     }
     if (t.executorId) {
       let es = executorStats.get(t.executorId);
-      if (!es) { es = { taskCount: 0, totalDuration: 0, inputBytes: 0, shuffleReadBytes: 0, shuffleWriteBytes: 0 }; executorStats.set(t.executorId, es); }
+      if (!es) { es = { taskCount: 0, totalDuration: 0, logDurationSum: 0, logDurationM2: 0, inputBytes: 0, shuffleReadBytes: 0, shuffleWriteBytes: 0 }; executorStats.set(t.executorId, es); }
+      // Task durations scatter multiplicatively, so the spread slowHost tests an executor against is
+      // kept on the log scale: a running sum and Welford's M2 of ln(1 + duration). A non-finite
+      // duration makes both NaN, which the detector reads as "no spread known".
+      const logDuration = Math.log1p(t.duration);
+      const logMeanBefore = es.taskCount > 0 ? es.logDurationSum / es.taskCount : 0;
+      const logMeanAfter = (es.logDurationSum + logDuration) / (es.taskCount + 1);
+      es.logDurationM2 += (logDuration - logMeanBefore) * (logDuration - logMeanAfter);
+      es.logDurationSum += logDuration;
       es.taskCount++;
       es.totalDuration += t.duration;
       es.inputBytes += t.inputBytes;
@@ -145,9 +154,9 @@ export function finalizeStage(
   const { p50: spillMemP50, p95: spillMemP95, max: spillMemMax } = computeFieldQuantiles(arr, FIELDS.MEM_SPILLED);
   const { p50: spillDiskP50, p95: spillDiskP95, max: spillDiskMax } = computeFieldQuantiles(arr, FIELDS.DISK_SPILLED);
 
-  // Straggler count: tasks with duration > 4 * P50, their summed excess over P50, and the longest
-  // task that isn't one.
-  const stragglerThreshold = 4 * p50;
+  // Straggler count: tasks with duration > STRAGGLER_FACTOR * P50, their summed excess over P50, and
+  // the longest task that isn't one.
+  const stragglerThreshold = STRAGGLER_FACTOR * p50;
   let stragglerCount = 0;
   let stragglerExcessMs = 0;
   let longestNonStragglerMs = 0;
@@ -160,6 +169,7 @@ export function finalizeStage(
     }
   }
 
+  const tailAttribution = p50 > 0 ? attributeTail(attempts.values()                                 , p50) : null;
   const peakConcurrentTasks = computePeakConcurrentTasks(arr);
   const tailReplayRecoveryMs = stragglerCount > 0
     ? computeTailReplayRecoveryMs(arr, p50, peakConcurrentTasks)
@@ -169,7 +179,7 @@ export function finalizeStage(
     ([host, s]) => ({ host, taskCount: s.taskCount, totalDuration: s.totalDuration })
   );
   const executorStatsArr = [...executorStats.entries()].map(
-    ([executorId, s]) => ({ executorId, taskCount: s.taskCount, totalDuration: s.totalDuration, inputBytes: s.inputBytes, shuffleReadBytes: s.shuffleReadBytes, shuffleWriteBytes: s.shuffleWriteBytes })
+    ([executorId, s]) => ({ executorId, taskCount: s.taskCount, totalDuration: s.totalDuration, logDurationSum: s.logDurationSum, logDurationM2: s.logDurationM2, inputBytes: s.inputBytes, shuffleReadBytes: s.shuffleReadBytes, shuffleWriteBytes: s.shuffleWriteBytes })
   );
   const failureReasonsArr = [...failureReasons.entries()].map(
     ([reason, count]) => ({ reason, count })
@@ -191,6 +201,7 @@ export function finalizeStage(
     peakExecutionMemoryMax,
     taskActiveMs: computeTaskActiveMs(arr),
     peakConcurrentTasks,
+    ...(tailAttribution ? { tailAttribution } : {}),
     taskDurationP50: p50,
     taskDurationP95: p95,
     taskDurationMax: max,
@@ -210,6 +221,127 @@ export function finalizeStage(
   return { type: 'stage', data };
 }
 
+// A task is in the tail when it runs over this many times the stage's median task: skew's own
+// P95/median ratio threshold, so every stage either detector flags on duration has a tail here.
+export const TAIL_FACTOR = 3;
+
+// A task is a straggler when it runs over this many times the stage's median task.
+export const STRAGGLER_FACTOR = 4;
+
+// The per-task fields attributeTail reads; a TaskRecord in event-handlers.ts has all of them.
+                    
+                  
+                   
+               
+                 
+                        
+                          
+                          
+                     
+                       
+                      
+                             
+ 
+
+// A hosts-concentration reading needs this many tail tasks on the host, at least this share of the
+// tail, and at least this multiple of the host's share of all tasks, before the host explains the
+// excess the other causes leave. One slow task on a host that ran a tenth of the stage is chance.
+const TAIL_HOST_MIN_TASKS = 3;
+const TAIL_HOST_MIN_TAIL_SHARE = 0.5;
+const TAIL_HOST_MIN_OVERREPRESENTATION = 2;
+
+// A median task that read next to nothing makes every ratio against it huge (a median of 4 bytes
+// against a 1 MB task reads as 250000x). Below these floors the median task is read as having
+// read almost nothing: there is no ratio to take, and a task counts as having read data only from
+// the floors up.
+const MIN_VOLUME_BYTES = 1024 * 1024;
+const MIN_VOLUME_RECORDS = 1000;
+
+// Why the stage's slow tasks (non-failed, over TAIL_FACTOR x P50) were slow, from what each task
+// logged. A task's excess is its duration over P50. Four causes claim it, in this order so no
+// millisecond counts twice:
+//   1. Data: had run time scaled with data volume, the task would have taken P50 x (volume ratio)
+//      against the median task, with volume the task's input plus shuffle-read, in bytes or in
+//      records, whichever is further above the median. This is the definition Spark's UI and AQE
+//      use for skew (a partition far larger than the median one), applied to run time.
+//   2. GC time over the median task's.
+//   3. Shuffle fetch wait over the median task's.
+//   4. Host: what is left of the tail tasks on the one host that holds most of the tail.
+// With a median task that read almost nothing there is no ratio to take, so GC and fetch wait claim their
+// excess first and a task that read data owns what they leave: the time was GC or waiting unless
+// the data is all that is left to explain it.
+// Null when the stage has no tail. `tasks` is consumed once.
+export function attributeTail(tasks                    , p50        )                         {
+  const all             = [];
+  const tail             = [];
+  for (const t of tasks) {
+    if (t.failed) continue; // a failed attempt's metrics stop where it died
+    all.push(t);
+    if (t.duration > TAIL_FACTOR * p50) tail.push(t);
+  }
+  if (tail.length === 0) return null;
+
+  const bytesOf = (t          ) => t.inputBytes + t.shuffleRead;
+  const recordsOf = (t          ) => t.inputRecords + t.shuffleReadRecords;
+  const medianBytes = medianOf(all, bytesOf);
+  const medianRecords = medianOf(all, recordsOf);
+  const medianGc = medianOf(all, (t) => t.gcTime);
+  const medianFetch = medianOf(all, (t) => t.fetchWaitTime);
+  const bytesComparable = medianBytes >= MIN_VOLUME_BYTES;
+  const recordsComparable = medianRecords >= MIN_VOLUME_RECORDS;
+  const hasVolume = bytesComparable || recordsComparable;
+  const volumeRatio = (t          )         => Math.max(
+    bytesComparable ? bytesOf(t) / medianBytes : 0,
+    recordsComparable ? recordsOf(t) / medianRecords : 0,
+  );
+  const readsData = (t          ) => bytesOf(t) >= MIN_VOLUME_BYTES || recordsOf(t) >= MIN_VOLUME_RECORDS;
+
+  const tailHosts = new Map                ();
+  const allHosts = new Map                ();
+  for (const t of all) if (t.host) allHosts.set(t.host, (allHosts.get(t.host) ?? 0) + 1);
+  for (const t of tail) if (t.host) tailHosts.set(t.host, (tailHosts.get(t.host) ?? 0) + 1);
+  let topHost                = null;
+  for (const [host, n] of tailHosts) if (topHost === null || n > tailHosts.get(topHost) ) topHost = host;
+  const hostConcentrated = topHost !== null
+    && tailHosts.get(topHost)  >= TAIL_HOST_MIN_TASKS
+    && tailHosts.get(topHost)  / tail.length >= TAIL_HOST_MIN_TAIL_SHARE
+    && (tailHosts.get(topHost)  / tail.length) / (allHosts.get(topHost)  / all.length) >= TAIL_HOST_MIN_OVERREPRESENTATION;
+
+  let excessMs = 0, dataMs = 0, gcMs = 0, fetchWaitMs = 0, hostMs = 0, cpuNs = 0, runMs = 0;
+  const ratios           = [];
+  for (const t of tail) {
+    const excess = t.duration - p50;
+    excessMs += excess;
+    let left = excess;
+    if (hasVolume) {
+      const ratio = volumeRatio(t);
+      ratios.push(ratio);
+      const data = Math.min(left, Math.max(0, p50 * (ratio - 1)));
+      dataMs += data; left -= data;
+    }
+    const gc = Math.min(left, Math.max(0, t.gcTime - medianGc));
+    gcMs += gc; left -= gc;
+    const fetch = Math.min(left, Math.max(0, t.fetchWaitTime - medianFetch));
+    fetchWaitMs += fetch; left -= fetch;
+    if (!hasVolume && readsData(t)) { dataMs += left; left = 0; }
+    if (hostConcentrated && t.host === topHost) { hostMs += left; }
+    cpuNs += t.executorCpuTime;
+    runMs += t.executorRunTime;
+  }
+  ratios.sort((a, b) => a - b);
+  return {
+    tasks: tail.length, excessMs, dataMs, gcMs, fetchWaitMs, hostMs,
+    host: hostConcentrated ? topHost : null,
+    hostTasks: hostConcentrated ? tailHosts.get(topHost )  : 0,
+    dataRatio: hasVolume ? medianOfSorted(Float64Array.from(ratios)) : null,
+    cpuPct: cpuNs > 0 && runMs > 0 ? Math.min(100, (cpuNs / 1e6 / runMs) * 100) : null,
+  };
+}
+
+function medianOf(tasks            , valueOf                         )         {
+  return medianOfSorted(Float64Array.from(tasks, valueOf).sort());
+}
+
 // Wall-clock time during which at least one of the stage's tasks was running: the union of its
 // [launch, finish) intervals. A stage's submittedAt..completedAt window also covers time it sat
 // open with no task running (waiting for a free slot, or between retried tasks), which no

package/vendor-core/stage-structure.js

@@ -3,7 +3,7 @@
 // removed) and the token sets its similarity scores read. Pure.
 
 import { cyrb53 } from './string-hash.js';
-import { normalizeStageName,                       } from './stage-identity.js';
+import { normalizeStageName, normalizedDetailOf,                       } from './stage-identity.js';
                                            
 
 // Plumbing nodes: codegen boundaries and input adapters change with how Spark fuses a stage, not with
@@ -43,7 +43,7 @@ export function structuralKey(nodes                     )                {
   const digestOf = (node          )         => {
     let d = digests.get(node);
     if (d === undefined) {
-      d = cyrb53(JSON.stringify([normalizeStageName(node.name ?? ''), attributeNames(node.detail ?? ''), below(node).sort()]));
+      d = cyrb53(JSON.stringify([normalizeStageName(node.name ?? ''), attributeNamesOf(node), below(node).sort()]));
       digests.set(node, d);
     }
     return d;
@@ -53,6 +53,20 @@ export function structuralKey(nodes                     )                {
   return cyrb53(JSON.stringify(roots));
 }
 
+// A node's attribute names are read by the structural key of every stage it ran in and by the
+// execution's plan structure; a node is never mutated after the parser posts it.
+const attributeNamesCache = new WeakMap                    ();
+
+/** `attributeNames` of a plan node's detail, memoized per node. The array is shared: do not change it. */
+export function attributeNamesOf(node          )                    {
+  let names = attributeNamesCache.get(node);
+  if (names === undefined) {
+    names = attributeNames(node.detail ?? '');
+    attributeNamesCache.set(node, names);
+  }
+  return names;
+}
+
 const TOKEN_SPLIT = /[^A-Za-z0-9_.$]+/;
 
 /** Sorted distinct tokens of a stage's attributed nodes (name and normalized detail) and its name.
@@ -64,7 +78,7 @@ export function detailTokens(stageName                    , nodes
   for (const node of nodes) {
     if (isPlumbing(node)) continue;
     add(normalizeStageName(node.name ?? ''));
-    add(normalize(node.detail ?? ''));
+    add(normalizedDetailOf(node, normalize));
   }
   return [...tokens].sort();
 }

package/vendor-core/threshold-overrides.js

@@ -88,11 +88,17 @@ export function tunedThresholdsOf(entry          , overrides
   return Object.keys(tuned).length > 0 ? tuned : null;
 }
 
-/** The tuned thresholds an entry's findings carry: its own, plus its `suppressedBy` entry's
- * (named `<suppressor>.<threshold>`), since tuning the suppressor changes which of them survive. */
+// straggler judges a tail with skew's resolved thresholds (DetectorCtx.skewThresholds), so tuning skew
+// changes which tails it reports without any `suppressedBy` link.
+const THRESHOLD_DEPENDENCY                                   = { straggler: 'skew' };
+
+/** The tuned thresholds an entry's findings carry: its own, plus its `suppressedBy` entry's or the
+ * entry it reads thresholds from (named `<entry>.<threshold>`), since tuning either changes which
+ * of them survive. */
 export function findingTunedThresholds(entry          , overrides                                )                         {
   const own = tunedThresholdsOf(entry, overrides);
-  const suppressor = entry.suppressedBy ? entries.find((d) => d.type === entry.suppressedBy) : undefined;
+  const suppressorType = entry.suppressedBy ?? THRESHOLD_DEPENDENCY[entry.type];
+  const suppressor = suppressorType ? entries.find((d) => d.type === suppressorType) : undefined;
   const bySuppressor = suppressor ? tunedThresholdsOf(suppressor, overrides) : null;
   if (!suppressor || !bySuppressor) return own;
   const prefixed = Object.fromEntries(Object.entries(bySuppressor).map(([name, t]) => [`${suppressor.type}.${name}`, t]));

package/vendor-core/types.js

@@ -43,8 +43,7 @@
                                                                              
 // Contract-pinned evidence `eventType`: the same fixed set as the parser's
 // `evidenceInputs` counter keys, so provenance stays addressable against the
-// documented `{ eventType, count }` shape (review: values must not diverge
-// from the counters they summarize).
+// documented `{ eventType, count }` shape.
                                
                                                                                          
                                                                                 
@@ -88,10 +87,10 @@
                        
                       
                        
-                                                                                                  
+                                                                                                                         
                                 
                                                                                         
-                                                                    
+               
                          
                                
                                                                                                
@@ -120,7 +119,7 @@
                            
                                   
                                                                                       
-                                                                           
+                          
                         
                                                                                         
                              
@@ -131,20 +130,51 @@
                                 
                                                                                             
                                
+                                                                                                
+                     
+                                    
                                                         
                    
                          
+ 
+
+// The tail of a stage: its non-failed tasks over TAIL_FACTOR x P50. Each cause is the summed time
+// (ms) of the tail's excess over P50 that the cause accounts for, taken in the order data, GC, fetch
+// wait, host so that no millisecond counts twice; what none of them accounts for is
+// excessMs - dataMs - gcMs - fetchWaitMs - hostMs.
+                                  
+                
+                   
+                                                                                                
+                                                                               
+                 
+               
+                      
+                                                                                                
+                                                                                                 
+                 
+                      
+                                                         
+                    
+                                                                                                    
+                                                                                                 
+                                                                      
+                           
+                                                                                                      
+                        
  
 
                            
                                                                              
-                                                                            
-                                                                        
-                                                                           
+                                                                               
+                                                                     
               
                
                   
-                                                                   
+                                                                                                
+                                                                                              
+                                                                                        
+                                                                                        
                        
                       
                                                                     
@@ -161,6 +191,8 @@
                            
                                                                                                         
                             
+                                                                                                                            
+                                           
                          
  
                                     
@@ -179,19 +211,14 @@
                                     
                                
                                                                            
-                                                                 
-                                                             
-                                                                          
-                                                                          
-                                                                        
-                                                                        
-                                                                      
-                                                                           
-                                                                           
-                                                                
+                                                                            
+                                                                       
+                                                                               
+                                                                                
+                                                                                
                                          
                                                                             
-                                                            
+                                    
                        
                                                                           
                                                                           
@@ -289,6 +316,9 @@
                       
                                
                                                                                  
+                                                                                               
+                                                                                            
+                                                               
                          
  
 
@@ -316,6 +346,9 @@
                                                                                          
                                                                                     
               
+                                                                                       
+                                               
+                
  
 
                                                                                             
@@ -332,6 +365,10 @@
                                                   
                                        
                             
+                                                                   
+                                                                                               
+                                                                                               
+                       
                                                                                                
                                                                                                
                                                                                              
@@ -363,7 +400,7 @@
                 
                  
                          
-                                                                             
+                                                                         
                                                                          
                                                                            
                    

package/vendor-core/wasted-core-hours.js

@@ -40,7 +40,6 @@ export function computeWastedCoreHours(
   runAggregates                      ,
   executorsRemoved                                                   = [],
 )                        {
-  // Nullish (not falsy) guard on times: a literal startTime:0 is valid.
   if (!runAggregates || app?.startTime == null || app?.endTime == null) return EMPTY;
   const appDurationMs = app.endTime - app.startTime;
   if (appDurationMs <= 0) return EMPTY;

package/vendor-core/write-targets.js

@@ -1,21 +1,14 @@
 // Write targets: every SQL write command in a run's plans, with the path or table it writes to.
 // Read by automation that halts a session when a target leaves its sandbox, so every rule here
-// fails toward "target unknown" (null), never toward a guessed or partial target:
-//   - a write node whose target cannot be parsed is still reported, with target null and its raw
-//     simpleString;
-//   - a target Spark cut short (no delimiter after it, or a "..." marker in it) is unparseable;
-//   - a write-like node outside the known list is reported as an unrecognized write;
-//   - an execution whose plan is not in the model (or whose start event could not be read) is
-//     listed, never skipped, and the count of unreadable log lines is reported;
-//   - a Delta target is set only when exactly one table or path is named (delta-targets.ts,
-//     api-merge-writes.ts).
-// Targets are verbatim from the simpleString: nothing is resolved (relative paths, ${var}
-// placeholders, catalog-relative table names all pass through as the log states them).
+// fails toward "target unknown" (null), never toward a guessed or partial target, and targets are
+// verbatim from the simpleString. Rules and fields: docs-site/user-guide/getting-started/ci-and-automation.md,
+// "Write targets" (a node whose target cannot be parsed, a cut-short target, an unrecognized write,
+// an unreadable execution, a Delta target named by exactly one table or path).
 import { walkPlanTree } from './plan-tree-walk.js';
 import { apiMergeWrites } from './api-merge-writes.js';
 import { indexByRoot, isStagedDeltaWrite, parseDeltaTableV2, resolveDeltaCommandTarget,                       } from './delta-targets.js';
 import { IDENT_PARTS, isCut, splitArgs, tableName,          } from './write-target-args.js';
-                                                         
+                                                                         
 
 /** The row counts a MergeIntoCommand node reports in its SQL metrics. */
                             
@@ -42,8 +35,25 @@ import { IDENT_PARTS, isCut, splitArgs, tableName,          } from './write-targ
      
                                                                    
                         
-                                                                                          
+     
+                                                                                                 
+                                                                                                  
+                                                                                                  
+                                                                                         
+     
                             
+     
+                                                                                                     
+                                                                                                     
+                
+     
+                                                                       
+     
+                                                                                                 
+                                                                                                     
+                                                                                                        
+     
+                             
                                                                                            
                                                                                                   
                                                     
@@ -216,6 +226,14 @@ const DELTA_COMMANDS = new Set([
   'CreateDeltaTableCommand', 'OptimizeTableCommand', 'RestoreTableCommand', 'DeltaReorgTableCommand',
 ]);
 
+// Row-level and maintenance commands: a stage sum would misstate their output (their figures come
+// from the command node's own counters, see commandOutputOf). A plain write is any other recognized
+// write command.
+const NOT_PLAIN_WRITES = new Set([
+  'MergeIntoCommand', 'UpdateCommand', 'DeleteCommand', 'DeltaMerge', 'ReplaceData', 'WriteDelta',
+  'CreateDeltaTableCommand', 'OptimizeTableCommand', 'RestoreTableCommand', 'DeltaReorgTableCommand',
+]);
+
 const ARG_PARSERS                                                 = {
   InsertIntoHadoopFsRelationCommand: parseHadoopFsPath,
   InsertIntoHiveTable: parseTableArg,
@@ -225,6 +243,11 @@ const ARG_PARSERS                                                 = {
   SaveIntoDataSourceCommand: parseSaveIntoDataSource,
 };
 
+/** True for a recognized command that writes its output rows (not MERGE, UPDATE, DELETE or table maintenance). */
+export function isPlainWrite(write                                             )          {
+  return write.recognized && !NOT_PLAIN_WRITES.has(write.command);
+}
+
 function isKnownWrite(command        )          {
   return command in ARG_PARSERS || V2_WRITES.has(command) || V2_TABLE_AS_SELECT.has(command) || DELTA_COMMANDS.has(command);
 }
@@ -254,6 +277,35 @@ function metricValue(node          , ...names          )                {
   return metric !== undefined && Number.isFinite(metric.value) ? metric.value : null;
 }
 
+// Delta 4.x labels the DELETE copied-rows counter with a trailing period ("number of rows copied.");
+// metricValue matches names exactly, so the counter lists both spellings.
+const COPIED_ROWS_LABELS = ['number of rows copied.', 'number of rows copied'];
+
+// Rows and bytes a MERGE, UPDATE or DELETE wrote to new files, from the command node's counters.
+// A MERGE writes its inserted, updated and copied rows; an UPDATE writes the rows it touched
+// (updated plus copied); a DELETE writes the target rows it copied. Rows removed by a DELETE or
+// MERGE delete clause are not written, so they are not output rows.
+function commandOutputOf(command        , node          )                                                {
+  if (command === 'MergeIntoCommand') {
+    const parts = [
+      metricValue(node, 'number of inserted rows'),
+      metricValue(node, 'number of updated rows'),
+      metricValue(node, 'number of target rows rewritten unmodified', 'number of rows copied'),
+    ];
+    return {
+      rows: parts.some((v) => v === null) ? null : parts.reduce((sum, v) => sum  + v , 0),
+      bytes: metricValue(node, 'number of target bytes added'),
+    };
+  }
+  if (command === 'UpdateCommand') {
+    return { rows: metricValue(node, 'number of rows touched (copied + updated)'), bytes: metricValue(node, 'number of bytes added') };
+  }
+  if (command === 'DeleteCommand') {
+    return { rows: metricValue(node, ...COPIED_ROWS_LABELS), bytes: metricValue(node, 'number of bytes added') };
+  }
+  return { rows: null, bytes: null };
+}
+
 function mergeRowsOf(command        , node          )                   {
   if (command !== 'MergeIntoCommand') return null;
   const rows = {
@@ -273,6 +325,9 @@ function collectWrites(exec              , root          , byRoot
     const [recognized, parsedFromDetail] = parseWrite(command, detail);
     // A Delta command's simpleString is bare on real logs: its target is found elsewhere.
     const parsed = parsedFromDetail ?? resolveDeltaCommandTarget(command, exec, node, byRoot);
+    const nodeRows = outputRowsOf(node);
+    const commandOutput = commandOutputOf(command, node);
+    const outputRows = nodeRows ?? commandOutput.rows;
     out.push({
       sqlExecutionId: exec.id,
       nodeId: node.id ?? null,
@@ -280,7 +335,9 @@ function collectWrites(exec              , root          , byRoot
       recognized,
       kind: parsed?.kind ?? null,
       target: parsed?.target ?? null,
-      outputRows: outputRowsOf(node),
+      outputRows,
+      outputRowsSource: nodeRows !== null ? 'nodeMetric' : outputRows !== null ? 'commandMetric' : null,
+      outputBytes: commandOutput.bytes,
       mergeRows: mergeRowsOf(command, node),
       raw: detail !== '' ? detail : node.name,
     });
@@ -301,7 +358,49 @@ function attributeStagedWrites(writes               , sql
   }
 }
 
+// The rows and bytes a plain write stored are reported by the stages of the executions under its
+// root: a Delta DataFrame write's node sits in the root execution while its stages run in a child.
+// Applied only when the root holds one logical write, so the stages cannot belong to another write.
+function attachStageOutput(writes               , sql                           , stages                             )       {
+  const byRoot = indexByRoot(sql);
+  const stagesByExecution = new Map                 ();
+  for (const stage of stages.values()) {
+    if (stage.sqlExecutionId == null) continue;
+    const list = stagesByExecution.get(stage.sqlExecutionId);
+    if (list) list.push(stage);
+    else stagesByExecution.set(stage.sqlExecutionId, [stage]);
+  }
+  const rootOf = (write             ) => sql.get(write.sqlExecutionId)?.rootExecutionId ?? write.sqlExecutionId;
+  // One logical write per root: the plain writes that name the same table or path count once (a
+  // CTAS or RTAS node and the append it runs, Iceberg's replace and its overwrite). A write with no
+  // target, a MERGE, UPDATE or DELETE, or a second target in the root makes the stages ambiguous.
+  const identitiesPerRoot = new Map                     ();
+  writes.forEach((write, i) => {
+    const key = isPlainWrite(write) && write.target !== null ? `${write.kind}:${write.target}` : `#${i}`;
+    const root = rootOf(write);
+    const keys = identitiesPerRoot.get(root);
+    if (keys) keys.add(key);
+    else identitiesPerRoot.set(root, new Set([key]));
+  });
+  for (const write of writes) {
+    const root = rootOf(write);
+    if (!isPlainWrite(write) || identitiesPerRoot.get(root)?.size !== 1) continue;
+    const group = [root, ...(byRoot.get(root) ?? []).map((exec) => exec.id)];
+    const grouped = group.flatMap((id) => stagesByExecution.get(id) ?? []);
+    if (grouped.length === 0) continue;
+    const withBytes = grouped.filter((stage) => stage.outputBytes != null);
+    if (withBytes.length > 0) write.outputBytes = withBytes.reduce((sum, stage) => sum + stage.outputBytes , 0);
+    if (write.outputRows !== null) continue;
+    const reported = grouped.filter((stage) => stage.outputRecords != null);
+    if (reported.length === 0) continue;
+    write.outputRows = reported.reduce((sum, stage) => sum + stage.outputRecords , 0);
+    write.outputRowsSource = 'stageSum';
+  }
+}
+
                                     
+                                                                                                                            
+                                       
                                                                                    
                                      
                         
@@ -320,6 +419,7 @@ export function extractWriteTargets(sql                           , input
   // DeltaTable API merges run no command node: they are found by their executions' descriptions.
   attributeStagedWrites(writes, sql);
   writes.push(...apiMergeWrites(sql));
+  if (input.stages) attachStageOutput(writes, sql, input.stages);
   writes.sort((a, b) => a.sqlExecutionId - b.sqlExecutionId);
   const executionsWithoutPlan = [...withoutPlan].sort((a, b) => a[0] - b[0])
     .map(([sqlExecutionId, reason]) => ({ sqlExecutionId, reason }));

package/vendor-core/zstd-worker-client.js

@@ -41,7 +41,7 @@ export const MAX_IN_FLIGHT_SLICES = 3;
 // `spawn` starts the decompress worker; it runs at most once, on the first zstd stream, and the
 // worker then serves every later stream (the files of a rolling log) one at a time. When it
 // cannot start (no nested workers, a blocked script, a crash), each stream is decoded by
-// `fallback` on the calling thread instead, the path used before this worker existed.
+// `fallback` on the calling thread instead.
 export function createWorkerZstdDecoders(
   spawn                      ,
   fallback                    ,