sparkforensics-mcp 0.4.0 → 0.5.0

package/vendor-core/stage-plan-nodes.js

@@ -0,0 +1,18 @@
+                                                                
+
+/** The plan nodes a stage actually ran: its SQL execution's resolved plan tree, filtered to the
+ * nodes attributed to it (`node.stageIds`). Empty when the stage has no SQL execution, the
+ * execution has no resolved plan, or no node is attributed to it. The one stage-to-plan mapping:
+ * stageIdentity's fingerprint and the Python-stage check both read it. */
+export function planNodesOfStage(stage                                      , sql                           )             {
+  const execId = stage.sqlExecutionId;
+  if (execId == null) return [];
+  const root = sql.get(execId)?.planTree ?? null;
+  if (!root) return [];
+  const nodes             = [];
+  (function collect(node          )       {
+    if (node.stageIds?.includes(stage.id)) nodes.push(node);
+    for (const child of node.children ?? []) collect(child);
+  })(root);
+  return nodes;
+}

package/vendor-core/stage-quantiles.js

@@ -56,6 +56,7 @@ const TASK_FIELD_PROPS = TASK_FIELD_DESCRIPTORS.map((d) => d.prop);
                           
                      
                       
+                               
  
 
 export function finalizeStage(
@@ -124,9 +125,11 @@ export function finalizeStage(
     acc.executorCpuTime += t.executorCpuTime;
     acc.inputBytes += t.inputBytes;
     acc.outputBytes += t.outputBytes;
+    if (t.outputRecords != null) acc.outputRecords = (acc.outputRecords ?? 0) + t.outputRecords;
     for (let i = 0; i < TASK_FIELD_PROPS.length; i++) buf.push(t[TASK_FIELD_PROPS[i]]);
   }
   stage.taskCount = taskCount;
+  stage.peakExecutionMemoryMax = peakExecutionMemoryMax;
   stage.failedTasks = failedTasks;
   stage.speculativeTasks = speculativeTasks;
   stage.taskAttempts = null; // no longer needed after finalize, freeing memory
@@ -201,6 +204,7 @@ export function finalizeStage(
   delete data.failureDetails; // internal-only intern table, summarized by failureGroups
   delete data.speculativeWinners; // internal-only late-TaskEnd pairing state, kept worker-side
   delete data.lateSpeculationWaste;
+  delete data.stageAttemptId;
 
   return { type: 'stage', data };
 }

package/vendor-core/types.js

@@ -57,6 +57,24 @@
  
                                                                                                  
 
+                                     
+                    
+                      
+                         
+                          
+                          
+                    
+                             
+                           
+                           
+                            
+                     
+                      
+                               
+                                 
+                            
+ 
+
                         
               
                 
@@ -70,6 +88,20 @@
                        
                       
                        
+                                                                                                  
+                                
+                                                                                        
+                                                                    
+                         
+                               
+                                                                                               
+                                                                                       
+                                              
+                                                                                                
+                                                                                            
+                                                                                                  
+                             
+                                              
                             
                           
                           
@@ -263,6 +295,10 @@
                          
                                       
                                                     
+                                                                               
+                        
+                                                                                
+                                     
  
 
                                                                    
@@ -272,6 +308,9 @@
                                  
                 
                      
+                                                                                         
+                                                                                    
+              
  
 
                                                                                             
@@ -288,11 +327,20 @@
                                                   
                                        
                             
+                                                                                               
+                                                                                               
+                                                                                             
+                                                                                                 
+                                                                                               
+                                                                                   
+                                                                                                                                  
+                                                                     
+                                                    
  
 
 // `Finding` is a union discriminated on `type`, one member per emitted finding type: see
 // finding-types.ts for each detector's shape and which of its fields are public evidence.
-                                                                                                                  
+                                                                                                                               
 
                                                                                      
 

package/vendor-core/wasted-core-hours.js

@@ -2,7 +2,7 @@
 // the memoryUtilization detector's idle-cores math uses: capacity core-time vs.
 // core-time that actually ran tasks. Feeds a report widget only, no detector.
 
-import { computeTotalCores } from './core-count.js';
+import { computePeakConcurrentCores } from './core-count.js';
 import { MS_PER_CORE_HOUR } from './format-utils.js';
 
 const TOP_N = 5;
@@ -30,21 +30,24 @@ const EMPTY                        = {
 
 // `app`/`runAggregates` are nullable because real callers pass null (app is
 // SparkAppInfo | null before parsing completes), which the guard below already
-// tolerates. `resources` is on app's shape so the computeTotalCores pass-through
-// type-checks; real app objects always carry it.
+// tolerates. `resources` is on app's shape so the computePeakConcurrentCores
+// pass-through type-checks; real app objects always carry it. An omitted
+// `executorsRemoved` means no executor left, where peak cores equal the sum.
 export function computeWastedCoreHours(
   app                                                                                                       ,
-  executorsAdded                                             = [],
+  executorsAdded                                                                                    = [],
   runAggregates                      ,
+  executorsRemoved                                                   = [],
 )                        {
   // Nullish (not falsy) guard on times: a literal startTime:0 is valid.
   if (!runAggregates || app?.startTime == null || app?.endTime == null) return EMPTY;
   const appDurationMs = app.endTime - app.startTime;
   if (appDurationMs <= 0) return EMPTY;
 
-  // Total cores: prefer real Executor-Added Total Cores, else peakExecutors ×
-  // configured cores (same fallback as the memoryUtilization detector).
-  const totalCores = computeTotalCores(app, executorsAdded);
+  // Total cores: peak concurrent Executor-Added Total Cores, else peak executors ×
+  // configured cores. Same capacity as the memoryUtilization detector, so an
+  // executor replaced mid-run is not counted twice.
+  const totalCores = computePeakConcurrentCores(app, executorsAdded, executorsRemoved);
   if (totalCores <= 0) return EMPTY;
 
   const totalCoreHours = (totalCores * appDurationMs) / MS_PER_CORE_HOUR;

package/vendor-core/write-targets.js

@@ -0,0 +1,312 @@
+// Write targets: every SQL write command in a run's plans, with the path or table it writes to.
+// Read by automation that halts a session when a target leaves its sandbox, so every rule here
+// fails toward "target unknown" (null), never toward a guessed or partial target:
+//   - a write node whose target cannot be parsed is still reported, with target null and its raw
+//     simpleString;
+//   - a target Spark cut short (no delimiter after it, or a "..." marker in it) is unparseable;
+//   - a write-like node outside the known list is reported as an unrecognized write;
+//   - an execution whose plan is not in the model (or whose start event could not be read) is
+//     listed, never skipped, and the count of unreadable log lines is reported.
+// Targets are verbatim from the simpleString: nothing is resolved (relative paths, ${var}
+// placeholders, catalog-relative table names all pass through as the log states them).
+import { walkPlanTree } from './plan-tree-walk.js';
+                                                         
+
+                              
+                         
+                                                                                    
+                        
+                                                                                         
+                  
+                                                                   
+                      
+     
+                                                                                               
+                                                                                                
+                                                                                                  
+                                                                                       
+     
+                                                                   
+                        
+                                                                                          
+                            
+                                                                             
+              
+ 
+
+                                       
+                         
+                                                                                                                                       
+                                       
+ 
+
+                                     
+                        
+                                                                                              
+                                                
+                                                                                                                    
+                              
+ 
+
+                                                                         
+
+// Write-like detection: a known command, any CamelCase word of the operator name in this set, or a
+// TableAsSelect name.
+const WRITE_WORDS = new Set([
+  'Write', 'Insert', 'Save', 'Overwrite', 'Append', 'Merge', 'Update', 'Delete', 'Truncate', 'Replace', 'Drop',
+  'Load', 'Vacuum', 'Convert', 'Clone', 'Restore', 'Optimize', 'Alter', 'Reorg', 'Rename', 'Call',
+]);
+// Names that contain no write word of their own: ADD/DROP/RENAME PARTITION (which can name a LOCATION).
+const WRITE_SUBSTRINGS = ['TableAsSelect', 'AddPartition', 'DropPartition', 'RenamePartition', 'RecoverPartitions'];
+// Plan nodes whose name matches a write word but which write no user data: WriteFiles is the child
+// of a write command (the command is the write), the rest are query or streaming-state operators.
+// Any join (SortMergeJoin) is excluded by name in isWriteLike.
+const NOT_WRITES = new Set([
+  'WriteFiles', 'AppendColumns', 'AppendColumnsWithObject', 'MergeRows', 'StateStoreSave',
+  'StateStoreRestore', 'SessionWindowStateStoreSave', 'SessionWindowStateStoreRestore',
+  'UpdateEventTimeWatermarkColumn',
+]);
+
+const IDENT_PART = String.raw`(?:\`(?:[^\`]|\`\`)*\`|[A-Za-z0-9_$]+)`;
+const IDENTIFIER = new RegExp(`^${IDENT_PART}(?:\\.${IDENT_PART})*$`);
+const IDENT_PARTS = new RegExp(IDENT_PART, 'g');
+const TRUNCATION_MARKER = /\.\.\.(?:\s*\d+ more fields)?/;
+
+function commandOf(nodeName        )         {
+  return nodeName.replace(/^Execute\s+/, '').trim().replace(/Exec$/, '');
+}
+
+// Only the operator name (the first word) is classified: a scan's node name goes on to print its
+// relation and table, whose identifiers can hold a write word (Scan JDBCRelation(dbo.PriceUpdate)).
+function isWriteLike(command        )          {
+  if (isKnownWrite(command)) return true;
+  const operator = command.split(/\s/, 1)[0];
+  if (NOT_WRITES.has(operator) || operator.includes('Join')) return false;
+  if (WRITE_SUBSTRINGS.some((part) => operator.includes(part))) return true;
+  return (operator.match(/[A-Z][a-z0-9]*/g) ?? []).some((word) => WRITE_WORDS.has(word));
+}
+
+                                                   
+
+// Splits at top-level commas, ignoring commas inside (), [], {} and backticks. `terminated` is
+// true when a comma follows the arg: the last arg of a cut-off string is never terminated.
+function splitArgs(args        )        {
+  const out        = [];
+  let depth = 0;
+  let quoted = false;
+  let start = 0;
+  for (let i = 0; i < args.length; i++) {
+    const c = args[i];
+    if (c === '`') quoted = !quoted;
+    else if (quoted) continue;
+    else if (c === '(' || c === '[' || c === '{') depth++;
+    else if (c === ')' || c === ']' || c === '}') depth--;
+    else if (c === ',' && depth === 0) {
+      out.push({ text: args.slice(start, i).trim(), terminated: true });
+      start = i + 1;
+    }
+  }
+  out.push({ text: args.slice(start).trim(), terminated: false });
+  return out;
+}
+
+// Text after the command name in the simpleString. Empty when the simpleString does not start
+// with the command (so nothing after it can be trusted).
+function argsOf(detail        , command        )         {
+  const m = /^(?:Execute\s+)?(\S+)\s*([\s\S]*)$/.exec(detail.trim());
+  if (!m || commandOf(m[1]) !== command) return '';
+  return m[2];
+}
+
+function isCut(text        )          {
+  return text === '' || TRUNCATION_MARKER.test(text);
+}
+
+// `db`.`t` or db.t -> db.t. A part that itself contains a dot or backtick keeps its quoting.
+function tableName(ident        )                {
+  if (!IDENTIFIER.test(ident)) return null;
+  const parts = (ident.match(IDENT_PARTS) ?? []).map((p) => {
+    if (!p.startsWith('`')) return p;
+    const inner = p.slice(1, -1).replace(/``/g, '`');
+    return /[.`]/.test(inner) ? p : inner;
+  });
+  return parts.join('.');
+}
+
+// First arg of a path-writing command. The next arg must be the boolean that follows it in
+// InsertIntoHadoopFsRelationCommand, or the static-partition map (INSERT OVERWRITE ... PARTITION
+// (dt='x')) that precedes that boolean, which Spark prints as `[dt=x]` (or `Map(dt -> x)`); this also rejects a path Spark printed with ", " inside.
+function parseHadoopFsPath(args       )                {
+  const [path, second, third] = args;
+  const flag = second && /^(Map\(.*\)|\[.*\])$/s.test(second.text) && second.terminated ? third : second;
+  if (!path?.terminated || isCut(path.text) || !flag || !/^(true|false)$/.test(flag.text)) return null;
+  return { kind: 'path', target: path.text };
+}
+
+// A quoted or plain identifier, or the `[Database: d, TableName: t, ...]` form Hive commands print.
+function parseTableArg(args       )                {
+  const first = args[0];
+  if (!first) return null;
+  const hive = /^\[Database: ([^,\]]+), TableName: ([^,\]]+)[,\]]/.exec(first.text);
+  if (hive && /\]$/.test(first.text)) {
+    const name = tableName(`${hive[1].trim()}.${hive[2].trim()}`);
+    return name && !isCut(name) ? { kind: 'table', target: name } : null;
+  }
+  if (!first.terminated || isCut(first.text)) return null;
+  const name = tableName(first.text);
+  return name ? { kind: 'table', target: name } : null;
+}
+
+// Options of `Map(k -> v, k2 -> v2)`, or null when the Map is missing or not closed.
+function parseOptionMap(text        )                             {
+  const m = /^Map\(([\s\S]*)\)$/.exec(text);
+  if (!m) return null;
+  const body = m[1];
+  const keys = [...body.matchAll(/(?:^|, )([A-Za-z0-9_.\-]+) -> /g)];
+  const options = new Map                ();
+  keys.forEach((key, i) => {
+    const valueStart = key.index  + key[0].length;
+    const valueEnd = i + 1 < keys.length ? keys[i + 1].index  : body.length;
+    options.set(key[1].toLowerCase(), body.slice(valueStart, valueEnd));
+  });
+  return options;
+}
+
+// SaveIntoDataSourceCommand: `<provider>, Map(options), <mode>`. The target is the `path` option or
+// the JDBC `dbtable`/`table` option; a redacted or missing value, or both options, is unparseable.
+function parseSaveIntoDataSource(args       )                {
+  const [, options, mode] = args;
+  if (!options?.terminated || !mode || mode.text === '') return null;
+  const map = parseOptionMap(options.text);
+  if (!map) return null;
+  const path = map.get('path');
+  const table = map.get('dbtable') ?? map.get('table');
+  if ((path === undefined) === (table === undefined)) return null;
+  const value = (path ?? table) ;
+  if (isCut(value) || /\*{5,}\(redacted\)/.test(value)) return null;
+  return path !== undefined ? { kind: 'path', target: value } : { kind: 'jdbcTable', target: value };
+}
+
+// The one target every match of `pattern` names, or null when there is none, when they name
+// different targets, or when one of them is cut: a string that also prints another plan (a
+// subquery) cannot tell which match is written. `pattern` captures the target, then its closing
+// delimiter; a match without the delimiter was cut before it.
+function soleMatch(detail        , pattern        , normalize                                 )                {
+  const targets = new Set               ();
+  for (const m of detail.matchAll(pattern)) targets.add(m[2] === undefined || isCut(m[1]) ? null : normalize(m[1]));
+  const [target] = targets;
+  return targets.size === 1 ? target : null;
+}
+
+// A V2 name states its catalog only as a leading part (catalog.namespace.table). With two parts or
+// fewer the catalog is implicit, so a database check on the name alone could match another catalog.
+function v2Table(name               )                {
+  if (!name) return null;
+  const parts = name.match(IDENT_PARTS)?.length ?? 0;
+  return { kind: parts >= 3 ? 'table' : 'unqualifiedTable', target: name };
+}
+
+// DataSource V2 writes print the connector's Write object; Iceberg's is IcebergWrite(table=t, ...).
+function parseV2Write(detail        )                {
+  return v2Table(soleMatch(detail, /\b\w*Write\(table=([^,()\s]*)([,)])?/g, tableName));
+}
+
+// V2 CTAS/RTAS: `<catalog>@<hash>, <identifier>, <query plan>, ...`. The catalog shape is checked
+// so a plan printed in a different layout is not misread.
+function parseV2TableAsSelect(args       )                {
+  const [catalog, ident] = args;
+  if (!catalog?.terminated || !/^[\w.$]+@[0-9a-f]+$/.test(catalog.text)) return null;
+  if (!ident?.terminated || isCut(ident.text)) return null;
+  return v2Table(tableName(ident.text));
+}
+
+// Delta prints a path-based table as delta.`<path>`. Only the command's own first argument is the
+// written table: later arguments print other plans (a source relation, a condition's subquery).
+function parseDeltaPath(args       )                {
+  const [first] = args;
+  const m = first?.terminated ? /^delta\.`([^`]*)`$/.exec(first.text) : null;
+  return m && !isCut(m[1]) ? { kind: 'path', target: m[1] } : null;
+}
+
+const V2_WRITES = new Set([
+  'AppendData', 'OverwriteByExpression', 'OverwritePartitionsDynamic', 'ReplaceData', 'WriteDelta',
+  'WriteToDataSourceV2',
+  // V1-fallback writers (Delta tables): their node names keep the V1 suffix, not ending in Exec.
+  'AppendDataExecV1', 'OverwriteByExpressionExecV1',
+]);
+const V2_TABLE_AS_SELECT = new Set([
+  'CreateTableAsSelect', 'AtomicCreateTableAsSelect', 'ReplaceTableAsSelect', 'AtomicReplaceTableAsSelect',
+]);
+const DELTA_COMMANDS = new Set([
+  'WriteIntoDelta', 'WriteIntoDeltaCommand', 'MergeIntoCommand', 'UpdateCommand', 'DeleteCommand',
+  'CreateDeltaTableCommand', 'OptimizeTableCommand', 'RestoreTableCommand', 'DeltaReorgTableCommand',
+]);
+
+const ARG_PARSERS                                                 = {
+  InsertIntoHadoopFsRelationCommand: parseHadoopFsPath,
+  InsertIntoHiveTable: parseTableArg,
+  CreateHiveTableAsSelectCommand: parseTableArg,
+  OptimizedCreateHiveTableAsSelectCommand: parseTableArg,
+  CreateDataSourceTableAsSelectCommand: parseTableArg,
+  SaveIntoDataSourceCommand: parseSaveIntoDataSource,
+};
+
+function isKnownWrite(command        )          {
+  return command in ARG_PARSERS || V2_WRITES.has(command) || V2_TABLE_AS_SELECT.has(command) || DELTA_COMMANDS.has(command);
+}
+
+// [recognized, parsed target or null] for one write-like node.
+function parseWrite(command        , detail        )                           {
+  const argParser = ARG_PARSERS[command];
+  if (argParser) return [true, argParser(splitArgs(argsOf(detail, command)))];
+  if (V2_WRITES.has(command)) return [true, parseV2Write(detail)];
+  if (V2_TABLE_AS_SELECT.has(command)) return [true, parseV2TableAsSelect(splitArgs(argsOf(detail, command)))];
+  // A MERGE prints its source relation too, so a delta. path in it may be the source, not the target.
+  if (command === 'MergeIntoCommand') return [true, null];
+  if (DELTA_COMMANDS.has(command)) return [true, parseDeltaPath(splitArgs(argsOf(detail, command)))];
+  return [false, null];
+}
+
+function outputRowsOf(node          )                {
+  const metric = node.metrics?.find((m) => m.name === 'number of output rows');
+  return metric !== undefined && Number.isFinite(metric.value) ? metric.value : null;
+}
+
+function collectWrites(executionId        , root          , out               )       {
+  walkPlanTree(root, (node) => {
+    const command = commandOf(node.name);
+    if (!isWriteLike(command)) return;
+    const detail = node.detail ?? '';
+    const [recognized, parsed] = parseWrite(command, detail);
+    out.push({
+      sqlExecutionId: executionId,
+      nodeId: node.id ?? null,
+      command,
+      recognized,
+      kind: parsed?.kind ?? null,
+      target: parsed?.target ?? null,
+      outputRows: outputRowsOf(node),
+      raw: detail !== '' ? detail : node.name,
+    });
+  }, { dedupe: true });
+}
+
+                                    
+                                                                                   
+                                     
+                        
+ 
+
+export function extractWriteTargets(sql                           , input                    = {})                     {
+  const writes                = [];
+  const withoutPlan = new Map                                        ();
+  for (const id of input.unreadableSqlExecutions ?? []) if (!sql.get(id)?.planTree) withoutPlan.set(id, 'unreadableStart');
+  for (const id of [...sql.keys()].sort((a, b) => a - b)) {
+    const planTree = sql.get(id) .planTree;
+    if (!planTree) withoutPlan.set(id, withoutPlan.get(id) ?? 'noPlan');
+    else collectWrites(id, planTree, writes);
+  }
+  const executionsWithoutPlan = [...withoutPlan].sort((a, b) => a[0] - b[0])
+    .map(([sqlExecutionId, reason]) => ({ sqlExecutionId, reason }));
+  return { writes, executionsWithoutPlan, skippedLines: input.skippedLines ?? null };
+}