sparkforensics-server 0.4.0 → 0.5.1

package/vendor-core/comparison-verdict.js

@@ -37,7 +37,7 @@ import { NEUTRAL_METRIC_KEYS,                        } from './run-comparison.js
                       
  
 
-/** A change under this share of run A's value reads as "about the same", for
+/** A change under this share of the baseline's value reads as "about the same", for
  * run time and cost metrics alike: run-to-run noise on a shared cluster easily
  * moves a job a percent or two. */
 export const SAME_CHANGE_SHARE = 0.02;
@@ -78,14 +78,14 @@ function namesWhere(net                     , keep                             )
   return [...net].filter(([, change]) => keep(change)).map(([name]) => name);
 }
 
-/** "Run B had 2 of 5 jobs fail", or, when every job failed, "Run B's only
- * job failed" / "All 3 of run B's jobs failed" rather than "1 of 1 jobs". */
-function jobsFailed(run           , { failedJobs, totalJobs }                   )         {
-  if (failedJobs < totalJobs) return `Run ${run} had ${failedJobs} of ${totalJobs} jobs fail`;
-  return totalJobs === 1 ? `Run ${run}'s only job failed` : `All ${totalJobs} of run ${run}'s jobs failed`;
+/** "The candidate had 2 of 5 jobs fail", or, when every job failed, "The candidate's only
+ * job failed" / "All 3 of the candidate's jobs failed" rather than "1 of 1 jobs". */
+function jobsFailed(run                          , { failedJobs, totalJobs }                   )         {
+  if (failedJobs < totalJobs) return `The ${run} had ${failedJobs} of ${totalJobs} jobs fail`;
+  return totalJobs === 1 ? `The ${run}'s only job failed` : `All ${totalJobs} of the ${run}'s jobs failed`;
 }
 
-/** Run A's failures in the parenthesis after run B's. */
+/** The baseline's failures in the parenthesis after the candidate's. */
 function baselineFailures({ failedJobs, totalJobs }                   )         {
   if (failedJobs === 0) return 'none';
   if (failedJobs < totalJobs) return `${failedJobs} of ${totalJobs}`;
@@ -93,18 +93,18 @@ function baselineFailures({ failedJobs, totalJobs }                   )
 }
 
 /** The headline when either run had failed jobs, as the run verdict leads
- * with a failure: a faster run B that dropped work is not an improvement,
+ * with a failure: a faster candidate that dropped work is not an improvement,
  * and with equal failure counts the tone stays neutral since a failing run
  * that ends sooner may just have failed earlier. Null when both runs
  * completed. */
 function failureHeadline(base                   , cand                   )                                                 {
   if (base.failedJobs === 0 && cand.failedJobs === 0) return null;
   const tone = cand.failedJobs > base.failedJobs ? 'worse' : cand.failedJobs < base.failedJobs ? 'better' : 'same';
-  if (cand.failedJobs === 0) return { title: `${jobsFailed('A', base)}; run B completed`, tone };
-  return { title: `${jobsFailed('B', cand)} (run A: ${baselineFailures(base)})`, tone };
+  if (cand.failedJobs === 0) return { title: `${jobsFailed('baseline', base)}; the candidate completed`, tone };
+  return { title: `${jobsFailed('candidate', cand)} (baseline: ${baselineFailures(base)})`, tone };
 }
 
-/** One plain answer to "did run B get better or worse than run A", from the
+/** One plain answer to "did the candidate get better or worse than the baseline", from the
  * comparison's own whole-run metrics and finding-category tallies: run time
  * first, then which cost metrics moved each way past run-to-run noise, then which finding
  * categories appeared or went away. When either run had failed jobs, that
@@ -119,25 +119,25 @@ export function summarizeComparison(
   jobs                                                                ,
 )                        {
   const wall = metrics.find((metric) => metric.key === 'wallClock');
-  let title = 'Run time could not be compared between run A and run B';
+  let title = 'Run time could not be compared between the baseline and the candidate';
   let tone                 = 'unknown';
   // A log with no end-of-run record stops where the run was cut off, so its
   // shorter time is not a speed-up: say how much each log covers, neutrally.
-  const incompleteRuns = jobs ? (['A', 'B']         ).filter((run) => (run === 'A' ? jobs.baseline : jobs.candidate).incomplete) : [];
+  const incompleteRuns = jobs ? (['baseline', 'candidate']         ).filter((run) => jobs[run].incomplete) : [];
   if (wall && wall.baseline != null && wall.baseline > 0 && wall.candidate != null && incompleteRuns.length > 0) {
     const change = wall.candidate - wall.baseline;
     title = Math.abs(change / wall.baseline) < SAME_CHANGE_SHARE
-      ? "Run B's log covers about as much run time as run A's"
-      : `Run B's log covers ${formatDuration(Math.abs(change))} ${change < 0 ? 'less' : 'more'} run time than run A's`;
+      ? "The candidate's log covers about as much run time as the baseline's"
+      : `The candidate's log covers ${formatDuration(Math.abs(change))} ${change < 0 ? 'less' : 'more'} run time than the baseline's`;
   } else if (wall && wall.baseline != null && wall.baseline > 0 && wall.candidate != null) {
     const change = wall.candidate - wall.baseline;
     const share = change / wall.baseline;
     if (Math.abs(share) < SAME_CHANGE_SHARE) {
-      title = 'Run B took about as long as run A';
+      title = 'The candidate took about as long as the baseline';
       tone = 'same';
     } else {
       const faster = change < 0;
-      title = `Run B finished ${formatDuration(Math.abs(change))} ${faster ? 'faster' : 'slower'} than run A (${Math.round(Math.abs(share) * 100)}%)`;
+      title = `The candidate finished ${formatDuration(Math.abs(change))} ${faster ? 'faster' : 'slower'} than the baseline (${Math.round(Math.abs(share) * 100)}%)`;
       tone = faster ? 'better' : 'worse';
     }
   }
@@ -156,17 +156,17 @@ export function summarizeComparison(
   if (incompleteRuns.length === 2) {
     sentences.push('Neither log has an end-of-run record, so their times cover only what each log captured, not how long the runs took.');
   } else if (incompleteRuns.length === 1) {
-    sentences.push(`Run ${incompleteRuns[0]}'s log has no end-of-run record, so its time covers only what the log captured, not how long the run took.`);
+    sentences.push(`The ${incompleteRuns[0]}'s log has no end-of-run record, so its time covers only what the log captured, not how long the run took.`);
   }
-  if (worse.length > 0) sentences.push(`Worse in run B: ${worse.join(', ')}.`);
-  if (better.length > 0) sentences.push(`Better in run B: ${better.join(', ')}.`);
+  if (worse.length > 0) sentences.push(`Worse in the candidate: ${worse.join(', ')}.`);
+  if (better.length > 0) sentences.push(`Better in the candidate: ${better.join(', ')}.`);
   if (cost.length > 0 && worse.length === 0 && better.length === 0) sentences.push('Other measured cost metrics look about the same.');
 
   const net = netByCategory(findings);
   const introduced = namesWhere(net, (change) => change > 0);
   const resolved = namesWhere(net, (change) => change < 0);
-  if (introduced.length > 0) sentences.push(`New or more frequent in run B: ${introduced.join(', ')}.`);
-  if (resolved.length > 0) sentences.push(`Less frequent in run B: ${resolved.join(', ')}.`);
+  if (introduced.length > 0) sentences.push(`New or more frequent in the candidate: ${introduced.join(', ')}.`);
+  if (resolved.length > 0) sentences.push(`Less frequent in the candidate: ${resolved.join(', ')}.`);
   return { title, tone, sentences };
 }
 

package/vendor-core/core-source-hash.txt

@@ -1 +1 @@
-b5b0e0b6178129de4de482d4b59fbf881822655d8911b374c4a60a33b38f63fb
+6e0ac29f74ae092b14c5700dfa86d3273fce7d1af59005c6e2e450a91da92d5e

package/vendor-core/delta-commands.js

@@ -0,0 +1,13 @@
+// Delta write commands whose root node is a bare name in sparkPlanInfo on real logs. The parser keeps
+// their `Arguments:` line (event-handlers.ts) and delta-targets.ts reads their target from it; the
+// handler table there is typed by this list, so a command added here must get a handler.
+export const DELTA_COMMANDS = [
+  'MergeIntoCommand', 'UpdateCommand', 'DeleteCommand', 'WriteIntoDelta', 'WriteIntoDeltaCommand',
+  'SaveIntoDataSourceCommand',
+]         ;
+
+                                                           
+
+export function isDeltaCommand(name        )                       {
+  return (DELTA_COMMANDS                     ).includes(name);
+}

package/vendor-core/delta-log-paths.js

@@ -0,0 +1,42 @@
+// The `<table path>/_delta_log` paths that plans name: the only trace of which Delta table a command
+// or API merge touched when its plan carries no target. A scan may read a source too, so callers
+// take a path only when exactly one is named.
+import { walkPlanTree } from './plan-tree-walk.js';
+import { isCut,                  } from './write-target-args.js';
+                                               
+
+// A URI or absolute path before `/_delta_log`, as printed in `... State #0 - <path>/_delta_log[cols]`
+// scans and `DeltaLogFileIndex [<path>/_delta_log/<file>, ...]`.
+const DELTA_LOG_PATH = /((?:[A-Za-z][A-Za-z0-9+.-]*:\/{1,3}|\/)[^\s,[\]()#]+?)\/_delta_log(?=[/[\],\s)]|$)/g;
+
+                                                              
+
+// Every distinct table path named in the plans; `unreliable` when a plan is missing or a path cut.
+function deltaLogPaths(executions                                  )           {
+  const paths = new Set        ();
+  let unreliable = false;
+  for (const exec of executions) {
+    if (!exec.planTree) { unreliable = true; continue; }
+    walkPlanTree(exec.planTree, (node) => {
+      for (const text of [node.name, node.detail ?? '']) {
+        const mentions = text.split('/_delta_log').length - 1;
+        if (mentions === 0) continue;
+        const matches = [...text.matchAll(DELTA_LOG_PATH)];
+        // A `/_delta_log` that is not read as a path is a table this set cannot name.
+        if (matches.length < mentions) unreliable = true;
+        for (const m of matches) {
+          if (isCut(m[1])) unreliable = true;
+          else paths.add(m[1].replace(/\/+$/, ''));
+        }
+      }
+    }, { dedupe: true });
+  }
+  return { paths, unreliable };
+}
+
+/** The one table path the plans of `executions` name, or null for none, several or an unreadable plan. */
+export function soleLogPath(executions                                  )                     {
+  const { paths, unreliable } = deltaLogPaths(executions);
+  const [path] = paths;
+  return !unreliable && paths.size === 1 ? { kind: 'path', target: path } : null;
+}

package/vendor-core/delta-targets.js

@@ -0,0 +1,127 @@
+// Delta write targets that the plan node's simpleString does not carry. A Delta command node is a
+// bare name in sparkPlanInfo on real logs; its target is read from the `Arguments:` line the parser
+// keeps (SqlExecution.commandArguments, one reader per command in COMMANDS), and for a command that
+// always scans its target (MERGE, UPDATE, DELETE) from the single `_delta_log` path of the executions
+// sharing its root. A write command may read another Delta table as its source, so it never takes
+// that step. Otherwise the target is null: a rule that cannot name exactly one target names none.
+import { isDeltaCommand,                   } from './delta-commands.js';
+import { soleLogPath } from './delta-log-paths.js';
+import { isCut, splitArgs, tableName,                  } from './write-target-args.js';
+                                                         
+                                                      
+
+                                                                                                       
+
+                                                                       
+
+/** Executions grouped by rootExecutionId, each group excluding the root itself. */
+export function indexByRoot(sql                               )                   {
+  const byRoot = new Map                            ();
+  for (const exec of sql.values()) {
+    if (exec.rootExecutionId === undefined || exec.rootExecutionId === exec.id) continue;
+    const list = byRoot.get(exec.rootExecutionId);
+    if (list) list.push(exec);
+    else byRoot.set(exec.rootExecutionId, [exec]);
+  }
+  return byRoot;
+}
+
+// The text after `<Command>\nArguments: `, or null when the execution kept none for this command.
+function argumentsOf(exec                  , command        )                {
+  const kept = exec.commandArguments;
+  const head = `${command}\nArguments: `;
+  return kept !== undefined && kept.startsWith(head) ? kept.slice(head.length) : null;
+}
+
+// A catalog table is printed as its quoted identifier, `catalog`.`db`.`t`, as one top-level
+// argument. Two or more such arguments (or none) leave the table unnamed. The session catalog is
+// implied by a Hive or V1 table name, so it is dropped: `spark_catalog`.`db`.`t` is db.t.
+const QUOTED_TABLE = /^`[^`]+`(?:\.`[^`]+`){1,2}$/;
+
+function tableFromArguments(args        )                     {
+  const candidates = splitArgs(args).filter((arg) => QUOTED_TABLE.test(arg.text));
+  if (candidates.length !== 1 || !candidates[0].terminated) return null;
+  const name = tableName(candidates[0].text);
+  if (!name || isCut(name)) return null;
+  const parts = name.split('.');
+  return { kind: 'table', target: parts.length === 3 && parts[0] === 'spark_catalog' ? parts.slice(1).join('.') : name };
+}
+
+// SaveIntoDataSourceCommand prints `<provider>@<hash>, [k=v, k2=v2], <mode>`. The target is the
+// one `path` option of a Delta provider.
+function pathFromSaveOptions(args        )                     {
+  const options = splitArgs(args)[1];
+  if (!options?.terminated || !/^\[[\s\S]*\]$/.test(options.text)) return null;
+  const body = options.text.slice(1, -1);
+  const keys = [...body.matchAll(/(?:^|, )([A-Za-z0-9_.-]+)=/g)];
+  const paths = keys.flatMap((key, i) => {
+    if (key[1].toLowerCase() !== 'path') return [];
+    return [body.slice(key.index  + key[0].length, i + 1 < keys.length ? keys[i + 1].index  : body.length)];
+  });
+  if (paths.length !== 1 || isCut(paths[0]) || /\*{5,}\(redacted\)/.test(paths[0])) return null;
+  return { kind: 'path', target: paths[0] };
+}
+
+                          
+                                                                       
+                                                  
+                                                                                                
+                       
+ 
+
+const tableCommand                 = { fromArguments: tableFromArguments, scansTarget: false };
+const scanningTableCommand                 = { ...tableCommand, scansTarget: true };
+
+// One entry per Delta command: a new command is added here (and to DELTA_COMMANDS), not in the resolver.
+const COMMANDS                                       = {
+  MergeIntoCommand: scanningTableCommand,
+  UpdateCommand: scanningTableCommand,
+  DeleteCommand: scanningTableCommand,
+  WriteIntoDelta: tableCommand,
+  WriteIntoDeltaCommand: tableCommand,
+  // Only the arguments say a save is a Delta one; a JDBC save with a Delta table in its source
+  // would otherwise take that table as its target.
+  SaveIntoDataSourceCommand: {
+    fromArguments: (args) => (args.includes('DeltaDataSource') ? pathFromSaveOptions(args) : null),
+    scansTarget: false,
+  },
+};
+
+/** Target of a Delta command whose simpleString names none; `node` must be the root of `exec`'s plan. */
+export function resolveDeltaCommandTarget(
+  command        , exec                  , node          , byRoot                  ,
+)                     {
+  if (!isDeltaCommand(command) || node !== exec.planTree) return null;
+  const handler = COMMANDS[command];
+  const args = argumentsOf(exec, command);
+  const found = args === null ? null : handler.fromArguments(args);
+  if (found || !handler.scansTarget) return found;
+  // Child executions are the root's own only when it is the root: a command run under another
+  // execution shares that execution's children, which may belong to unrelated work.
+  return exec.rootExecutionId === exec.id ? soleLogPath(byRoot.get(exec.id) ?? []) : null;
+}
+
+// An append or overwrite of an existing Delta table (saveAsTable, INSERT INTO or OVERWRITE) is an
+// `AppendDataExecV1` or `OverwriteByExpressionExecV1` whose first argument is the table object:
+// `DeltaTableV2(<session>,<path>,Some(CatalogTable(\nCatalog: c\nDatabase: d\nTable: t\n...)),...)`.
+// Its CatalogTable block names the table; a path-based table has `None` there and only the path.
+const DELTA_TABLE_V2 = /^\w+ DeltaTableV2\([^,\s]*,([^,\n]*),(?:Some\(CatalogTable\(\n(?:Catalog: (\S+)\n)?Database: (\S+)\nTable: (\S+)\n|None,)/;
+const SIMPLE_NAME = /^[A-Za-z0-9_$]+$/;
+
+/** Target of a V1-fallback write on a DeltaTableV2 table, or null when the detail has another shape. */
+export function parseDeltaTableV2(detail        )                     {
+  const m = DELTA_TABLE_V2.exec(detail);
+  if (!m) return null;
+  const [, path, catalog, database, table] = m;
+  if (table === undefined) return path && !isCut(path) ? { kind: 'path', target: path } : null;
+  if (![database, table, catalog ?? 'spark_catalog'].every((part) => SIMPLE_NAME.test(part))) return null;
+  // The session catalog is implied by a V1 name, so it is dropped (as for a Delta command).
+  const name = `${database}.${table}`;
+  return { kind: 'table', target: catalog === undefined || catalog === 'spark_catalog' ? name : `${catalog}.${name}` };
+}
+
+/** The append a Delta CTAS or RTAS runs under its own root: its staged table has no name. */
+export function isStagedDeltaWrite(write                                      )          {
+  return (write.command === 'AppendDataExecV1' || write.command === 'OverwriteByExpressionExecV1')
+    && /^\w+ \S*DeltaCatalog\$StagedDeltaTableV2@/.test(write.raw);
+}