sparkforensics-server 0.6.0 → 0.8.1

package/vendor-core/python-stage.js

@@ -1,5 +1,5 @@
 import { planNodesOfStage } from './stage-plan-nodes.js';
-                                                      
+                                                                
 
 // Plan operators that hand rows to a Python worker process: the row-at-a-time and Arrow Python UDF
 // evaluators, the pandas/Arrow grouped and map operators, and the PythonRDD scan of an RDD
@@ -9,6 +9,17 @@ import { planNodesOfStage } from './stage-plan-nodes.js';
 // with "*(n) " in some plan strings.
 const PYTHON_PLAN_NODE = /^(?:\*\(\d+\)\s*)?(?:PythonRDD|BatchEvalPython|ArrowEvalPython|ArrowAggregatePython|ArrowWindowPython|\w+InPandas|\w+InArrow)\w*\b/;
 
+// The row-at-a-time Python UDF evaluator: each row is pickled to a Python worker and its result
+// pickled back. The Arrow-based ArrowEvalPython is deliberately not matched, and neither are the
+// UDTF variants (BatchEvalPythonUDTF), which spark.sql.execution.pythonUDF.arrow.enabled does not
+// govern.
+const BATCH_EVAL_PYTHON_NODE = /^(?:\*\(\d+\)\s*)?BatchEvalPython$/;
+
+/** True for a plan node that runs row-at-a-time Python UDFs (BatchEvalPython). */
+export function isBatchEvalPythonNode(node                        )          {
+  return BATCH_EVAL_PYTHON_NODE.test(node.name ?? '');
+}
+
 // An RDD lambda or map function has no SQL plan to match, and a stage whose plan could not be
 // matched is left with nothing but its name and call site.
 const PYTHON_STAGE_NAME = /PythonRDD/;

package/vendor-core/recommendation-rollup.js

@@ -1,7 +1,7 @@
 // Explicit .ts extensions: plain Node's ESM resolver (the runtime CLI/MCP path
 // runs under) requires the exact specifier, unlike a bundler.
 import { mergeIntervals } from './intervals.js';
-import { formatRawWaste, formatWallClockRange, readsAsZero, worstImpactBand, IMPACT_BAND_ORDER } from './format-utils.js';
+import { asCoreTime, formatRawWaste, formatWallClockRange, readsAsZero, worstImpactBand, IMPACT_BAND_ORDER } from './format-utils.js';
                                                                     
 
                          
@@ -25,7 +25,7 @@ export function computeStageUnionMs(
     // incomplete/truncated log (the case `incompleteRun` flags) leaves
     // `completedAt` unset, and coercing it to 0 would contribute a hugely
     // negative interval and a nonsense negative recoverable-time figure.
-    // Same filter `computeWallClock` (src/wall-clock.ts) already applies.
+    // Same filter `computeWallClock` (wall-clock.ts) applies.
     if (stage.submittedAt == null || stage.completedAt == null) continue;
     intervals.push([stage.submittedAt, stage.completedAt]);
   }
@@ -126,7 +126,7 @@ export function buildRecommendationRollup(
 
 /** A resource group's summed waste as the board and CLI print it; null when it reads as zero. */
 export function resourceGroupTotal(group                                       )                {
-  const text = formatRawWaste({ value: group.total, unit: group.unit });
+  const text = formatRawWaste(asCoreTime({ value: group.total, unit: group.unit }));
   return readsAsZero(text) ? null : text;
 }
 
@@ -161,10 +161,10 @@ export function rollupGroupStat(group             )
  * else that decides whether a REGISTRY type has "something to show" (an
  * active widget card vs. a Clean-checks line): memoryUtilization's
  * memoryBand/dataUnavailable variant reports a missing-evidence caveat
- * (spark.eventLog.logStageExecutorMetrics=true not on for this run), not an
+ * (the log carries no executor heap peaks: Spark before 3.0 or local mode), not an
  * optimization or a clean bill of health; the exact same fact already lives
  * in the Evidence availability ledger's own `executorMetrics` entry
- * (src/evidence-availability.ts), so it belongs there, not as its own card. */
+ * (evidence-availability.ts), so it belongs there, not as its own card. */
 export function isRealFinding(finding         )          {
   if (finding.type === 'memoryUtilization' && finding.variant === 'memoryBand' && finding.dataUnavailable) return false;
   return true;
@@ -173,7 +173,9 @@ export function isRealFinding(finding         )          {
 /** True when a finding is a candidate for the "fix these first" ranking.
  * Shared by evidence-report.ts (CLI/MCP path) and FixTheseFirst.tsx (the
  * dashboard widget) so these hardcoded exclusions can't drift between the two
- * surfaces. Doesn't check `REGISTRY[finding.type] != null`: `REGISTRY` lives
+ * surfaces. The verdict's finding count is the eligible findings, so it leaves out a log-completeness
+ * note (incompleteRun) and a missing-evidence caveat; the verdict wording says so (`verdictSummary`).
+ * Doesn't check `REGISTRY[finding.type] != null`: `REGISTRY` lives
  * in a `.tsx` file, not importable from this core module, so FixTheseFirst.tsx
  * layers that extra check on top of this one. */
 export function isEligible(finding         )          {
@@ -192,10 +194,9 @@ export function isEligible(finding         )          {
 /** Ranking tier for a single finding: time-based findings (a real `wallClock`
  * claim) are the only ones whose magnitudes share a unit, so they are the
  * only ones ranked numerically. `resourceOnly` findings carry `rawWaste` in
- * whatever unit their detector emitted, which the spec (section 5) forbids
- * comparing against a time figure or against another resourceOnly finding's
- * own raw magnitude; they rank by impact band instead, below every time-based
- * finding. Informational findings (no magnitude at all) rank last, also by
+ * whatever unit their detector emitted, which cannot be compared against a
+ * time figure or against another resourceOnly finding's own raw magnitude;
+ * they rank by impact band instead, below every time-based finding. Informational findings (no magnitude at all) rank last, also by
  * impact band. */
 const TIER_TIME = 0;
 const TIER_RESOURCE = 1;
@@ -211,8 +212,8 @@ function tierOf(finding         )         {
 /** Ranks a group's members so the "representative" finding (the one whose
  * action label/tag donate to the group's row) is always the highest-impact
  * one, not just the first one `buildRecommendationRollup` happened to
- * collect. Used to pick each type-group's own highest-impact member and to
- * order that group's expanded list; cross-group ordering is
+ * collect. Picks each type-group's own highest-impact member and orders
+ * that group's expanded list; cross-group ordering is
  * `buildRecommendationRollup`'s own job, not this function's. */
 export function rankFindings(findings           )            {
   return [...findings].sort((a, b) => {

package/vendor-core/redact.js

@@ -9,6 +9,7 @@
 
 import { decodeCollections, encodeCollections,                    } from './export-data.js';
                                                     
+import { isDetectorConfKey, REDACTED_CONF_VALUE } from './spark-conf.js';
 import { REDACTED_TEXT, redactTaskFailureGroup,                        } from './task-failure.js';
 
 // Host / IP identifier patterns. Used to enumerate host names that surface only
@@ -23,13 +24,10 @@ const HOST_PATTERNS = [
   /\b(?:\d{1,3}\.){3}\d{1,3}\b/g, // bare IPv4: 10.1.2.3
 ];
 
-// Spark application id (e.g. application_1690000000000_0001). Besides
-// redactReport's structured summary.app.id field, an app id can also surface
-// as a residual free-text token (e.g. embedded in a finding's evidence or
-// recommendation string), same as a host/IP token embedded in a stage name
-// or plan detail. redactComparison has no structured app-id field at all
-// (baselineLabel/candidateLabel are caller-supplied labels, not Spark app
-// ids), so free text is its *only* source of app ids.
+// Spark application id (e.g. application_1690000000000_0001). Besides the
+// structured summary.app.id field, an app id can surface as a free-text token
+// in evidence or recommendation strings. redactComparison has no structured
+// app-id field, so free text is its only source of app ids.
 const APP_ID_PATTERNS = [/\bapplication_\d{10,}_\d+\b/g];
 
 // Walk every string in the tree once, collecting matches for each `{ patterns,
@@ -56,12 +54,11 @@ function scanTokens(node         , sinks
 }
 
 // Recursively collects every string value found under a key literally named
-// `host`, anywhere in the tree. Host names surface at several depths, a
-// finding's own `host`, `evidence.host`, and now
-// `evidence.failedTaskDetails[].host` / `evidence.retriedTaskDetails[].host`,
-// and only some of them are host-*pattern*-shaped (a plain FQDN like
-// `worker-3.internal` matches neither HOST_PATTERN). Walking by key name rather
-// than enumerating known paths keeps any future nested `host` field covered.
+// `host`, anywhere in the tree. Host names surface at several depths (a
+// finding's own `host`, `evidence.host`, `evidence.failedTaskDetails[].host`,
+// `evidence.retriedTaskDetails[].host`), and only some are host-pattern-shaped
+// (a plain FQDN like `worker-3.internal` matches neither HOST_PATTERNS).
+// Walking by key name keeps any nested `host` field covered.
 function collectHostFields(node         , hosts             )       {
   if (Array.isArray(node)) {
     for (const n of node) collectHostFields(n, hosts);
@@ -77,9 +74,7 @@ function collectHostFields(node         , hosts             )       {
 
 // A failed-task error's message and stack text can carry file paths and data values that no
 // host/app-id pattern recognizes, so they are dropped rather than pseudonymized: every array under
-// a key named `failureGroups` (a `failures` finding's, or a stage's in the HTML export) gets
-// redactTaskFailureGroup applied. Walking by key name, like collectHostFields, needs no path list.
-// Returns a fresh tree.
+// a key named `failureGroups` gets redactTaskFailureGroup applied. Returns a fresh tree.
 function redactFailureGroups   (node   )    {
   if (Array.isArray(node)) return node.map((n) => redactFailureGroups(n))     ;
   if (node && typeof node === 'object') {
@@ -95,10 +90,8 @@ function redactFailureGroups   (node   )    {
 }
 
 // A stage's failure reason is Spark's free-text message and can carry file paths and data values
-// too, so it is replaced outright, like a failure group's message: a `stageFailed` finding's
-// `valueText`, the `stageFailureReason` of a stage record, a job's `exception` (the run outcome's
-// fallback reason), and the report's `failureReason` copy. Walking by key name, like
-// collectHostFields, needs no path list. Returns a fresh tree.
+// too, so it is replaced outright: a `stageFailed` finding's `valueText`, a stage's
+// `stageFailureReason`, a job's `exception` and the report's `failureReason`. Returns a fresh tree.
 const REASON_KEYS = new Set(['stageFailureReason', 'exception', 'failureReason']);
 function redactStageFailureReasons   (node   )    {
   if (Array.isArray(node)) return node.map((n) => redactStageFailureReasons(n))     ;
@@ -123,13 +116,11 @@ function redactFailureText   (node   )    {
 // spark.yarn.am.hostname) carry plain FQDN host names that neither
 // HOST_PATTERNS matches (no IP/EC2 shape) nor collectHostFields's by-key-name
 // walk catches (the literal key is the dotted Spark property name, never
-// `host` itself). app.config is a flat Record<string, string> unique to
-// redactRunModel: no other redact* export ships a raw Spark config dict.
-// Known gap: a hostname value under a differently-named key isn't caught by
-// this suffix check. Confirmed against a real cluster config: spark.master,
-// spark.yarn.historyServer.address, and the plural YARN proxy/HA keys
-// (...AmIpFilter.param.PROXY_HOSTS, ...RM_HA_URLS) all carry real hostnames
-// through un-redacted today.
+// `host` itself). app.config and each SQL execution's modifiedConfigs are
+// flat Record<string, string> dicts that only the run-tree redaction ships.
+// Known gap: a hostname under a differently-named key is not caught by this
+// suffix check (spark.master, spark.yarn.historyServer.address and the YARN
+// proxy/HA keys); ci-and-automation.md lists them.
 function collectConfigHostValues(config                                    , hosts             )       {
   if (!config) return;
   for (const [key, value] of Object.entries(config)) {
@@ -138,11 +129,9 @@ function collectConfigHostValues(config                                    , hos
   }
 }
 
-// Known locations of the identifiers, so we don't have to guess which strings
-// are sensitive: the app id lives at summary.app.id; host names live under any
-// `host` key in the findings tree (see collectHostFields), plus a free-text
-// scan for hosts/IPs and app ids that appear only inside string values (e.g. a
-// finding's evidence/recommendation text).
+// Known locations of the identifiers: the app id lives at summary.app.id; host
+// names live under any `host` key in the findings tree (see collectHostFields),
+// plus a free-text scan for hosts/IPs and app ids inside string values.
                             
                                                                           
                                                                               
@@ -227,15 +216,10 @@ export function redactReport                            (input   )    {
   return app ? { ...out, summary: { ...out.summary, app: withRedactedName(app) } } : out;
 }
 
-// Run-comparison counterpart: no single app-id *field* to pseudonymize
-// (baselineLabel/candidateLabel are caller-supplied labels, not Spark app
-// ids), but stage names surface throughout the tree (FindingsDeltaRow.stages,
-// baseStages/candStages[].name) and, like the evidence report's findings, can
+// Run-comparison counterpart: stage names surface throughout the tree and can
 // carry a host/IP token or an app id as free text (e.g. `collect at
 // application_1690000000000_0001 worker-10-1-2-3.scala:45`). Scans the whole
-// comparison tree rather than enumerating those fields individually, so a
-// future CompareRunsResult field carrying free text is covered without this
-// function needing to change.
+// tree rather than enumerating fields, so any free-text field is covered.
 export function redactComparison   (comparison   )    {
   const hosts = new Set        ();
   const appIds = new Set        ();
@@ -244,14 +228,34 @@ export function redactComparison   (comparison   )    {
 }
 
 // HTML-export counterpart to redactReport/redactComparison. Unlike
-// redactReport (scoped to EvidenceReportJson's summary.app.id + findings),
-// this also walks executors.added/removed for their literal `host` field
-// (ExecutorAddedEvent.host), since raw executor records: not just findings
-//: reach data.js.
-                                                                                       
+// redactReport, this also walks executors.added/removed for their literal
+// `host` field, since raw executor records reach the export, not just findings.
+                                                                                          
+                                                                                                    
+                                                      
+                                                            
+  
+
+// A per-query setting is whatever the job passed to spark.conf.set: a path, a bucket name, a host. A
+// detector reads only the Spark SQL tuning keys, so every other value is replaced by Spark's own
+// placeholder for a hidden value and keeps no job-written text.
+function withoutJobSetQueryValues                                                        (exec   )    {
+  if (!exec.modifiedConfigs) return exec;
+  const kept = Object.fromEntries(Object.entries(exec.modifiedConfigs)
+    .map(([key, value]) => [key, isDetectorConfKey(key) ? value : REDACTED_CONF_VALUE]));
+  return { ...exec, modifiedConfigs: kept };
+}
+
+// An executor's removal reason is Spark's free-text message and can carry a path or a data value. No
+// surface displays it. It is replaced by field, not through REASON_KEYS: `reason` also names unrelated
+// fields elsewhere, and the key walk would blank those.
+function withoutExecutorRemovalReasons                   (data   )    {
+  const removed = data.executors.removed.map((e) => ({ ...e, reason: REDACTED_TEXT }));
+  return { ...data, executors: { ...data.executors, removed } };
+}
 
 function redactRunTree                   (input   )    {
-  const data = redactFailureText(input);
+  const data = withoutExecutorRemovalReasons(redactFailureText(input));
   const appIds = new Set        ();
   const hosts = new Set        ();
   const appId = data.app?.id;
@@ -261,8 +265,10 @@ function redactRunTree                   (input   )    {
   collectHostFields(data.catalog, hosts);
   collectHostFields(data.configFindings, hosts);
   collectConfigHostValues(data.app?.config, hosts);
+  for (const exec of data.sql ?? []) collectConfigHostValues(exec.modifiedConfigs, hosts);
   scanTokens(data, [{ patterns: HOST_PATTERNS, out: hosts }, { patterns: APP_ID_PATTERNS, out: appIds }]);
-  const out = applyReplacements(data, { appIds, hosts });
+  const replaced = applyReplacements(data, { appIds, hosts });
+  const out = replaced.sql ? { ...replaced, sql: replaced.sql.map(withoutJobSetQueryValues) } : replaced;
   if (!out.app) return out;
   const app = withRedactedName(out.app);
   // spark.app.name repeats the name in the config the HTML export ships.

package/vendor-core/run-aggregates.js

@@ -1,12 +1,10 @@
-// Whole-run aggregation pass. Runs INSIDE parser-worker.js over the retained
-// per-task launch/finish timestamps in taskStore, so raw task data never
-// reaches the main thread: only the compact summary below is posted. Pure and
-// worker-agnostic so it is unit-testable without a Worker.
+// Whole-run aggregation pass. Runs INSIDE parser-worker.ts over the retained per-task
+// launch/finish timestamps in taskStore, so raw task data never reaches the main thread: only the
+// compact summary below is posted. Pure and worker-agnostic so it is unit-testable without a Worker.
 //
-// Uses computeCoreTimeSeries in coreCount mode (O(n log n)) to derive the
-// busy-core histogram; the time-bucket mode is intentionally NOT used here
-// because it is O(buckets × segments) and a whole run can hold millions of
-// tasks (see core-time-series.js PERF note).
+// Uses computeCoreTimeSeries in coreCount mode (O(n log n)) to derive the busy-core histogram; the
+// time-bucket mode is intentionally NOT used here because it is O(buckets × segments) and a whole
+// run can hold millions of tasks.
 
 import { FIELDS } from './stage-quantiles.js';
 import { computeCoreTimeSeries } from './core-time-series.js';