sparkforensics-mcp 0.3.0 → 0.5.0

package/vendor-core/docs-config.js

@@ -1,5 +1,3 @@
-import { detectorCatalog } from './detectors.js';
-
 // Single source of the docs-panel URL surface. DOCS_BASE_DIR is the built docs-site path that
 // serves the tuning reference, relative to the app's origin.
 export const DOCS_BASE_DIR         = 'docs/tuning-reference';
@@ -90,40 +88,6 @@ export function isKnownDocAnchor(anchor         )          {
   return KNOWN_DOC_ANCHORS.has(String(anchor));
 }
 
-// Finding types that never appear as their own DETECTORS entry's type: the parent entry declares
-// a different type because one plan-walk covers two rules (see broadcastSizing). Map to the parent.
-const TYPE_ALIASES                         = {
-  underBroadcast: 'broadcastSizing',
-  overBroadcast: 'broadcastSizing',
-};
-
-// DETECTORS is static, so this grouping is built once (lazily) instead of re-scanning per
-// docAnchorForType call (called once per TagBadge per render).
-let anchorsByTypeCache                                                  ;
-
-function anchorsByType()                                       {
-  if (!anchorsByTypeCache) {
-    anchorsByTypeCache = new Map();
-    for (const entry of detectorCatalog()) {
-      const anchors = anchorsByTypeCache.get(entry.type) ?? new Set();
-      anchors.add(entry.docAnchor);
-      anchorsByTypeCache.set(entry.type, anchors);
-    }
-  }
-  return anchorsByTypeCache;
-}
-
-/** Resolves a finding `type` to its documented anchor from detectorCatalog(). Returns undefined
- * when entries sharing the type disagree on docAnchor (only configAudit today), or when the
- * resolved anchor isn't in the allowlist (isKnownDocAnchor, the same gate DocsLink uses). */
-export function docAnchorForType(type        )                     {
-  const resolvedType = TYPE_ALIASES[type] ?? type;
-  const anchors = anchorsByType().get(resolvedType) ?? new Set();
-  if (anchors.size !== 1) return undefined;
-  const [anchor] = anchors;
-  return anchor && isKnownDocAnchor(anchor) ? anchor : undefined;
-}
-
 /** The docAnchor every finding in `findings` carries, or undefined when they disagree or any lacks
  * one. For a badge standing for several findings (a widget header, a grouped row) whose type-level
  * docAnchorForType can't pick one: configAudit's sub-checks each stamp their own anchor. */

package/vendor-core/docs-content/chapters/nav-index.json

@@ -3,6 +3,7 @@
     "anchor": "intro",
     "section": "Getting Started",
     "title": "Introduction",
+    "brief": "Guide overview: what the reference covers, the severity-dot legend, and the tag system linking into the Bottleneck Reference.",
     "keywords": [
       "severity-dot",
       "tag-system"
@@ -14,6 +15,7 @@
     "anchor": "spark-architecture",
     "section": "Optimization Guide",
     "title": "Spark Execution Model",
+    "brief": "Spark's execution model: Driver/Executor split, DAGScheduler/TaskScheduler roles, stage boundaries at shuffles, and pipelining of narrow transformations.",
     "keywords": [
       "driver-executor",
       "dagscheduler",
@@ -27,6 +29,7 @@
     "anchor": "memory-model",
     "section": "Optimization Guide",
     "title": "Memory Management",
+    "brief": "Unified memory management: on-heap/off-heap execution vs. storage regions, spill triggers, and executor memory overhead sizing.",
     "keywords": [
       "unified-memory",
       "execution-memory",
@@ -41,6 +44,7 @@
     "anchor": "partitioning",
     "section": "Optimization Guide",
     "title": "Partitioning",
+    "brief": "Partition sizing and reshaping: repartition vs. coalesce mechanics, shuffle-partition tuning, and skew detection thresholds.",
     "keywords": [
       "repartition",
       "coalesce",
@@ -53,6 +57,7 @@
     "anchor": "joins",
     "section": "Optimization Guide",
     "title": "Join Optimization",
+    "brief": "Physical join strategy selection: broadcast vs. shuffle joins, join hints, bucketing, and AQE's runtime join-strategy switching.",
     "keywords": [
       "broadcast",
       "shuffle-join",
@@ -66,6 +71,7 @@
     "anchor": "shuffle",
     "section": "Optimization Guide",
     "title": "Shuffle",
+    "brief": "Shuffle internals: SortShuffleManager mechanics, shuffle-avoidance paths (bucketing, Storage Partition Join), and shuffle-tuning configs.",
     "keywords": [
       "sortshufflemanager",
       "bucketing",
@@ -78,6 +84,7 @@
     "anchor": "data-formats",
     "section": "Optimization Guide",
     "title": "Data Formats",
+    "brief": "Columnar file format tradeoffs: Parquet vs. ORC internals, splittability, predicate pushdown, and compression codec choice.",
     "keywords": [
       "parquet",
       "orc",
@@ -91,6 +98,7 @@
     "anchor": "table-formats",
     "section": "Optimization Guide",
     "title": "Table Formats",
+    "brief": "Lakehouse table-format tuning across Delta Lake, Iceberg, and Hudi: file sizing, compaction/OPTIMIZE, clustering (Z-order/liquid/sort), metadata and manifest overhead, snapshot/version expiry, and deletion vectors vs. merge-on-read/copy-on-write.",
     "keywords": [
       "delta",
       "iceberg",
@@ -107,6 +115,7 @@
     "anchor": "caching",
     "section": "Optimization Guide",
     "title": "Caching & Persistence",
+    "brief": "cache()/persist() semantics, storage levels, eviction behavior, and checkpointing as a lineage-truncation alternative.",
     "keywords": [
       "cache",
       "persist",
@@ -120,6 +129,7 @@
     "anchor": "pyspark",
     "section": "Optimization Guide",
     "title": "PySpark Specifics",
+    "brief": "PySpark-specific performance: UDF serialization cost, Arrow-optimized and pandas UDF types, and Python-worker memory configs.",
     "keywords": [
       "udf",
       "arrow",
@@ -133,6 +143,7 @@
     "anchor": "aqe",
     "section": "Optimization Guide",
     "title": "Adaptive Query Execution",
+    "brief": "AQE's runtime re-optimization loop: post-shuffle partition coalescing, join-strategy promotion, and skew-partition splitting.",
     "keywords": [
       "partition-coalescing",
       "join-strategy-promotion",
@@ -145,6 +156,7 @@
     "anchor": "cluster-config",
     "section": "Optimization Guide",
     "title": "Cluster Tuning",
+    "brief": "Cluster-level sizing: executor core/memory formulas, container memory budgeting, dynamic allocation, and locality-wait tuning.",
     "keywords": [
       "executor-sizing",
       "container-memory",
@@ -158,6 +170,7 @@
     "anchor": "anti-patterns",
     "section": "Optimization Guide",
     "title": "Anti-Patterns",
+    "brief": "Checklist of a dozen recurring Spark performance anti-patterns, each with what it is, how to detect it, why it hurts, and how to fix it.",
     "keywords": [
       "checklist",
       "anti-pattern"
@@ -169,6 +182,7 @@
     "anchor": "bottleneck-skew",
     "section": "Detector Catalog",
     "title": "Task Skew",
+    "brief": "Detecting and mitigating task skew: oversized shuffle partitions vs. the median, AQE skew-join splitting, and manual salting as the pre-AQE fallback.",
     "keywords": [
       "skew",
       "aqe-skew-join",
@@ -181,6 +195,7 @@
     "anchor": "bottleneck-shuffle",
     "section": "Detector Catalog",
     "title": "Shuffle I/O",
+    "brief": "Detecting and mitigating shuffle I/O bottlenecks: read/write byte thresholds, fetch-wait time, and shuffle-avoidance and tuning levers.",
     "keywords": [
       "shuffle-bytes",
       "fetch-wait"
@@ -192,6 +207,7 @@
     "anchor": "bottleneck-spill",
     "section": "Detector Catalog",
     "title": "Memory / Disk Spill",
+    "brief": "Detecting and mitigating memory/disk spill: TaskMemoryManager pressure, the skew-vs-volume spill classification, and matching the fix to the cause.",
     "keywords": [
       "spill",
       "taskmemorymanager"
@@ -203,6 +219,7 @@
     "anchor": "bottleneck-gc",
     "section": "Detector Catalog",
     "title": "GC Pressure",
+    "brief": "Detecting and mitigating GC pressure: the gcPct warning/critical thresholds, oversized executors and memory.fraction as causes, and off-heap/collector fixes.",
     "keywords": [
       "gc-pressure",
       "memory-fraction",
@@ -215,6 +232,7 @@
     "anchor": "bottleneck-cold-start",
     "section": "Detector Catalog",
     "title": "Cold Start",
+    "brief": "Detecting and mitigating cold start: the firstStageSubmittedAt gap before the first stage is submitted, and executor-sizing/dynamic-allocation readiness fixes.",
     "keywords": [
       "cold-start",
       "dynamic-allocation"
@@ -226,6 +244,7 @@
     "anchor": "bottleneck-utilization",
     "section": "Detector Catalog",
     "title": "Executor Utilization",
+    "brief": "Detecting and mitigating low executor utilization: the avg/peak active-executor ratio, under-partitioning and coalesce-caused parallelism loss, and dynamic-allocation over-provisioning.",
     "keywords": [
       "executor-utilization",
       "under-partitioning",
@@ -238,6 +257,7 @@
     "anchor": "bottleneck-slow-host",
     "section": "Detector Catalog",
     "title": "Slow Host",
+    "brief": "Detecting and mitigating a slow host: host-wide task duration inflation versus the cluster median, and speculative-execution or locality-wait fixes.",
     "keywords": [
       "slow-host",
       "speculative-execution"
@@ -249,6 +269,7 @@
     "anchor": "bottleneck-failures",
     "section": "Detector Catalog",
     "title": "Task Failures",
+    "brief": "Detecting and mitigating task failures: the TaskEndReason taxonomy, failed-task-share thresholds, and distinguishing memory-driven ExecutorLostFailure from application-code ExceptionFailure.",
     "keywords": [
       "taskendreason",
       "executorlostfailure",
@@ -261,6 +282,7 @@
     "anchor": "bottleneck-straggler",
     "section": "Detector Catalog",
     "title": "Stragglers",
+    "brief": "Detecting and mitigating straggler tasks: the 4x-median-duration rule, distinguishing stragglers from GC pauses or skew, and speculative-execution tuning.",
     "keywords": [
       "straggler",
       "speculative-execution"
@@ -272,6 +294,7 @@
     "anchor": "bottleneck-retry-waste",
     "section": "Detector Catalog",
     "title": "Retry Waste",
+    "brief": "Detecting and mitigating wasted retry compute: executorRunTime burned by superseded task attempts, and distinguishing executor-loss from fetch-failure causes.",
     "keywords": [
       "retry-waste",
       "fetch-failure"
@@ -283,6 +306,7 @@
     "anchor": "bottleneck-tiny-tasks",
     "section": "Detector Catalog",
     "title": "Tiny Tasks",
+    "brief": "Detecting and mitigating tiny-task overhead: scheduling/serialization cost dominating once partitions shrink too far, and coalesce/repartition sizing fixes.",
     "keywords": [
       "tiny-tasks",
       "scheduling-overhead"
@@ -294,6 +318,7 @@
     "anchor": "bottleneck-job-failure-rate",
     "section": "Detector Catalog",
     "title": "Job Failure Rate",
+    "brief": "Detecting and mitigating job-level failure: the task/stage/executor retry-budget escalation ladder that turns isolated failures into an aborted job.",
     "keywords": [
       "job-failure-rate",
       "retry-budget"
@@ -305,6 +330,7 @@
     "anchor": "bottleneck-memory-utilization",
     "section": "Detector Catalog",
     "title": "Memory Utilization",
+    "brief": "Detecting and mitigating held-but-idle executor memory: cores sitting idle below allocated slots, dynamic-allocation idle timeouts that reclaim an executor's memory band, and the low-confidence peak-vs-allocated waste estimate.",
     "keywords": [
       "memory-utilization",
       "idle-timeout"
@@ -316,6 +342,7 @@
     "anchor": "bottleneck-duplicate-plan-subtree",
     "section": "Detector Catalog",
     "title": "Duplicate Plan Subtree",
+    "brief": "Detecting a repeated scan or exchange subtree in a physical plan, how exchange reuse and cache()/persist() avoid recomputing it, and why inferring duplication from the plan is a medium-confidence signal.",
     "keywords": [
       "duplicate-plan-subtree",
       "exchange-reuse"
@@ -327,6 +354,7 @@
     "anchor": "bottleneck-small-files",
     "section": "Detector Catalog",
     "title": "Small Files",
+    "brief": "Detecting and mitigating the small-files problem on read and write: metadata and I/O overhead of many tiny files, and coalesce()/repartition()/compaction and AQE advisoryPartitionSizeInBytes as output-size controls.",
     "keywords": [
       "small-files",
       "advisorypartitionsizeinbytes"
@@ -338,6 +366,7 @@
     "anchor": "bottleneck-broadcast-sizing",
     "section": "Detector Catalog",
     "title": "Broadcast Sizing",
+    "brief": "Detecting under- and over-broadcast joins: spark.sql.autoBroadcastJoinThreshold and its default, when to broadcast the smaller side versus shuffle, and the driver-collection and executor-memory failure modes of broadcasting too large a table.",
     "keywords": [
       "broadcast-threshold",
       "driver-collection"
@@ -349,6 +378,7 @@
     "anchor": "metrics",
     "section": "Reference",
     "title": "Metrics Glossary",
+    "brief": "Glossary of every metric surfaced elsewhere on the site: what each one measures, its event-log source field, and what a problematic value looks like.",
     "keywords": [
       "metrics-glossary",
       "event-log"
@@ -360,6 +390,7 @@
     "anchor": "config",
     "section": "Reference",
     "title": "Spark Config Quick-Reference",
+    "brief": "Cross-reference of the Spark/PySpark config properties and JVM GC flags discussed elsewhere in the guide, with defaults and version notes.",
     "keywords": [
       "spark-config",
       "gc-flags"

package/vendor-core/docs-content/detection/cache.md

@@ -1,7 +1,8 @@
 ### `CACHE`: Caching opportunity {#cache}
 
-A reusable dataset (re-read via the same SQL relation more than once) may be
-worth persisting between stages. Self-flags a confidence that scales with
+A reusable dataset (re-read via the same SQL relation more than once, or a
+join/union result recomputed by two or more executions, matched by plan
+shape) may be worth persisting between stages. Self-flags a confidence that scales with
 how many executions reuse the same relation: reuse is only inferred, from
 plan-scan identity across SQL executions, so confirm the reads really do
 hit the same data before you cache anything.

package/vendor-core/docs-content/detection/cfg.md

@@ -1,15 +1,16 @@
 ### `CFG`: Configuration audit {#cfg}
 
 Flags configuration settings that may cause reliability or efficiency
-problems, independent of any one stage's behavior. Four properties are
-audited today:
+problems, independent of any one stage's behavior. Four checks run:
 
 - `spark.shuffle.service.enabled`: flagged when dynamic allocation is on
   but the external shuffle service is off, since shuffle data won't survive
   executor removal.
-- `spark.dynamicAllocation.maxExecutors`: flagged for inverted bounds or a
-  missing upper bound.
-- `spark.serializer`: flagged when still on the default Java serializer;
-  `org.apache.spark.serializer.KryoSerializer` is faster and produces
-  smaller buffers.
-- `spark.executor.memoryOverhead`: flagged when set below a safe floor.
+- `spark.dynamicAllocation.minExecutors`/`maxExecutors`: with dynamic
+  allocation on, flagged when min exceeds max (reported on `minExecutors`)
+  or when no max is set.
+- `spark.serializer`: flagged when not set to Kryo (the default is the Java
+  serializer); `org.apache.spark.serializer.KryoSerializer` is faster and
+  produces smaller buffers.
+- `spark.executor.memoryOverhead`: flagged when set below max(384 MiB, 10%
+  of executor memory).

package/vendor-core/docs-content/detection/chrn.md

@@ -5,5 +5,4 @@ re-provisioning churn rather than normal scale-down. Raise
 `spark.dynamicAllocation.executorIdleTimeout`, or widen the
 `minExecutors`/`maxExecutors` bounds to reduce flapping. Self-flags a
 confidence that scales with how far the short-lived-executor share sits
-past the threshold: these thresholds are still a design spike, not yet
-validated against real-world runs.
+past the threshold.

package/vendor-core/docs-content/detection/cold.md

@@ -1,4 +1,6 @@
 ### `COLD`: Executor cold start {#cold}
 
-New executors take time to become available for work. Pre-warm the cluster,
-or use dynamic allocation.
+The first stage waited more than 30 s for an executor. Keep a warm pool of
+executors, or, with dynamic allocation, raise
+`spark.dynamicAllocation.minExecutors`/`initialExecutors` so the app doesn't
+scale up from zero.

package/vendor-core/docs-content/detection/cstor.md

@@ -2,3 +2,12 @@
 
 A persisted dataset is not fully cached in memory, or is spilling to disk.
 Raise executor memory, or shrink the cached dataset.
+
+The cached-partition counts and sizes come from `SparkListenerBlockUpdated`
+events, which Spark writes only when
+`spark.eventLog.logBlockUpdates.enabled=true`. Since Spark 2.3 the RDD
+storage figures in stage-submission events are always 0, and they are
+used only as a fallback. When a Spark 2.3+ run persists RDDs but its log has
+neither and block-update logging was off, the check reports that the cache could not be
+checked instead of passing it: turn `spark.eventLog.logBlockUpdates.enabled`
+on and rerun to measure eviction and disk spillover.

package/vendor-core/docs-content/detection/fail.md

@@ -5,5 +5,6 @@ instability or data-driven errors. The finding names the dominant error: the
 exception class, or the executor loss reason (for example "Container killed
 by YARN for exceeding memory limits"). It lists up to five distinct failures,
 each with its message and a short stack excerpt. With redaction on, messages
-and the message text inside excerpts are replaced, since they can carry file
-paths and data values; class names and stack frames stay.
+become `[redacted]` and message lines inside excerpts are dropped, since they
+can carry file paths and data values; class names, stack frames and the
+executor loss reason stay (hosts in it are pseudonymized).

package/vendor-core/docs-content/detection/gc.md

@@ -1,6 +1,7 @@
 ### `GC`: Garbage collection pressure {#gc}
 
-Tasks spend an unusually large share of time reclaiming memory. Reduce
+Tasks spend more than 10% of executor run time reclaiming memory. Reduce
 object creation: use primitive types, avoid UDFs, or raise executor memory.
-A stage with very little GC gets an informational note that executor memory
+A stage with GC below 5% gets an informational note that executor memory
 may be over-provisioned, only on stages that take at least 0.5% of the run.
+Both need at least 10 s of executor run time on the stage.

package/vendor-core/docs-content/detection/host.md

@@ -1,6 +1,7 @@
 ### `HOST`: Slow host {#host}
 
-One executor is much slower than its peers. It may just hold data locality
+One executor is much slower than its peers, or carries most of the stage's
+task time or bytes. It may just hold data locality
 for its tasks or carry one heavy stage, rather than a hardware fault.
 Enable `spark.speculation` to relaunch a lagging task automatically. Only
 flagged on stages that take at least 0.5% of the run.

package/vendor-core/docs-content/detection/local.md

@@ -1,6 +1,6 @@
 ### `LOCAL`: Core usage locality {#local}
 
 Tasks run without process- or node-local data placement more often than
-expected. Check `spark.locality.wait` settings and executor/data colocation.
+expected. Check executor/data colocation.
 Self-flags a confidence that scales with the non-local ratio and sample
 size: the thresholds are our own noise floor for this metric.

package/vendor-core/docs-content/detection/mem.md

@@ -1,10 +1,13 @@
 ### `MEM`: Memory utilization {#mem}
 
-Executor memory or core capacity may be over- or under-provisioned. Some
+Executor memory or core capacity may be over- or under-provisioned: more
+than 50% of available core time ran no task, an executor's heap peaked above
+95% of its allocation, or it stayed below 70%. Some
 detail here needs `spark.eventLog.logStageExecutorMetrics=true` on the run
 being analyzed; without it, per-executor memory usage can't be broken down.
 Review `spark.executor.memory` and executor count if allocated memory sat
 largely idle over the run. That idle-memory variant self-flags a confidence
 that scales with how far the estimated waste sits past a 1.5x buffer: it
-estimates waste from allocated-versus-used memory-time. Check it against
+estimates waste from allocated memory-time versus task run time (not
+measured heap usage). Check it against
 the Spark UI before resizing anything.

package/vendor-core/docs-content/detection/plan.md

@@ -7,7 +7,8 @@ this tag:
   plan. When the repeats have the same shape but different filters, columns
   or tables, the finding stays informational and claims no time. Only flagged
   when the repeat's stages take at least 0.5% of the run.
-- Small files: reading an excessive number of small files.
+- Small files: one plan node reads or writes more than 100 files averaging
+  under 3 MB. Compact upstream output, or coalesce before writing.
 - Under-broadcast: the smaller side of a Sort Merge Join looks well under
   the broadcast threshold; consider a `broadcast()` hint or raising
   `spark.sql.autoBroadcastJoinThreshold`.

package/vendor-core/docs-content/detection/sfail.md

@@ -2,4 +2,5 @@
 
 A stage attempt failed outright rather than losing individual tasks within
 it. Inspect the driver log for the failure reason and the job that triggered
-it.
+it. With redaction on, the failure reason becomes `[redacted]`, since it can
+carry file paths and data values.

package/vendor-core/docs-content/detection/shape.md

@@ -1,6 +1,7 @@
 ### `SHAPE`: Stage shape {#shape}
 
-The stage has an inefficient task count, output shape, or task-to-stage
-balance: for example, one straggler task taking a large fraction of the
-stage's wall-clock time. A too-low task count is only flagged on stages that
-take at least 0.5% of the run.
+The stage has an inefficient task count, output shape (output more than 10×
+input), or task-to-stage balance: one straggler task running for more than
+half the stage's wall-clock time and over 3× the median task, so it alone
+sets when the stage ends. A too-low task count and a straggler are only
+flagged on stages that take at least 0.5% of the run.

package/vendor-core/docs-content/detection/skew.md

@@ -3,4 +3,6 @@
 A small number of tasks take much longer than their peers in the same
 stage. For join-driven skew, enable AQE skew-join handling
 (`spark.sql.adaptive.skewJoin.enabled`); otherwise salt the key or
-repartition on a better key.
+repartition on a better key. Flagged when P95 task time (the longest task,
+on a stage with fewer than 20 tasks) exceeds 3x the median and the
+recoverable tail is at least 0.5% of the run.

package/vendor-core/docs-content/detection/slow.md

@@ -1,6 +1,6 @@
 ### `SLOW`: Stage slowness {#slow}
 
-A stage ran long overall without a more specific cause getting flagged.
-Often a partition-count problem: raise parallelism via
+A stage ran for 15 minutes or more and no slow host was flagged on it. It
+can appear alongside other findings on the same stage. Often a partition-count problem: raise parallelism via
 `spark.sql.shuffle.partitions` or `spark.default.parallelism`, or check for a
 large per-task data volume driving heavy shuffle and spill.

package/vendor-core/docs-content/detection/spec.md

@@ -2,7 +2,6 @@
 
 Speculative task attempts used a lot of executor time without confirming a
 genuine straggler. Self-flags a confidence that scales with how far the
-wasted time sits past the threshold: these thresholds are still a design
-spike, not yet validated against real-world runs. If task durations are
-just naturally variable rather than genuine stragglers, tune
+wasted time sits past the threshold. If task durations are just naturally
+variable rather than genuine stragglers, tune
 `spark.speculation.multiplier`/`spark.speculation.quantile`.

package/vendor-core/docs-content/detection/spill.md

@@ -4,4 +4,4 @@ Tasks are writing data out of memory, which slows execution. Two spill
 patterns get flagged differently: skew spill, where a few heavy tasks spill
 while most don't (rebalance partitioning), and volume spill, where most
 tasks spill because the data genuinely exceeds available memory (add
-partitions). Only flagged on stages that take at least 0.5% of the run.
+partitions or executor memory). Only flagged on stages that take at least 0.5% of the run.

package/vendor-core/docs-site-config.js

@@ -14,3 +14,6 @@ import { typeTag } from './format-utils.js';
 export function findingGuideUrl(type        )         {
   return `docs/user-guide/understanding-findings.html#${typeTag(type).toLowerCase()}`;
 }
+
+// How to find a Spark event log: enabling event logging, the History Server, managed platforms, SSH bastions.
+export const ALTERNATIVE_LOG_RETRIEVAL_URL = 'docs/user-guide/alternative-log-retrieval.html';

package/vendor-core/effective-conf.js

@@ -0,0 +1,107 @@
+// The effective Spark configuration of a run for the CLI's JSON output, so a caller can check
+// that a --conf overlay took effect. Every key is listed; a value is withheld when the key or the
+// value matches a secret pattern, and credentials inside URL-like values are stripped. A withheld
+// key is shown as present with no value, and no derivative of the value (no hash, length or
+// prefix) is emitted.
+                                               
+
+export const EFFECTIVE_CONF_SCHEMA_VERSION = 1;
+
+/** Spark's default spark.redaction.regex, in JS syntax (the JVM pattern is `(?i)` + this). */
+export const DEFAULT_SECRET_PATTERN = 'secret|password|token|access[.]?key';
+/** Further credential names no Spark default covers, tested on keys (Azure `fs.azure.account.key.*`,
+ * `apiKey`, `pwd`, a bare `pass` or `sas` segment, `credential`). */
+export const EXTRA_SECRET_KEY_PATTERN = 'passwd|pwd|(?:^|[._-])pass(?:[._-]|$)|api[._-]?key|account[._-]?key|private[._-]?key|credential|(?:^|[._-])sas(?:[._-]|$)|(?:^|[._-])sig(?:nature)?(?:[._-]|$)';
+const REDACTED = '[redacted]';
+
+                                
+                        
+                                                                
+                                 
+                                                         
+                       
+                                                                                     
+                       
+ 
+
+                                       
+                                         
+                  
+                                                                                                  
+                       
+ 
+
+/** Compiles a JVM regex, which may start with inline flags such as (?i), as a JS RegExp. Null when
+ * it uses syntax JS lacks. */
+export function compileJvmPattern(source        )                {
+  const lead = /^\(\?([a-z]+)\)/.exec(source);
+  const flags = new Set        ();
+  let body = source;
+  if (lead) {
+    body = source.slice(lead[0].length);
+    for (const f of lead[1]) {
+      if (f === 'i' || f === 'm' || f === 's') flags.add(f);
+      else if (f !== 'u') return null;
+    }
+  }
+  try { return new RegExp(body, [...flags].join('')); } catch { return null; }
+}
+
+// Sensitive query/parameter names in a URL-like value: signatures of pre-signed and SAS URLs.
+const SIGNATURE_PARAMS = 'sig|signature|x-amz-signature|x-amz-credential|x-amz-security-token|x-goog-signature|x-goog-credential';
+
+// A `name=value` credential parameter in any value (a JDBC or ODBC string, a JVM option, a query
+// string): `password`, `pwd`, `pass`, `apikey`, `accountkey`, `sas`, `sig` and the like, alone or as
+// the tail of a longer name (`-Djavax.net.ssl.keyStorePassword=`).
+const CREDENTIAL_PARAM = new RegExp(
+  '((?:^|[\\s;&,?:"\'(]|-D)(?:[\\w.-]*?(?:password|passwd|pwd|secret|token|api[_.-]?key|account[_.-]?key|access[_.-]?key|private[_.-]?key)|pass|sas|credential|'
+  + `${SIGNATURE_PARAMS})\\s*=\\s*)("[^"]*"|'[^']*'|[^;&,\\s"']*)`, 'gi');
+
+/** Strips credentials from a value: `name=value` credential parameters anywhere (JDBC
+ * `password=`/`pwd=`/`pass=`, `apiKey=`, signature parameters such as Azure SAS `sig=`), and, in a
+ * value that looks like a URL, userinfo (user:password@host) and the Oracle thin form
+ * (jdbc:oracle:thin:user/password@host). */
+export function stripUrlCredentials(value        )         {
+  const urlLike = /^[a-z][a-z0-9+.-]*:/i.test(value) || value.includes('://');
+  const withoutUserinfo = urlLike
+    ? value
+      .replace(/(:\/\/)[^/\s@?#,]*@/g, `$1${REDACTED}@`)
+      .replace(/^((?:[a-z][a-z0-9+.-]*:)+[^\s:/@]+\/)[^\s@]+@/i, `$1${REDACTED}@`)
+    : value;
+  return withoutUserinfo.replace(CREDENTIAL_PARAM, `$1${REDACTED}`);
+}
+
+/** The run's Spark Properties as the CLI reports them; null when the log recorded none. */
+export function buildEffectiveConf(app                     , options                       = {})                       {
+  const config = app?.config;
+  if (config == null) return null;
+  const defaultPattern = new RegExp(DEFAULT_SECRET_PATTERN, 'i');
+  const extraKeyPattern = new RegExp(EXTRA_SECRET_KEY_PATTERN, 'i');
+  const jobSource = config['spark.redaction.regex'] ?? null;
+  const jobPattern = jobSource != null ? compileJvmPattern(jobSource) : null;
+  // A job pattern this runtime cannot evaluate withholds every value rather than guess.
+  const jobPatternUsable = jobSource == null || jobPattern != null;
+  const userSource = options.userPattern ?? null;
+  const userPattern = userSource != null ? compileJvmPattern(userSource) : null;
+  if (userSource != null && userPattern == null) throw new Error(`invalid secret pattern: ${userSource}`);
+
+  const wanted = options.keys ? new Set(options.keys) : null;
+  const values                         = {};
+  const maskedKeys           = [];
+  const matches = (pattern               , key        , value        ) => (pattern?.test(key) ?? false) || (pattern?.test(value) ?? false);
+  for (const key of Object.keys(config).sort()) {
+    if (wanted && !wanted.has(key)) continue;
+    const value = config[key];
+    // Spark applies its redaction pattern to the key and the value.
+    const masked = !jobPatternUsable
+      || matches(defaultPattern, key, value) || extraKeyPattern.test(key) || matches(jobPattern, key, value) || matches(userPattern, key, value);
+    if (masked) maskedKeys.push(key);
+    else values[key] = stripUrlCredentials(value);
+  }
+  return {
+    schemaVersion: EFFECTIVE_CONF_SCHEMA_VERSION,
+    values,
+    maskedKeys,
+    absentKeys: wanted ? [...wanted].filter((k) => !(k in config)).sort() : [],
+  };
+}

package/vendor-core/efficiency-model.js

@@ -1,13 +1,14 @@
 // §5 Efficiency/wastage model. DESIGN SPIKE:
 // decomposes available compute-hours into driver-bound vs executor-bound waste, plus two floors.
 import { computeWallClock } from './wall-clock.js';
-import { computeTotalCores } from './core-count.js';
+import { computePeakConcurrentCores } from './core-count.js';
                                                                              
 
-export function computeEfficiencyModel({ app, stages, executorsAdded, runAggregates }   
+export function computeEfficiencyModel({ app, stages, executorsAdded, executorsRemoved, runAggregates }   
                            
                                
                                   
+                                    
                                       
  )   
                                 
@@ -17,10 +18,11 @@ export function computeEfficiencyModel({ app, stages, executorsAdded, runAggrega
                           
                                               
   {
-  // `app ?? {}`: computeTotalCores falls back to the executor core sum when resources is absent,
-  // and callers tolerate a null app (malformed logs); `app!` would crash on app.resources.
-  // Cast: computeTotalCores reads only totalCores, absent on ExecutorRemovedEvent, so the union mismatches.
-  const totalCores = computeTotalCores(app ?? {}, executorsAdded                                  );
+  // Capacity is the peak concurrent core count, the one the utilization and idle-cores detectors
+  // use, so the Unused core time tile matches the verdict's idle figure. Summing every addition
+  // (computeTotalCores) counts a replaced executor's cores alongside its replacement's.
+  // `app ?? {}`: callers tolerate a null app (malformed logs); `app!` would crash on app.resources.
+  const totalCores = computePeakConcurrentCores(app ?? {}, executorsAdded, executorsRemoved);
   const appDurationMs = (app?.endTime ?? 0) - (app?.startTime ?? 0);
   const wc = computeWallClock(app, stages                                                               );