sparkforensics-server 0.5.2 → 0.8.1

package/vendor-core/docs-config.js

@@ -60,11 +60,9 @@ export const METRIC_ANCHORS                         = {
 // section, and gating keeps that from becoming a dead "Learn more" link. docs-config.test.js
 // asserts this stays a subset of the real ids so it can't drift.
 export const KNOWN_DOC_ANCHORS              = new Set([
-  // Page sections
   '#intro', '#spark-architecture', '#memory-model', '#partitioning', '#joins',
   '#shuffle', '#data-formats', '#table-formats', '#caching', '#pyspark', '#aqe',
   '#cluster-config', '#anti-patterns', '#metrics', '#config',
-  // Bottleneck sections
   '#bottleneck-cold-start', '#bottleneck-failures', '#bottleneck-gc',
   '#bottleneck-job-failure-rate', '#bottleneck-retry-waste', '#bottleneck-shuffle',
   '#bottleneck-skew', '#bottleneck-slow-host', '#bottleneck-spill',
@@ -75,10 +73,8 @@ export const KNOWN_DOC_ANCHORS              = new Set([
   '#bottleneck-partition-sizing', '#bottleneck-speculation-waste',
   '#bottleneck-core-locality', '#bottleneck-caching-opportunity',
   '#bottleneck-autoscaling-churn', '#bottleneck-cache-utilization',
-  // Config-audit sections
   '#config-autoscale-bounds', '#config-memory-overhead', '#config-serializer',
   '#config-shuffle-service',
-  // Metric glossary entries
   ...Object.values(METRIC_ANCHORS),
 ]);
 

package/vendor-core/docs-content/detection/cache.md

@@ -5,4 +5,5 @@ join/union result recomputed by two or more executions, matched by plan
 shape) may be worth persisting between stages. Self-flags a confidence that scales with
 how many executions reuse the same relation: reuse is only inferred, from
 plan-scan identity across SQL executions, so confirm the reads really do
-hit the same data before you cache anything.
+hit the same data before you cache anything. The figure is the input those
+queries scanned and how many queries scanned it, not a time.

package/vendor-core/docs-content/detection/cfg.md

@@ -1,16 +1,24 @@
 ### `CFG`: Configuration audit {#cfg}
 
 Flags configuration settings that may cause reliability or efficiency
-problems, independent of any one stage's behavior. Four checks run:
+problems, independent of any one stage's behavior. Three checks run:
 
-- `spark.shuffle.service.enabled`: flagged when dynamic allocation is on
-  but the external shuffle service is off, since shuffle data won't survive
-  executor removal.
-- `spark.dynamicAllocation.minExecutors`/`maxExecutors`: with dynamic
-  allocation on, flagged when min exceeds max (reported on `minExecutors`)
-  or when no max is set.
+- `spark.dynamicAllocation.maxExecutors`: with dynamic allocation on,
+  flagged when the effective max is Spark's unbounded default, so the cluster
+  can grow without a cap.
 - `spark.serializer`: flagged when not set to Kryo (the default is the Java
-  serializer); `org.apache.spark.serializer.KryoSerializer` is faster and
-  produces smaller buffers.
-- `spark.executor.memoryOverhead`: flagged when set below max(384 MiB, 10%
-  of executor memory).
+  serializer), and only on a run with stages outside any SQL execution.
+  DataFrame and SQL shuffles and caches use Spark's own row format, so the
+  serializer only matters for RDD work.
+  `org.apache.spark.serializer.KryoSerializer` is faster and produces smaller
+  buffers.
+- `spark.executor.memoryOverhead`: flagged when set below the overhead Spark
+  computes by default, max(`spark.executor.minMemoryOverhead`, executor memory
+  times `spark.executor.memoryOverheadFactor`). The minimum is 384 MiB and the
+  factor 10% unless the run sets them; each setting counts only on a Spark
+  version that reads it (the factor from 3.3, the minimum from 4.0).
+
+Two settings are never flagged because Spark rejects them at startup, so no
+event log carries them: dynamic allocation with neither the external shuffle
+service, shuffle tracking, shuffle-block decommissioning nor a reliable
+shuffle storage plugin, and `minExecutors` above `maxExecutors`.

package/vendor-core/docs-content/detection/cold.md

@@ -4,4 +4,5 @@ The first stage waited more than 30 s for an executor. Keep a warm pool of
 executors, or, with dynamic allocation, raise
 `spark.dynamicAllocation.minExecutors`/`initialExecutors` so the app doesn't
 scale up from zero. With dynamic allocation off (`evidence.dynamicAllocation`
-is `off`) no executor-count property applies and `remediation` is empty.
+is `off`, which includes a run that never set it, as Spark defaults it off) no
+executor-count property applies and `remediation` is empty.

package/vendor-core/docs-content/detection/gc.md

@@ -2,6 +2,10 @@
 
 Tasks spend more than 10% of executor run time reclaiming memory. Reduce
 object creation: use primitive types, avoid UDFs, or raise executor memory.
-A stage with GC below 5% gets an informational note that executor memory
-may be over-provisioned, only on stages that take at least 0.5% of the run.
-Both need at least 10 s of executor run time on the stage.
+When the log carries no measured executor heap peaks, or the executor memory
+is unknown, so [`MEM`](#mem) cannot judge the heap, a stage with GC below 5%
+gets an informational note that executor memory may be over-provisioned, only
+on stages that take at least 0.5% of the run, and never on a stage that failed
+or lost tasks. Both need at least 10 s of executor run time on the stage. A
+stage with advice to raise executor memory (high GC, volume spill, a GC-bound
+straggler tail) gets no advice to lower it.

package/vendor-core/docs-content/detection/host.md

@@ -4,4 +4,14 @@ One executor is much slower than its peers, or carries most of the stage's
 task time or bytes. It may just hold data locality
 for its tasks or carry one heavy stage, rather than a hardware fault.
 Enable `spark.speculation` to relaunch a lagging task automatically. Only
-flagged on stages that take at least 0.5% of the run.
+flagged on stages that take at least 0.5% of the run. A task-time grade
+rests on the largest of N executor means, which a larger executor count reaches
+by chance more often, so the finding states `confidence: low` with the executor
+count and the task count.
+
+An executor whose tasks ran slower than the median executor's is flagged only
+when the gap is larger than the stage's own spread of task times and the
+executor's task count allow. The slowest of 75 executors tops the median by
+chance far more often than the slowest of 3, so the gap it needs grows with
+the number of executors. A stage whose per-task timings are missing flags no
+executor on task time.

package/vendor-core/docs-content/detection/mem.md

@@ -1,13 +1,24 @@
 ### `MEM`: Memory utilization {#mem}
 
 Executor memory or core capacity may be over- or under-provisioned: more
-than 50% of available core time ran no task, an executor's heap peaked above
-95% of its allocation, or it stayed below 70%. Some
-detail here needs `spark.eventLog.logStageExecutorMetrics=true` on the run
-being analyzed; without it, per-executor memory usage can't be broken down.
+than 50% of allocated core time ran no task, or the busiest executor's heap
+peak stayed below 70% of `spark.executor.memory`. Heap peaks come from the
+executor metrics Spark 3+ writes on every task end (and from stage executor
+metrics when `spark.eventLog.logStageExecutorMetrics=true`). Spark samples them
+at executor heartbeat, so a peak is a lower bound, and the heap-used metric
+counts uncollected garbage, so a peak near the limit is not reported as a
+memory risk. A log with no peaks (every task ended between two heartbeats, so
+each reports zeros, as in local mode) gets a note that executor memory sizing was
+not measured; it suggests `spark.eventLog.logStageExecutorMetrics=true` while that
+is off. A run-level finding that advises lowering executor memory (a heap under
+70%, idle allocated memory-time) keeps its figures but not that advice when any
+stage asks for more memory, and names those stages; a stage's low-GC note is
+dropped on a stage that asks for more. The
+idle-memory figure counts the executors' alive time, not the peak executor count
+over the whole run.
 Review `spark.executor.memory` and executor count if allocated memory sat
 largely idle over the run. That idle-memory variant self-flags a confidence
 that scales with how far the estimated waste sits past a 1.5x buffer: it
-estimates waste from allocated memory-time versus task run time (not
-measured heap usage). Check it against
+estimates waste from allocated memory-time versus task run time per
+executor core (not measured heap usage). Check it against
 the Spark UI before resizing anything.

package/vendor-core/docs-content/detection/part.md

@@ -2,4 +2,6 @@
 
 Shuffle partitions are too large, too uneven, or too few for the work. A
 single shuffle partition over 5 GB, for example, will OOM or spill heavily:
-repartition to break it up before the stage runs.
+repartition to break it up before the stage runs. The partition-skew rule
+words its advice the same way a `SKEW` finding does, including
+`evidence.aqeSkew`.

package/vendor-core/docs-content/detection/plan.md

@@ -1,6 +1,6 @@
 ### `PLAN`: Plan advisor {#plan}
 
-Flags patterns in the SQL execution plan worth reviewing. Four checks share
+Flags patterns in the SQL execution plan worth reviewing. Six checks share
 this tag:
 
 - Duplicate plan subtree: the same subtree recomputed more than once in the
@@ -9,16 +9,58 @@ this tag:
   when the repeat's stages take at least 0.5% of the run.
 - Small files: one plan node reads or writes more than 100 files averaging
   under 3 MB. Compact upstream output, or coalesce before writing.
-- Under-broadcast: the smaller side of a Sort Merge Join looks well under
-  the broadcast threshold; consider a `broadcast()` hint or raising
-  `spark.sql.autoBroadcastJoinThreshold`. When the effective threshold
-  (logged, else Spark's 10 MiB) already admits the smaller side
-  (`evidence.broadcastThreshold` is `notLimiting`), the threshold is not what
-  stopped the broadcast, so `remediation` is empty and the advice is a hint or
-  table statistics.
+- Nested loop join: a `BroadcastNestedLoopJoin` or `CartesianProduct` whose
+  output has at least 1,000,000 rows (the rows a `CartesianProduct`'s condition
+  kept, when it has one). A `BroadcastNestedLoopJoin` must also
+  produce at least 10 times the rows of its larger input; a `CartesianProduct`
+  is judged on its output alone, because it re-reads each input once per
+  partition of the other side, so the input row counts the executors report
+  are not row counts. Spark plans these when a join has no equi-join key. The
+  finding carries the join condition and is graded by the time of the stages
+  that run the join. The advice is to add an equi-join key, to bucket the range
+  of a range join and join on the bucket, or to confirm that a cross join is
+  intended. It needs the executors' `number of output rows` metrics, so a join
+  with no reported row counts is not flagged.
+- Row-at-a-time Python UDF: a plan runs `BatchEvalPython`, which pickles
+  every row to a Python worker and back. Flagged when its stages sent at
+  least 64 MiB to the workers (`data sent to Python workers`) and ran for at
+  least 30 seconds together; the finding reports the bytes sent and returned
+  and the stage time, and is informational. `ArrowEvalPython` (Arrow-optimized
+  and pandas UDFs) is never flagged. The advice is
+  `spark.sql.execution.pythonUDF.arrow.enabled=true` (Spark 3.4 and later;
+  already the default from 4.2, where the UDF may have opted out with
+  `useArrow=False`, take no arguments, or have been created before the property
+  was set) or a pandas UDF. These floors are conservative guesses,
+  not tuned against real workloads. A plan that reports no
+  `data sent to Python workers` value is skipped.
+- Under-broadcast: a side of a Sort Merge Join that its join type can
+  broadcast (`evidence.buildSide`, sized by the finding's value) was not
+  broadcast although it is small enough: it is over a small effective
+  threshold (`evidence.broadcastThreshold` is `limits`), or automatic
+  broadcast is disabled with `-1` (`disabled`); consider a `broadcast()` hint
+  or setting the threshold property. The size is
+  the shuffle's `data size`, the metric adaptive execution compares at runtime.
+  The threshold is `spark.sql.adaptive.autoBroadcastJoinThreshold` when the
+  plan is adaptive and that property is set, else
+  `spark.sql.autoBroadcastJoinThreshold` (the query's own setting, else the
+  logged one, else Spark's 10 MiB). A full outer join never fires, and a side
+  under 1 MiB or over the over-broadcast limit is skipped. When the effective
+  threshold already admits the side (`evidence.broadcastThreshold` is
+  `notLimiting`), the threshold is not what stopped the broadcast, so
+  `remediation` is empty and the advice is a hint or table statistics (for an
+  adaptive plan, `spark.sql.adaptive.nonEmptyPartitionRatioForBroadcastJoin`:
+  AQE keeps a side with a low share of non-empty partitions out of a broadcast
+  whatever its size). Both
+  sides need a shuffle `data size` of their own, so a join over another join's
+  output is skipped, and so is a join whose only buildable side is larger than
+  the other side, since broadcasting it saves nothing. The build side is
+  always the smaller one and `evidence.largerSideBytes` is the other side.
 - Over-broadcast: a broadcast exceeds the 1 GB threshold; check for a
-  misapplied broadcast hint or a misconfigured
-  `spark.sql.autoBroadcastJoinThreshold`. When the effective threshold is
-  below the broadcast or auto-broadcast is disabled (`evidence.broadcastThreshold`
-  is `notLimiting` or `disabled`), a hint forced it: remove the hint, and
-  `remediation` is empty.
+  misapplied broadcast hint or a misconfigured threshold property. Under
+  adaptive execution a broadcast planned up front is admitted by
+  `spark.sql.autoBroadcastJoinThreshold` and one converted at runtime by
+  `spark.sql.adaptive.autoBroadcastJoinThreshold` (when set), so the finding
+  names whichever of them admits the broadcast. When every applicable
+  threshold is below the broadcast or disabled (`evidence.broadcastThreshold`
+  is `notLimiting`, or `disabled` when all are disabled), a hint forced it:
+  remove the hint, and `remediation` is empty.

package/vendor-core/docs-content/detection/shfl.md

@@ -4,8 +4,14 @@ Tasks move a large amount of intermediate data between stages. Raise
 `spark.sql.shuffle.partitions`, or add a broadcast join. The partition-count
 advice follows the effective conf (`evidence.partitions`): `raise` when the
 property limits the stage, `sufficient` when the tasks already read a good
-size each, `aqeCoalesced` when AQE merged the partitions (lower
-`spark.sql.adaptive.advisoryPartitionSizeInBytes`), and `ownPartitioning` when
-the property is already high enough (the stage's own `repartition(n)` or RDD
-parallelism limits it). Only flagged on stages that take at least 0.5% of the
-run.
+size each, `aqeCoalesced` when AQE merged the partitions of a stage in a SQL
+execution (lower `spark.sql.adaptive.advisoryPartitionSizeInBytes`), and
+`ownPartitioning` when the property is already high enough (the stage's own
+`repartition(n)` or RDD parallelism limits it, as for any stage outside a SQL
+execution). With AQE coalescing on,
+`spark.sql.adaptive.coalescePartitions.initialPartitionNum` replaces the
+property as the starting count when it is set, and a stage that ran more tasks
+than that, with a skew split in the final plan or no final plan to rule one
+out, is not blamed on its own `repartition(n)`: the advice states the task and
+configured counts and names `repartition(n)` only when the plan holds one the
+job wrote. Only flagged on stages that take at least 0.5% of the run.

package/vendor-core/docs-content/detection/skew.md

@@ -8,7 +8,54 @@ stage. The fix depends on what the stage reads, which the finding's
 already has it; otherwise salt the key or repartition on a better key. A
 stage that reads files with uneven sizes (`inputScan`) gets compaction of
 small files or a lower `spark.sql.files.maxPartitionBytes`. Any other stage
-(`other`) gets the salting advice and no conf. Flagged when P95 task time
+(`other`) gets the salting advice and no conf, as a `code` entry in
+`remediation`. Flagged when P95 task time
 (the longest task, on a stage with fewer than 20 tasks) exceeds 3x the median
 and the recoverable tail is at least 0.5% of the run. The median is the
 textbook one: on an even task count, the mean of the two middle values.
+
+Skew is a statement about data volume, as in Spark's UI and AQE. Each stage's
+slow tasks (over 3x the median task) are compared with the median task's input
+plus shuffle-read bytes and records. If run time scaled with that volume, the
+share of the slow tasks' extra time it accounts for is the data share; at 50% or
+more the tail is skew (`evidence.cause` is `data`, `evidence.dataRatio` is the
+median slow task's volume over the median task's). A tail that mostly is not data
+is reported as `STRAG` with its cause. Ratios are taken only against a median task that read at least
+1 MiB or 1,000 records, so a median of a few bytes never prints a huge multiple. When the median task read less than that floor (almost nothing),
+there is no ratio to take (`evidence.dataRatio` is absent, and the advice says the
+median task read almost nothing): GC and fetch wait claim their share of the slow
+tasks' extra time first, and a slow task that read data (at least 1 MiB or 1,000
+records) owns what is left. When the data share is under 50%, the cause is the largest
+of GC, fetch wait, host and what none of them accounts for. With no data volume
+to compare, what none of them accounts for is labelled `unattributed` (no share is
+reported) instead of `unexplained`; GC, fetch wait or host still win when larger. A
+stage with no tail attribution is `unattributed` too. An `unattributed` tail is
+reported as `STRAG` only, never as `SKEW`.
+
+With AQE skew-join handling on, a `shuffleJoin` stage's advice says why
+handling did or did not act on that stage's join, read from the execution's
+final plan and its effective conf. `evidence.aqeSkew` records the case:
+
+| `aqeSkew` | What the final plan and conf show | Advice |
+|---|---|---|
+| `split` | The join is marked `skew=true` and a shuffle read says `skewed` | AQE already split the skewed partitions, so what remains is not join skew: look at GC, a slow host or an expensive key. When the tail is data (`evidence.cause` is `data`), the slow tasks still read far more data than the median task, and the advice is the key remedy. |
+| `evenReads` | One join side shuffles next to nothing, so the task read is the other side's, and the largest task read is under twice the median | The slow tail is not partition-size skew: look at GC, a slow host or an expensive key. Never given when `evidence.cause` is `data`, which counts input and records as well as shuffle-read bytes. |
+| `belowThreshold` | The largest task read is under `spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes` (256 MB by default), which bounds every partition of both sides; or one side's read is at least twice the median but not far enough over it for `spark.sql.adaptive.skewJoin.skewedPartitionFactor` (5) | Lower the threshold or the factor for the query. |
+| `planShape` | An aggregate, window or other operator sits between the join and its shuffle | AQE splits only a shuffle that feeds the join directly: salt the key. |
+| `userRepartition` | The shuffle under the join is a `repartition` or `rebalance` in the job's code | AQE leaves a shuffle you asked for alone: drop it, or salt the key. |
+| `joinType` | The join type does not let AQE split the skewed side: neither side of a full outer join, only the left side of a left outer, left semi or left anti join, only the right side of a right outer join | Join the hot key on its own and union the results, or salt the key. Swapping the sides does not help: the null-supplying or filtering table stays unsplittable. |
+| `extraShuffle` | An aggregate, window or join above this join needs its partitioning, so a split would add a shuffle | Set `spark.sql.adaptive.forceOptimizeSkewedJoin` to `true` when that shuffle costs less than the tail (Spark 3.3 and later; before that, salt the key). |
+| `notSplit` | Nothing above explains it | Salt the key. |
+
+The stage's join is the one whose plan node ran in that stage; a plan whose
+nodes carry no stage ids is read as one join only when the stage's shuffle read
+matches what the join's two shuffles wrote. A task reads one partition of each
+join side, so the stage's largest task read covers both sides. Spark judges each
+side's partitions against that side's own median, which the log does not give, so
+the finding calls the reads even or under the factor only when one side shuffles
+at most 5% of the join's bytes; otherwise it can only show a partition is under
+the absolute threshold (a task's read bounds every partition in it). The finding
+does not say which side is skewed; for a join type that splits only one side it
+names both possibilities. A run whose final plan is missing or ties no join to
+the stage, or whose coalesced read could be many small partitions, keeps the
+general advice.

package/vendor-core/docs-content/detection/slow.md

@@ -2,8 +2,10 @@
 
 A stage ran for 15 minutes or more and no slow host was flagged on it. It
 can appear alongside other findings on the same stage. On a stage that reads a
-shuffle, often a partition-count problem: raise parallelism via
-`spark.sql.shuffle.partitions` or `spark.default.parallelism`, or check for a
+shuffle, often a partition-count problem: raise `spark.sql.shuffle.partitions`
+when the stage ran that many tasks, lower
+`spark.sql.adaptive.advisoryPartitionSizeInBytes` when AQE coalesced them, or
+raise the stage's own `repartition(n)` or RDD parallelism; or check for a
 large per-task data volume driving heavy shuffle and spill. On a stage that
 reads input files and no shuffle, check input file sizes and lower
 `spark.sql.files.maxPartitionBytes`. `evidence.reads` says which case

package/vendor-core/docs-content/detection/spec.md

@@ -4,4 +4,9 @@ Speculative task attempts used a lot of executor time without confirming a
 genuine straggler. Self-flags a confidence that scales with how far the
 wasted time sits past the threshold. If task durations are just naturally
 variable rather than genuine stragglers, tune
-`spark.speculation.multiplier`/`spark.speculation.quantile`.
+`spark.speculation.multiplier`/`spark.speculation.quantile`. The
+recommendation names the run's effective values: Spark relaunches a task that
+runs over the multiplier times the median once the quantile of the stage's
+tasks has finished, which is 1.5x and 75% before Spark 4.0 and 3x and 90% from
+4.0 unless the job sets them. A speculative-attempt `straggler` and a
+`slowHost` finding word their speculation advice the same way.

package/vendor-core/docs-content/detection/spill.md

@@ -4,7 +4,10 @@ Tasks are writing data out of memory, which slows execution. Two spill
 patterns get flagged differently: skew spill, where a few heavy tasks spill
 while most don't (rebalance partitioning), and volume spill, where most
 tasks spill because the data genuinely exceeds available memory (add
-partitions or executor memory). `spark.sql.shuffle.partitions` is suggested
-only for a stage that reads a shuffle; `evidence.reads` says what the stage
-reads (`shuffle`, `input` or `other`). Only flagged on stages that take at
+partitions or executor memory). Partition-count advice is given only for a
+stage that reads a shuffle; `evidence.reads` says what the stage reads
+(`shuffle`, `input` or `other`). It names what sized the stage:
+`spark.sql.shuffle.partitions` when the stage ran that many tasks,
+`spark.sql.adaptive.advisoryPartitionSizeInBytes` when AQE coalesced them, and
+the stage's own `repartition(n)` or RDD parallelism otherwise. Only flagged on stages that take at
 least 0.5% of the run.

package/vendor-core/docs-content/detection/strag.md

@@ -1,8 +1,21 @@
 ### `STRAG`: Straggler tasks {#strag}
 
-A few tasks run much slower than the rest of their stage. Rule out a GC
-pause or a slow shuffle fetch before assuming a hardware issue. If uneven
-data is the cause, the advice is the one a skew finding gives for the same
-stage (`evidence.origin`, see `SKEW`): AQE skew-join handling for a shuffle
-feeding a join, file sizes for a scan, salting otherwise. Only flagged on
-stages that take at least 0.5% of the run.
+A few tasks run much slower than the rest of their stage for a reason other
+than reading more data (a data-driven tail is `SKEW`). `evidence.cause` names
+what the slow tasks' extra time went to, taken in this order so no millisecond
+counts twice: data volume, then GC time over the median task's, then shuffle
+fetch wait over the median task's, then one host that holds most of the slow
+tasks out of proportion to its share of the stage (`evidence.host`). What none of
+them accounts for is `unexplained` (`unattributed` with no data volume to compare); an
+`unexplained` tail whose slow tasks read at least twice the median task's data says so
+rather than "no more data", and keeps the `SKEW` data advice when data is its largest share; `evidence.cpuPct` (the slow tasks' CPU time
+over their run time) says whether those tasks mostly waited or were busy. With
+no data volume to compare, what is left is `unattributed` when it is the largest share, as is a stage
+with no tail attribution: the advice names no fix, only the slow tasks' input sizes, GC time and hosts to check.
+A stage is flagged when more than 5% of its tasks run over 4x the median (2.5%
+when the tail clears 0.5% of the run), a speculative task ran, or the `SKEW`
+duration test holds on a tail that is not data. A stage whose tail is data but
+which `SKEW` does not flag by duration keeps a `STRAG` finding with cause `data`. A finding only the `SKEW` duration
+test admits reports that test's ratio (`metric` `P95/median` or `max/median`, `unit` `ratio`)
+instead of a share.
+Stages under 0.5% of the run are skipped unless the `SKEW` duration test holds.

package/vendor-core/docs-content/detection/tiny.md

@@ -1,6 +1,12 @@
 ### `TINY`: Tiny tasks {#tiny}
 
 Many very short tasks add scheduling overhead out of proportion to the work
-each one does. Repartition to fewer, larger tasks; a stage that reads a shuffle
-(`evidence.reads` is `shuffle`) can also lower `spark.sql.shuffle.partitions`.
-Only flagged on stages that take at least 0.5% of the run.
+each one does. Repartition to fewer, larger tasks. On a stage that reads a
+shuffle (`evidence.reads` is `shuffle`) the advice names what sized it: lower
+`spark.sql.shuffle.partitions` when the stage ran that many tasks; when AQE
+coalesced them and still kept tiny ones, set
+`spark.sql.adaptive.coalescePartitions.parallelismFirst` to `false` (AQE then
+targets `spark.sql.adaptive.advisoryPartitionSizeInBytes` rather than
+`defaultParallelism` tasks) and then raise the advisory size, which alone changes
+nothing while `parallelismFirst` is on; otherwise lower the
+`repartition(n)` or RDD partition count in the code. Only flagged on stages that take at least 0.5% of the run.

package/vendor-core/docs-content/detection/util.md

@@ -1,4 +1,8 @@
 ### `UTIL`: Low utilization {#util}
 
-Allocated executors sit idle for a large share of the application run.
-Consider a smaller cluster, or enable dynamic allocation.
+Allocated executors sit idle for a large share of the application run: under
+60% of the core time the run allocated (cores times the time each executor was
+alive) ran a task. Consider a smaller cluster. With dynamic allocation off,
+either lower `spark.executor.instances` or enable dynamic allocation; the two are
+alternatives. With it on, lower `spark.dynamicAllocation.maxExecutors`, and
+`spark.dynamicAllocation.minExecutors` too when the run sets it above 0.

package/vendor-core/effective-conf.js

@@ -1,8 +1,7 @@
 // The effective Spark configuration of a run for the CLI's JSON output, so a caller can check
-// that a --conf overlay took effect. Every key is listed; a value is withheld when the key or the
-// value matches a secret pattern, and credentials inside URL-like values are stripped. A withheld
-// key is shown as present with no value, and no derivative of the value (no hash, length or
-// prefix) is emitted.
+// that a --conf overlay took effect. A withheld key is shown as present with no value, and no
+// derivative of the value (no hash, length or prefix) is emitted. Patterns and output shape:
+// docs-site/user-guide/getting-started/ci-and-automation.md, "Effective conf".
                                                
 
 export const EFFECTIVE_CONF_SCHEMA_VERSION = 1;
@@ -50,16 +49,14 @@ export function compileJvmPattern(source        )                {
 // Sensitive query/parameter names in a URL-like value: signatures of pre-signed and SAS URLs.
 const SIGNATURE_PARAMS = 'sig|signature|x-amz-signature|x-amz-credential|x-amz-security-token|x-goog-signature|x-goog-credential';
 
-// A `name=value` credential parameter in any value (a JDBC or ODBC string, a JVM option, a query
-// string): `password`, `pwd`, `pass`, `apikey`, `accountkey`, `sas`, `sig` and the like, alone or as
-// the tail of a longer name (`-Djavax.net.ssl.keyStorePassword=`).
+// A `name=value` credential parameter in any value (JDBC or ODBC string, JVM option, query string),
+// alone or as the tail of a longer name (`-Djavax.net.ssl.keyStorePassword=`).
 const CREDENTIAL_PARAM = new RegExp(
   '((?:^|[\\s;&,?:"\'(]|-D)(?:[\\w.-]*?(?:password|passwd|pwd|secret|token|api[_.-]?key|account[_.-]?key|access[_.-]?key|private[_.-]?key)|pass|sas|credential|'
   + `${SIGNATURE_PARAMS})\\s*=\\s*)("[^"]*"|'[^']*'|[^;&,\\s"']*)`, 'gi');
 
-/** Strips credentials from a value: `name=value` credential parameters anywhere (JDBC
- * `password=`/`pwd=`/`pass=`, `apiKey=`, signature parameters such as Azure SAS `sig=`), and, in a
- * value that looks like a URL, userinfo (user:password@host) and the Oracle thin form
+/** Strips credentials from a value: `name=value` credential parameters anywhere, and, in a value
+ * that looks like a URL, userinfo (user:password@host) and the Oracle thin form
  * (jdbc:oracle:thin:user/password@host). */
 export function stripUrlCredentials(value        )         {
   const urlLike = /^[a-z][a-z0-9+.-]*:/i.test(value) || value.includes('://');

package/vendor-core/efficiency-model.js

@@ -1,12 +1,13 @@
-// §5 Efficiency/wastage model. DESIGN SPIKE:
-// decomposes available compute-hours into driver-bound vs executor-bound waste, plus two floors.
-import { computeWallClock } from './wall-clock.js';
+// Efficiency/wastage model: decomposes available compute-hours into driver-bound vs executor-bound
+// waste, plus the zero-skew makespan floor.
+import { stageWindows } from './wall-clock.js';
+import { allocatedCoreMs, allocatedCoreMsWithin } from './allocation.js';
 import { computePeakConcurrentCores } from './core-count.js';
                                                                              
 
 export function computeEfficiencyModel({ app, stages, executorsAdded, executorsRemoved, runAggregates }   
                            
-                               
+                                                                      
                                   
                                     
                                       
@@ -18,23 +19,25 @@ export function computeEfficiencyModel({ app, stages, executorsAdded, executorsR
                           
                                               
   {
-  // Capacity is the peak concurrent core count, the one the utilization and idle-cores detectors
-  // use, so the Unused core time tile matches the verdict's idle figure. Summing every addition
-  // (computeTotalCores) counts a replaced executor's cores alongside its replacement's.
-  // `app ?? {}`: callers tolerate a null app (malformed logs); `app!` would crash on app.resources.
-  const totalCores = computePeakConcurrentCores(app ?? {}, executorsAdded, executorsRemoved);
-  const appDurationMs = (app?.endTime ?? 0) - (app?.startTime ?? 0);
-  const wc = computeWallClock(app, stages                                                               );
+  // Capacity is the run's allocation (cores x time alive, metrics.allocation.coreHours), the one the
+  // utilization and idle-cores detectors use, so the Unused core time tile matches the verdict's
+  // idle figure. Peak concurrent cores times the whole run would count cores the run never held
+  // under dynamic allocation. Null without executor cores: no capacity, so no figure.
+  const input = { app, stages, executors: { added: executorsAdded, removed: executorsRemoved } };
+  const allocatedMs = allocatedCoreMs(input);
+  // Split by where the allocated cores were held, from the same alive intervals: inside the union
+  // of the stages' windows, or outside it (startup, gaps between stages, after the last stage).
+  // Executors that join after startup hold no cores there, so spreading the allocation evenly
+  // over the run would bill the driver for cores that never existed.
+  const allocatedActiveCoreMs = allocatedMs == null ? 0 : allocatedCoreMsWithin(input, stageWindows(stages)) ?? 0;
 
-  const availableComputeHours = totalCores * (appDurationMs / 3600000);
+  const availableComputeHours = (allocatedMs ?? 0) / 3600000;
 
-  // Driver-bound: cores idle during startup + gaps + idle (no active stage).
-  const driverIdleMs = wc.startup + wc.gaps + wc.idle;
-  const driverWasteHours = totalCores * (driverIdleMs / 3600000);
+  // Driver-bound: cores held while no stage was active (startup, gaps, idle).
+  const driverWasteHours = Math.max(0, (allocatedMs ?? 0) - allocatedActiveCoreMs) / 3600000;
 
   // Executor-bound: allocated capacity beyond busy cores during stagesActive. Tasks only run in
   // active windows, so whole-run busyCoreMs already lives inside stagesActive.
-  const allocatedActiveCoreMs = totalCores * wc.stagesActive;
   const busyCoreMs = runAggregates?.busyCoreMs ?? 0;
   const executorWasteHours = Math.max(0, allocatedActiveCoreMs - busyCoreMs) / 3600000;
 
@@ -44,6 +47,11 @@ export function computeEfficiencyModel({ app, stages, executorsAdded, executorsR
   let totalTaskMs = 0;
   const perStage = runAggregates?.perStage ?? {};
   for (const k of Object.keys(perStage)) totalTaskMs += perStage[k].totalTaskDurationSum;
+  // The floor is the least time the work takes on the widest the cluster ever was: peak concurrent
+  // cores (not the allocation), summing every addition (computeTotalCores) would count a replaced
+  // executor's cores alongside its replacement's. `app ?? {}`: callers tolerate a null app
+  // (malformed logs); `app!` would crash on app.resources.
+  const totalCores = computePeakConcurrentCores(app ?? {}, executorsAdded, executorsRemoved);
   const floorZeroSkewMs = totalCores > 0 ? totalTaskMs / totalCores : 0;
 
   let dominantWaste                               = null;

package/vendor-core/etl-phases.js

@@ -1,5 +1,4 @@
-// §7 ETL-phase time attribution (Onehouse Spark Analyzer). Heuristic: classify each stage by
-// byte-flow shape. A stage can be both Transform and Load, so buckets overlap and need not sum to
+// ETL-phase time attribution. Heuristic: classify each stage by byte-flow shape. A stage can be both Transform and Load, so buckets overlap and need not sum to
 // wall-clock. Storage-format-aware attribution (Hudi/Delta/Iceberg) isn't portable, out of scope.