sparkforensics-mcp 0.2.2 → 0.2.4

package/vendor-core/docs-config.js

@@ -4,15 +4,27 @@ import { detectorCatalog } from './detectors.js';
 // serves the tuning reference, relative to the app's origin.
 export const DOCS_BASE_DIR         = 'docs/tuning-reference';
 
+// Bottleneck sub-anchors that are sections of another entry's page, not pages of their own.
+// Most live on a sibling bottleneck page; autoscaling-churn and cache-utilization live on the
+// cluster-config and memory-model chapters. Keep in sync with spark-tuning-reference's anchor-map.
+const SUB_ANCHOR_PAGES                         = {
+  'bottleneck-stage-shape': 'bottleneck-skew',
+  'bottleneck-stage-slowness': 'bottleneck-slow-host',
+  'bottleneck-partition-sizing': 'bottleneck-shuffle',
+  'bottleneck-speculation-waste': 'bottleneck-straggler',
+  'bottleneck-core-locality': 'bottleneck-utilization',
+  'bottleneck-caching-opportunity': 'bottleneck-utilization',
+  'bottleneck-autoscaling-churn': 'cluster-config',
+  'bottleneck-cache-utilization': 'memory-model',
+};
+
 // Some anchors are in-page fragments on another entry's page: config-audit sub-findings and
-// metric-glossary entries live on the 'config'/'metrics' pages, and the "stage-*" sub-anchors
-// live on their owning bottleneck's page. Keep in sync with spark-tuning-reference's anchor-map.
+// metric-glossary entries live on the 'config'/'metrics' pages, and SUB_ANCHOR_PAGES lists the
+// bottleneck sub-anchors.
 export function pageForAnchor(anchor        )         {
   if (anchor.startsWith('metric-')) return 'metrics';
   if (anchor.startsWith('config-')) return 'config';
-  if (anchor === 'bottleneck-stage-shape') return 'bottleneck-skew';
-  if (anchor === 'bottleneck-stage-slowness') return 'bottleneck-slow-host';
-  return anchor;
+  return SUB_ANCHOR_PAGES[anchor] ?? anchor;
 }
 
 // Build a docs URL for an anchor like '#bottleneck-skew'. The leading '#' is
@@ -45,7 +57,7 @@ export const METRIC_ANCHORS                         = {
   'stage-duration': '#metric-stage-duration',
 };
 
-// Allowlist of every anchor that exists in the committed tuning-reference markdown. DocsLink
+// Allowlist of every anchor that exists in the generated tuning-reference markdown. DocsLink
 // renders a link only for anchors here: a detector may declare a docAnchor for an unwritten
 // section, and gating keeps that from becoming a dead "Learn more" link. docs-config.test.js
 // asserts this stays a subset of the real ids so it can't drift.
@@ -62,6 +74,9 @@ export const KNOWN_DOC_ANCHORS              = new Set([
   '#bottleneck-memory-utilization', '#bottleneck-broadcast-sizing',
   '#bottleneck-duplicate-plan-subtree', '#bottleneck-small-files',
   '#bottleneck-stage-shape', '#bottleneck-stage-slowness',
+  '#bottleneck-partition-sizing', '#bottleneck-speculation-waste',
+  '#bottleneck-core-locality', '#bottleneck-caching-opportunity',
+  '#bottleneck-autoscaling-churn', '#bottleneck-cache-utilization',
   // Config-audit sections
   '#config-autoscale-bounds', '#config-memory-overhead', '#config-serializer',
   '#config-shuffle-service',
@@ -69,7 +84,7 @@ export const KNOWN_DOC_ANCHORS              = new Set([
   ...Object.values(METRIC_ANCHORS),
 ]);
 
-// True when `anchor` resolves to a real section in the committed
+// True when `anchor` resolves to a real section in the generated
 // tuning-reference markdown.
 export function isKnownDocAnchor(anchor         )          {
   return KNOWN_DOC_ANCHORS.has(String(anchor));
@@ -109,10 +124,21 @@ export function docAnchorForType(type        )                     {
   return anchor && isKnownDocAnchor(anchor) ? anchor : undefined;
 }
 
+/** The docAnchor every finding in `findings` carries, or undefined when they disagree or any lacks
+ * one. For a badge standing for several findings (a widget header, a grouped row) whose type-level
+ * docAnchorForType can't pick one: configAudit's sub-checks each stamp their own anchor. */
+export function sharedDocAnchor(findings                                       )                     {
+  const anchors = new Set(findings.map((f) => f.docAnchor));
+  if (anchors.size !== 1) return undefined;
+  const [anchor] = anchors;
+  return anchor;
+}
+
 // Resolves a docAnchorForType() result to the tuning-doc file slug under docs-content/tuning/.
 // Reuses pageForAnchor's sub-anchor resolution so a sub-anchor like bottleneck-stage-shape maps
 // to its owning page (skew.md), not a stage-shape.md that never exists. Returns null for anchors
-// with no bottleneck tuning doc (metric-/config-prefixed, or a page section like #memory-model).
+// with no bottleneck tuning doc (metric-/config-prefixed, a page section like #memory-model, or a
+// sub-anchor hosted on a chapter like #bottleneck-autoscaling-churn).
 export function tuningDocSlugForAnchor(anchor        )                {
   const bare = String(anchor).replace(/^#/, '');
   const page = pageForAnchor(bare);

package/vendor-core/docs-content/chapters/03-memory-model.md

@@ -58,6 +58,45 @@ When enabling off-heap memory, always pair `spark.memory.offHeap.enabled=true` w
 
 > **PySpark:** if you need PySpark's own memory bounded rather than folded silently into the overhead, set `spark.executor.pyspark.memory` explicitly, though its enforcement relies on Python's `resource` module and won't work on Windows and won't actually limit anything on macOS[^4].
 
+## Cache utilization {#bottleneck-cache-utilization}
+
+<span class="tag">CSTOR</span>
+
+Spark's event log carries no block-access or hit-rate events, so cache utilization is read
+from periodic per-RDD storage snapshots instead: how much of a persisted RDD is actually
+resident where it was asked to be.
+
+### How it's detected
+
+A persisted RDD, one whose storage level requests memory and/or disk with at least one
+partition actually cached, is evaluated on two independent measures and can register on
+both at once:
+
+| Signal | Info | Warning |
+|---|---|---|
+| Cached ratio (cached partitions ÷ total partitions) | < 90% | < 50% |
+| Disk ratio (disk bytes ÷ (memory bytes + disk bytes)), `MEMORY_AND_DISK*` levels only | > 15% | > 40% |
+
+Both ratios come from a point-in-time storage snapshot taken at stage-submission events,
+not a runtime block-access count, so confirm against the Spark UI's Storage tab before
+acting. Confidence scales with the RDD's partition count: below 10 partitions the ratio
+is noisy enough to call low confidence, at 50 or more it's high.
+
+### Why it matters
+
+A partially cached RDD still pays the recomputation cost for its evicted share on every
+later read, undercutting the reason it was cached in the first place. Disk spillover
+under a `MEMORY_AND_DISK*` level keeps that data around instead of forcing a recompute,
+but a disk read is still far slower than serving it out of memory.
+
+### How to fix it
+
+- Increase executor memory, or reduce the size of the dataset being cached, so more of it
+  fits in the storage region without eviction.
+- If the RDD is only cached for the occasional narrow scan, weigh whether persisting it
+  at all is worth the partial-cache overhead, versus recomputing the slice you actually
+  need.
+
 ## Sources
 
 [^1]: [Task Memory Management in Spark](https://raw.githubusercontent.com/spoddutur/spark-notes/master/task_memory_management_in_spark.md)

package/vendor-core/docs-content/chapters/11-cluster-config.md

@@ -160,6 +160,46 @@ the target executor count dynamic allocation would otherwise compute[^3]. Raise
 so proportionally reduces concurrent task slots per executor
 (`spark.executor.cores / spark.task.cpus`)[^3].
 
+## Autoscaling churn {#bottleneck-autoscaling-churn}
+
+<span class="tag">CHRN</span> <span class="tag">EXPERIMENTAL</span>
+
+Dynamic allocation adds executors once tasks back up and frees them once they go
+idle[^6]. Churn is a narrower failure mode inside that same mechanism: executors that get
+stood up and torn down again before they've done much useful work, cycling through
+provisioning and JVM startup cost repeatedly instead of running tasks.
+
+### How it's detected
+
+An executor counts as short-lived when its lifetime, from its `ExecutorAdded` event to
+either its `ExecutorRemoved` event or the end of the run if it was never removed, is
+under 2 minutes. This is evaluated once a run has added at least 5 executors, a floor
+that keeps a two- or three-executor job from registering on ordinary scale-down.
+
+| Signal | Warning | Critical |
+|---|---|---|
+| Share of added executors that are short-lived (< 2 min lifetime) | > 30% | > 60% |
+
+The 2-minute lifetime cutoff and the 30%/60% split are unvalidated heuristics rather than
+figures published in Spark's own documentation or benchmarks, so treat a finding here as
+a prompt to look at the executor timeline rather than a calibrated verdict.
+
+### Why it matters
+
+Every short-lived executor pays the same provisioning and JVM-startup cost as a
+long-lived one, but returns little task work in exchange. A run that keeps flapping
+between scaling up and down spends more of its wall-clock time paying that repeated
+overhead than one that settles into a stable executor count.
+
+### How to fix it
+
+- Raise `spark.dynamicAllocation.executorIdleTimeout` (default 60s[^3]) so an executor
+  survives a brief lull instead of being released the moment it goes idle, only to be
+  requested again shortly after.
+- Widen the gap between `spark.dynamicAllocation.minExecutors` and `.maxExecutors`[^3]:
+  bounds set too close together force Spark to repeatedly add and remove executors to
+  track small fluctuations in the task backlog instead of settling into a stable range.
+
 ## Sources
 
 [^1]: [Distribution of Executors, Cores and Memory for a Spark Application](https://raw.githubusercontent.com/spoddutur/spark-notes/master/distribution_of_executors_cores_and_memory_for_spark_application.md)

package/vendor-core/docs-content/detection/gc.md

@@ -2,3 +2,5 @@
 
 Tasks spend an unusually large share of time reclaiming memory. Reduce
 object creation: use primitive types, avoid UDFs, or raise executor memory.
+A stage with very little GC gets an informational note that executor memory
+may be over-provisioned, only on stages that take at least 0.5% of the run.

package/vendor-core/docs-content/detection/host.md

@@ -2,4 +2,5 @@
 
 One executor is much slower than its peers. It may just hold data locality
 for its tasks or carry one heavy stage, rather than a hardware fault.
-Enable `spark.speculation` to relaunch a lagging task automatically.
+Enable `spark.speculation` to relaunch a lagging task automatically. Only
+flagged on stages that take at least 0.5% of the run.

package/vendor-core/docs-content/detection/plan.md

@@ -4,7 +4,9 @@ Flags patterns in the SQL execution plan worth reviewing. Four checks share
 this tag:
 
 - Duplicate plan subtree: the same subtree recomputed more than once in the
-  plan.
+  plan. When the repeats have the same shape but different filters, columns
+  or tables, the finding stays informational and claims no time. Only flagged
+  when the repeat's stages take at least 0.5% of the run.
 - Small files: reading an excessive number of small files.
 - Under-broadcast: the smaller side of a Sort Merge Join looks well under
   the broadcast threshold; consider a `broadcast()` hint or raising

package/vendor-core/docs-content/detection/shape.md

@@ -2,4 +2,5 @@
 
 The stage has an inefficient task count, output shape, or task-to-stage
 balance: for example, one straggler task taking a large fraction of the
-stage's wall-clock time.
+stage's wall-clock time. A too-low task count is only flagged on stages that
+take at least 0.5% of the run.

package/vendor-core/docs-content/detection/shfl.md

@@ -1,4 +1,5 @@
 ### `SHFL`: Shuffle I/O {#shfl}
 
 Tasks move a large amount of intermediate data between stages. Raise
-`spark.sql.shuffle.partitions`, or add a broadcast join.
+`spark.sql.shuffle.partitions`, or add a broadcast join. Only flagged on
+stages that take at least 0.5% of the run.

package/vendor-core/docs-content/detection/spill.md

@@ -4,4 +4,4 @@ Tasks are writing data out of memory, which slows execution. Two spill
 patterns get flagged differently: skew spill, where a few heavy tasks spill
 while most don't (rebalance partitioning), and volume spill, where most
 tasks spill because the data genuinely exceeds available memory (add
-partitions).
+partitions). Only flagged on stages that take at least 0.5% of the run.

package/vendor-core/docs-content/detection/strag.md

@@ -2,4 +2,5 @@
 
 A few tasks run much slower than the rest of their stage. Rule out a GC
 pause or a slow shuffle fetch before assuming a hardware issue; if a skewed
-key is the real cause, that's a candidate for AQE's skew-join handling.
+key is the real cause, that's a candidate for AQE's skew-join handling. Only
+flagged on stages that take at least 0.5% of the run.

package/vendor-core/docs-content/detection/tiny.md

@@ -1,4 +1,5 @@
 ### `TINY`: Tiny tasks {#tiny}
 
 Many very short tasks add scheduling overhead out of proportion to the work
-each one does. Repartition to fewer, larger tasks.
+each one does. Repartition to fewer, larger tasks. Only flagged on stages that
+take at least 0.5% of the run.

package/vendor-core/docs-content/tuning/failures.md

@@ -56,7 +56,7 @@ Config-only levers for memory-driven `ExecutorLostFailure`s and mid-shuffle exec
 
 ```properties
 # Cover off-heap + PySpark memory the default overhead budget does NOT include.
-# Default is max(384m, 10% of executor memory); 2g is an example, size to real off-heap/PySpark use.
+# Default is max(384m, 10% of executor memory); 2g is an example — size to real off-heap/PySpark use.
 spark.executor.memoryOverhead=2g
 
 # Keep shuffle output alive when dynamic allocation reclaims an executor mid-shuffle

package/vendor-core/docs-content/tuning/gc.md

@@ -16,12 +16,19 @@ pauses and overstate how long the task actually ran.[^1]
 ## How it's detected
 
 Because `jvmGCTime` sits inside `executorRunTime` rather than alongside it, the ratio between
-the two gives a bounded read on how much of a task's wall-clock time went to garbage collection:
+the two gives a bounded read on how much of a task's wall-clock time went to garbage collection.
+Both directions below apply once a stage's `executorRunTime` reaches 10 seconds, a floor that
+keeps short stages from reading as noise:
 
-| gcPct = jvmGCTime / executorRunTime | Level |
+| Signal (gcPct = jvmGCTime / executorRunTime) | Fires when |
 |---|---|
-| > 10% | Warning |
-| > 20% | Critical |
+| High GC | > 10% |
+| Low GC (cost signal) | < 5% |
+
+A stage lands in at most one of these two opposite-direction bands at a time. Severity for
+both tracks the estimated recoverable time as a share of the app's total runtime: ≥2% is
+critical, ≥0.5% is warning, anything smaller is info. Low GC signals a possible memory
+over-provisioning cost rather than time lost to garbage collection.
 
 ## Why it matters
 

package/vendor-core/docs-content/tuning/shuffle.md

@@ -26,11 +26,13 @@ read from local disk, and `totalBytesRead` is their sum[^6].
 
 ## How it's detected
 
-| Shuffle read/write bytes | Level |
+| Signal | Fires when |
 |---|---|
-| > 50 MB | Info |
-| > 500 MB | Warning |
-| > 1 GB | Critical |
+| Shuffle read bytes in a stage | > 50 MB |
+
+50 MB marks a stage as shuffle-heavy enough to flag. Severity then tracks the estimated
+recoverable time as a share of the app's total runtime: ≥2% is critical, ≥0.5% is
+warning, anything smaller is info.
 
 Beyond raw byte volume, the executor-side wait is captured by `fetchWaitTime`: time a task
 spends blocked on a remote shuffle block it needs next, not counting time spent prefetching
@@ -91,7 +93,25 @@ spark.shuffle.file.buffer=1m
 spark.reducer.maxSizeInFlight=48m
 ```
 
-## Partition sizing <span class="tag">PART</span>
+## Partition sizing {#bottleneck-partition-sizing}
+
+<span class="tag">PART</span>
+
+### How it's detected
+
+A stage's shuffle-read partition sizes surface three distinct problems:
+
+| Signal | Fires when | Level |
+|---|---|---|
+| Largest partition vs. median | > 5× the median **and** > 256 MB | Warning |
+| Low parallelism | ≥ 1 GB of shuffle read spread across ≤ 7 tasks | Warning |
+| Oversized partition | Largest partition ≥ 5 GB | Critical |
+
+Skew and low-parallelism severity track the estimated recoverable time as a share of the
+app's total runtime, the same wall-clock model used across this reference. An oversized
+partition is a fixed safety signal instead: it reports critical purely on its own size,
+because a partition past 5 GB is an OOM/crash risk regardless of how much wall-clock time
+fixing it would recover, so it stays critical even on a stage that barely dents the run.
 
 Adaptive Query Execution re-optimizes the plan while the query runs: as each shuffle stage
 materializes, it reads the real shuffle-file sizes and resizes partitions before launching the

package/vendor-core/docs-content/tuning/skew.md

@@ -13,10 +13,18 @@ more data than everyone else in the same stage.
 
 ## How it's detected
 
-| Signal | Warning | Critical |
-|---|---|---|
-| P95 / median task duration | > 3× | > 5× |
-| Max / median task duration (task count < 20) | > 3× | > 5× |
+| Signal | Fires when |
+|---|---|
+| P95 / median task duration (stage has ≥ 20 tasks) | > 3× |
+| Max / median task duration (stage has < 20 tasks) | > 3× |
+
+A 3× ratio marks a stage as skewed. Beyond that ratio, the estimated recoverable time (the
+P95-minus-median, or max-minus-median, delta) needs to clear 0.5% of the app's total
+runtime before it registers, a floor that filters out a 3× ratio sitting on a few
+milliseconds. Severity then tracks that same recoverable-time estimate as a share of the
+app's total runtime: ≥2% is critical, ≥0.5% is warning, anything smaller is info. A 3×
+ratio on a stage that barely dents an eight-hour run typically surfaces as info; the same
+ratio on a stage that dominates a short run reads as critical.
 
 ## Why it matters
 
@@ -53,12 +61,12 @@ The AQE toggle, plus the manual salting fallback as runnable code:
 ```python
 from pyspark.sql import functions as fn
 
-# AQE skew-join handling, active once AQE itself is enabled
+# AQE skew-join handling — active once AQE itself is enabled
 spark.conf.set("spark.sql.adaptive.enabled", "true")
 spark.conf.set("spark.sql.adaptive.skewJoin.enabled", "true")
 
 # Manual salting fallback (pre-AQE): spread the skewed key across N salted variants.
-# N is an example, size it to how badly the key is skewed.
+# N is an example — size it to how badly the key is skewed.
 N = 16
 salted_big = big.withColumn("salt", (fn.rand() * N).cast("int"))
 salted_small = small.withColumn(

package/vendor-core/docs-content/tuning/small-files.md

@@ -21,15 +21,20 @@ inefficient to read a whole block when you only need a few rows[^5].
 
 ## How it's detected
 
-Reading the SQL plan surfaces write paths that will emit far more output files
-than the data warrants, keying off the one-file-per-output-partition rule[^4]. The signal is
-a high output-partition count against a modest data volume, which foreshadows a directory
-full of tiny files.
+Each SQL plan node carries file-count and file-size metrics Spark reports directly:
+`number of files read`/`size of files read` on the read side, `number of written
+files`/`written output` on the write side. A node reads as a small-files problem once
+its file count on a given side is high and the resulting average file size is small.
 
-| Signal | What it points to |
+| Signal (per plan node, checked separately for read and write) | Fires when |
 |---|---|
-| Output partition count high vs. data volume | Many tiny files on write |
-| Read partitioning below `spark.sql.files.maxPartitionBytes` (128 MB) | Over-split input, excess filesystem I/O[^2][^3] |
+| File count | > 100 |
+| Average file size (bytes ÷ file count) | < 3 MB |
+
+Both conditions have to hold together, so a node with thousands of files that are each
+big enough, or a handful of genuinely tiny ones, doesn't register. The signal comes
+entirely from those file-count and file-size metrics, independent of
+`spark.sql.files.maxPartitionBytes`.
 
 ## Why it matters
 

package/vendor-core/docs-content/tuning/straggler.md

@@ -93,6 +93,40 @@ Validated.
 
 The 4x-median rule flags a slow task, but slow is not the same as broken. The same threshold trips on a skewed key that simply has more data to process, or on a task that spent its time in a GC pause rather than doing extra work, so a flagged task is not automatically a slow host. Small stages make this worse: with only a handful of tasks the median is unstable, and one moderately slow task can look like a straggler against a median computed from too few peers.
 
+## Speculation waste {#bottleneck-speculation-waste}
+
+<span class="tag">SPEC</span>
+
+A straggler is reported by whichever signal, straggler share or speculative-task count,
+best explains it. Speculation waste is a separate, narrower signal: the executor time
+speculative attempts burned without confirming a genuine straggler, whether the
+speculative copy lost the race to the original attempt or the other way around. Either
+way, the losing attempt's executor time is pure waste.
+
+### How it's detected
+
+| Signal | Fires when |
+|---|---|
+| Wasted speculative attempts in a stage | ≥ 5 |
+| Wasted executor time from those attempts | ≥ 60 seconds |
+
+Both conditions have to hold together. Severity tracks the estimated recoverable time as
+a share of the app's total runtime, the same model used across this page: ≥2% is
+critical, ≥0.5% is warning, anything smaller is info.
+
+### Why it matters
+
+Every wasted speculative attempt occupies an executor slot that could have run other
+work, so a stage generating a lot of speculative waste is trading cluster capacity for
+copies that never pay off.
+
+### How to fix it
+
+If task durations are naturally variable rather than genuine stragglers, speculation is
+firing too eagerly: tune `spark.speculation.multiplier` (require a bigger gap from the
+median before speculating) or `spark.speculation.quantile` (wait for more of the stage to
+finish first) so fewer ordinary slow tasks get speculated in the first place.
+
 ## Related
 
 - **When the real cause is a skewed key:** [Partitioning](#partitioning), [Adaptive Query Execution](#aqe)

package/vendor-core/docs-content/tuning/tiny-tasks.md

@@ -70,7 +70,7 @@ The three sizing calls, side by side:
 # Collapse many tiny output files without a shuffle (use when distribution is already even)
 df.coalesce(100).write.parquet(path)
 
-# Full reshuffle to a target count, use when the distribution itself needs rebalancing
+# Full reshuffle to a target count — use when the distribution itself needs rebalancing
 df = df.repartition(200)
 
 # Middle ground: reduce partition count but still rebalance (pays a shuffle)

package/vendor-core/docs-content/tuning/utilization.md

@@ -4,17 +4,17 @@
 
 ## What it is
 
-Executor utilization measures how much of the cluster's allocated executor capacity a job actually keeps busy. When the average number of active executors trails the peak number allocated, the cluster is holding compute (cores and memory) that isn't running any tasks.
+Executor utilization measures how much of the cluster's allocated core-time a job actually keeps busy. When busy core-time trails the core-time available across the run, the cluster is holding compute (cores and memory) that isn't running any tasks.
 
 ## How it's detected
 
-The signal is the ratio of average active executors to the peak active executor count observed over the job's lifetime:
+The signal is the ratio of busy executor core-time to the core-time available over the run: peak concurrent cores multiplied by the app's duration.
 
-| avg active executors / peak | Level |
+| Signal | Fires when |
 |---|---|
-| < 60% | Info |
-| < 40% | Warning |
-| < 20% | Critical |
+| busy core-time / available core-time | < 60% |
+
+A finding here always reports at the info level.
 
 ## Why it matters
 
@@ -41,12 +41,62 @@ spark.conf.set("spark.dynamicAllocation.executorAllocationRatio", "0.5")  # 0.5
 
 A low average-to-peak ratio isn't always waste. A bursty or I/O-bound job legitimately holds executors while tasks wait on external systems rather than burning cores, and the ratio is sensitive to short stages, where a brief spike in allocation skews the average without meaning the cluster was genuinely idle.
 
-## Caching opportunity
+## Core locality {#bottleneck-core-locality}
+
+<span class="tag">LOCAL</span>
+
+Idle cores are one half of wasted capacity; this is the other half. A core running a task
+without process- or node-local data placement is still busy, but that task now has to
+fetch its input across the network or from a different process instead of reading it in
+place, work the core wouldn't have to do at all if it were scheduled on data it already
+holds.
+
+### How it's detected
+
+The share of tasks that ran RACK_LOCAL or ANY, out of every task with a recorded locality
+level, is the non-local task share. NO_PREF tasks stay in the denominator only, since
+shuffle-read stages legitimately report that level without it indicating a placement
+problem. This applies once a run has logged at least 50 total tasks.
+
+| Signal | Warning | Critical |
+|---|---|---|
+| Non-local task share | ≥ 15% | ≥ 35% |
+
+### Why it matters
+
+A task denied process- or node-local placement pulls its input over the network or
+through inter-process I/O instead of reading it from local memory or disk, adding latency
+to every task that lands that way. Spread across a whole run, a high non-local share adds
+up to a meaningful share of total task time spent moving data that a better-placed
+schedule wouldn't have had to move.
+
+### How to fix it
+
+- Check `spark.locality.wait` (default 3s) and its per-level overrides
+  (`.process`/`.node`/`.rack`): a wait set too short gives Spark less time to find a
+  local slot before it falls back to a less-local one.
+- Check executor and data colocation: if the executors are running far from where the
+  data actually lives (a different rack, a different zone), no amount of locality-wait
+  tuning fixes a placement that isn't available to begin with.
+
+## Caching opportunity {#bottleneck-caching-opportunity}
 
 <span class="tag">CACHE</span>
 
 When the same DataFrame, RDD, or input is scanned more than once, low utilization can trace back to repeated recomputation rather than idle cores. Spark keeps nothing between actions: transformations only build a DAG, and once an action finishes its intermediate results are discarded[^6]. Call a second action on the same logic and Spark re-runs the whole DAG from the source, which can mean re-reading a terabyte from S3, re-reading Kafka, or repeating expensive decompression[^6]. Fork that logic into two pipeline branches and you sign up to recompute everything twice[^6].
 
+### How it's detected
+
+An input relation that recurs across at least 2 SQL executions in one run is flagged by
+its scan format and path identity. A join or union subtree that recurs across at least 2
+executions is matched by operator name, metric names, and its join or filter condition
+normalized to strip per-analysis ids and canonicalize commutative operand order, while
+columns and literal values are kept intact: two executions of the exact same join or
+filter still match even when Spark's internal ids differ between runs, but a genuinely
+different filter or join value does not. A qualifying join or union subtree is reported
+as one finding covering everything beneath it. Every finding here reports at the info
+level.
+
 <img class="light-only" src="../diagrams/duplicate-plan-subtree.svg" alt="How two branches that repeat the same scan and operators each recompute it, until a shared cached or reused node lets both read one materialized result.">
 <img class="dark-only" src="../diagrams/duplicate-plan-subtree.dark.svg" alt="How two branches that repeat the same scan and operators each recompute it, until a shared cached or reused node lets both read one materialized result.">
 

package/vendor-core/docs-content/upstream.json

@@ -0,0 +1,4 @@
+{
+  "repository": "https://github.com/shuffle-works/spark-tuning-reference.git",
+  "commit": "1d0f90dce8c0e04ad0ee6f1f724ae28d821b42cd"
+}