sparkforensics-mcp 0.2.4 → 0.4.0

package/vendor-core/docs-config.js

@@ -1,5 +1,3 @@
-import { detectorCatalog } from './detectors.js';
-
 // Single source of the docs-panel URL surface. DOCS_BASE_DIR is the built docs-site path that
 // serves the tuning reference, relative to the app's origin.
 export const DOCS_BASE_DIR         = 'docs/tuning-reference';
@@ -90,40 +88,6 @@ export function isKnownDocAnchor(anchor         )          {
   return KNOWN_DOC_ANCHORS.has(String(anchor));
 }
 
-// Finding types that never appear as their own DETECTORS entry's type: the parent entry declares
-// a different type because one plan-walk covers two rules (see broadcastSizing). Map to the parent.
-const TYPE_ALIASES                         = {
-  underBroadcast: 'broadcastSizing',
-  overBroadcast: 'broadcastSizing',
-};
-
-// DETECTORS is static, so this grouping is built once (lazily) instead of re-scanning per
-// docAnchorForType call (called once per TagBadge per render).
-let anchorsByTypeCache                                                  ;
-
-function anchorsByType()                                       {
-  if (!anchorsByTypeCache) {
-    anchorsByTypeCache = new Map();
-    for (const entry of detectorCatalog()) {
-      const anchors = anchorsByTypeCache.get(entry.type) ?? new Set();
-      anchors.add(entry.docAnchor);
-      anchorsByTypeCache.set(entry.type, anchors);
-    }
-  }
-  return anchorsByTypeCache;
-}
-
-/** Resolves a finding `type` to its documented anchor from detectorCatalog(). Returns undefined
- * when entries sharing the type disagree on docAnchor (only configAudit today), or when the
- * resolved anchor isn't in the allowlist (isKnownDocAnchor, the same gate DocsLink uses). */
-export function docAnchorForType(type        )                     {
-  const resolvedType = TYPE_ALIASES[type] ?? type;
-  const anchors = anchorsByType().get(resolvedType) ?? new Set();
-  if (anchors.size !== 1) return undefined;
-  const [anchor] = anchors;
-  return anchor && isKnownDocAnchor(anchor) ? anchor : undefined;
-}
-
 /** The docAnchor every finding in `findings` carries, or undefined when they disagree or any lacks
  * one. For a badge standing for several findings (a widget header, a grouped row) whose type-level
  * docAnchorForType can't pick one: configAudit's sub-checks each stamp their own anchor. */

package/vendor-core/docs-content/chapters/nav-index.json

@@ -3,6 +3,7 @@
     "anchor": "intro",
     "section": "Getting Started",
     "title": "Introduction",
+    "brief": "Guide overview: what the reference covers, the severity-dot legend, and the tag system linking into the Bottleneck Reference.",
     "keywords": [
       "severity-dot",
       "tag-system"
@@ -14,6 +15,7 @@
     "anchor": "spark-architecture",
     "section": "Optimization Guide",
     "title": "Spark Execution Model",
+    "brief": "Spark's execution model: Driver/Executor split, DAGScheduler/TaskScheduler roles, stage boundaries at shuffles, and pipelining of narrow transformations.",
     "keywords": [
       "driver-executor",
       "dagscheduler",
@@ -27,6 +29,7 @@
     "anchor": "memory-model",
     "section": "Optimization Guide",
     "title": "Memory Management",
+    "brief": "Unified memory management: on-heap/off-heap execution vs. storage regions, spill triggers, and executor memory overhead sizing.",
     "keywords": [
       "unified-memory",
       "execution-memory",
@@ -41,6 +44,7 @@
     "anchor": "partitioning",
     "section": "Optimization Guide",
     "title": "Partitioning",
+    "brief": "Partition sizing and reshaping: repartition vs. coalesce mechanics, shuffle-partition tuning, and skew detection thresholds.",
     "keywords": [
       "repartition",
       "coalesce",
@@ -53,6 +57,7 @@
     "anchor": "joins",
     "section": "Optimization Guide",
     "title": "Join Optimization",
+    "brief": "Physical join strategy selection: broadcast vs. shuffle joins, join hints, bucketing, and AQE's runtime join-strategy switching.",
     "keywords": [
       "broadcast",
       "shuffle-join",
@@ -66,6 +71,7 @@
     "anchor": "shuffle",
     "section": "Optimization Guide",
     "title": "Shuffle",
+    "brief": "Shuffle internals: SortShuffleManager mechanics, shuffle-avoidance paths (bucketing, Storage Partition Join), and shuffle-tuning configs.",
     "keywords": [
       "sortshufflemanager",
       "bucketing",
@@ -78,6 +84,7 @@
     "anchor": "data-formats",
     "section": "Optimization Guide",
     "title": "Data Formats",
+    "brief": "Columnar file format tradeoffs: Parquet vs. ORC internals, splittability, predicate pushdown, and compression codec choice.",
     "keywords": [
       "parquet",
       "orc",
@@ -91,6 +98,7 @@
     "anchor": "table-formats",
     "section": "Optimization Guide",
     "title": "Table Formats",
+    "brief": "Lakehouse table-format tuning across Delta Lake, Iceberg, and Hudi: file sizing, compaction/OPTIMIZE, clustering (Z-order/liquid/sort), metadata and manifest overhead, snapshot/version expiry, and deletion vectors vs. merge-on-read/copy-on-write.",
     "keywords": [
       "delta",
       "iceberg",
@@ -107,6 +115,7 @@
     "anchor": "caching",
     "section": "Optimization Guide",
     "title": "Caching & Persistence",
+    "brief": "cache()/persist() semantics, storage levels, eviction behavior, and checkpointing as a lineage-truncation alternative.",
     "keywords": [
       "cache",
       "persist",
@@ -120,6 +129,7 @@
     "anchor": "pyspark",
     "section": "Optimization Guide",
     "title": "PySpark Specifics",
+    "brief": "PySpark-specific performance: UDF serialization cost, Arrow-optimized and pandas UDF types, and Python-worker memory configs.",
     "keywords": [
       "udf",
       "arrow",
@@ -133,6 +143,7 @@
     "anchor": "aqe",
     "section": "Optimization Guide",
     "title": "Adaptive Query Execution",
+    "brief": "AQE's runtime re-optimization loop: post-shuffle partition coalescing, join-strategy promotion, and skew-partition splitting.",
     "keywords": [
       "partition-coalescing",
       "join-strategy-promotion",
@@ -145,6 +156,7 @@
     "anchor": "cluster-config",
     "section": "Optimization Guide",
     "title": "Cluster Tuning",
+    "brief": "Cluster-level sizing: executor core/memory formulas, container memory budgeting, dynamic allocation, and locality-wait tuning.",
     "keywords": [
       "executor-sizing",
       "container-memory",
@@ -158,6 +170,7 @@
     "anchor": "anti-patterns",
     "section": "Optimization Guide",
     "title": "Anti-Patterns",
+    "brief": "Checklist of a dozen recurring Spark performance anti-patterns, each with what it is, how to detect it, why it hurts, and how to fix it.",
     "keywords": [
       "checklist",
       "anti-pattern"
@@ -169,6 +182,7 @@
     "anchor": "bottleneck-skew",
     "section": "Detector Catalog",
     "title": "Task Skew",
+    "brief": "Detecting and mitigating task skew: oversized shuffle partitions vs. the median, AQE skew-join splitting, and manual salting as the pre-AQE fallback.",
     "keywords": [
       "skew",
       "aqe-skew-join",
@@ -181,6 +195,7 @@
     "anchor": "bottleneck-shuffle",
     "section": "Detector Catalog",
     "title": "Shuffle I/O",
+    "brief": "Detecting and mitigating shuffle I/O bottlenecks: read/write byte thresholds, fetch-wait time, and shuffle-avoidance and tuning levers.",
     "keywords": [
       "shuffle-bytes",
       "fetch-wait"
@@ -192,6 +207,7 @@
     "anchor": "bottleneck-spill",
     "section": "Detector Catalog",
     "title": "Memory / Disk Spill",
+    "brief": "Detecting and mitigating memory/disk spill: TaskMemoryManager pressure, the skew-vs-volume spill classification, and matching the fix to the cause.",
     "keywords": [
       "spill",
       "taskmemorymanager"
@@ -203,6 +219,7 @@
     "anchor": "bottleneck-gc",
     "section": "Detector Catalog",
     "title": "GC Pressure",
+    "brief": "Detecting and mitigating GC pressure: the gcPct warning/critical thresholds, oversized executors and memory.fraction as causes, and off-heap/collector fixes.",
     "keywords": [
       "gc-pressure",
       "memory-fraction",
@@ -215,6 +232,7 @@
     "anchor": "bottleneck-cold-start",
     "section": "Detector Catalog",
     "title": "Cold Start",
+    "brief": "Detecting and mitigating cold start: the firstStageSubmittedAt gap before the first stage is submitted, and executor-sizing/dynamic-allocation readiness fixes.",
     "keywords": [
       "cold-start",
       "dynamic-allocation"
@@ -226,6 +244,7 @@
     "anchor": "bottleneck-utilization",
     "section": "Detector Catalog",
     "title": "Executor Utilization",
+    "brief": "Detecting and mitigating low executor utilization: the avg/peak active-executor ratio, under-partitioning and coalesce-caused parallelism loss, and dynamic-allocation over-provisioning.",
     "keywords": [
       "executor-utilization",
       "under-partitioning",
@@ -238,6 +257,7 @@
     "anchor": "bottleneck-slow-host",
     "section": "Detector Catalog",
     "title": "Slow Host",
+    "brief": "Detecting and mitigating a slow host: host-wide task duration inflation versus the cluster median, and speculative-execution or locality-wait fixes.",
     "keywords": [
       "slow-host",
       "speculative-execution"
@@ -249,6 +269,7 @@
     "anchor": "bottleneck-failures",
     "section": "Detector Catalog",
     "title": "Task Failures",
+    "brief": "Detecting and mitigating task failures: the TaskEndReason taxonomy, failed-task-share thresholds, and distinguishing memory-driven ExecutorLostFailure from application-code ExceptionFailure.",
     "keywords": [
       "taskendreason",
       "executorlostfailure",
@@ -261,6 +282,7 @@
     "anchor": "bottleneck-straggler",
     "section": "Detector Catalog",
     "title": "Stragglers",
+    "brief": "Detecting and mitigating straggler tasks: the 4x-median-duration rule, distinguishing stragglers from GC pauses or skew, and speculative-execution tuning.",
     "keywords": [
       "straggler",
       "speculative-execution"
@@ -272,6 +294,7 @@
     "anchor": "bottleneck-retry-waste",
     "section": "Detector Catalog",
     "title": "Retry Waste",
+    "brief": "Detecting and mitigating wasted retry compute: executorRunTime burned by superseded task attempts, and distinguishing executor-loss from fetch-failure causes.",
     "keywords": [
       "retry-waste",
       "fetch-failure"
@@ -283,6 +306,7 @@
     "anchor": "bottleneck-tiny-tasks",
     "section": "Detector Catalog",
     "title": "Tiny Tasks",
+    "brief": "Detecting and mitigating tiny-task overhead: scheduling/serialization cost dominating once partitions shrink too far, and coalesce/repartition sizing fixes.",
     "keywords": [
       "tiny-tasks",
       "scheduling-overhead"
@@ -294,6 +318,7 @@
     "anchor": "bottleneck-job-failure-rate",
     "section": "Detector Catalog",
     "title": "Job Failure Rate",
+    "brief": "Detecting and mitigating job-level failure: the task/stage/executor retry-budget escalation ladder that turns isolated failures into an aborted job.",
     "keywords": [
       "job-failure-rate",
       "retry-budget"
@@ -305,6 +330,7 @@
     "anchor": "bottleneck-memory-utilization",
     "section": "Detector Catalog",
     "title": "Memory Utilization",
+    "brief": "Detecting and mitigating held-but-idle executor memory: cores sitting idle below allocated slots, dynamic-allocation idle timeouts that reclaim an executor's memory band, and the low-confidence peak-vs-allocated waste estimate.",
     "keywords": [
       "memory-utilization",
       "idle-timeout"
@@ -316,6 +342,7 @@
     "anchor": "bottleneck-duplicate-plan-subtree",
     "section": "Detector Catalog",
     "title": "Duplicate Plan Subtree",
+    "brief": "Detecting a repeated scan or exchange subtree in a physical plan, how exchange reuse and cache()/persist() avoid recomputing it, and why inferring duplication from the plan is a medium-confidence signal.",
     "keywords": [
       "duplicate-plan-subtree",
       "exchange-reuse"
@@ -327,6 +354,7 @@
     "anchor": "bottleneck-small-files",
     "section": "Detector Catalog",
     "title": "Small Files",
+    "brief": "Detecting and mitigating the small-files problem on read and write: metadata and I/O overhead of many tiny files, and coalesce()/repartition()/compaction and AQE advisoryPartitionSizeInBytes as output-size controls.",
     "keywords": [
       "small-files",
       "advisorypartitionsizeinbytes"
@@ -338,6 +366,7 @@
     "anchor": "bottleneck-broadcast-sizing",
     "section": "Detector Catalog",
     "title": "Broadcast Sizing",
+    "brief": "Detecting under- and over-broadcast joins: spark.sql.autoBroadcastJoinThreshold and its default, when to broadcast the smaller side versus shuffle, and the driver-collection and executor-memory failure modes of broadcasting too large a table.",
     "keywords": [
       "broadcast-threshold",
       "driver-collection"
@@ -349,6 +378,7 @@
     "anchor": "metrics",
     "section": "Reference",
     "title": "Metrics Glossary",
+    "brief": "Glossary of every metric surfaced elsewhere on the site: what each one measures, its event-log source field, and what a problematic value looks like.",
     "keywords": [
       "metrics-glossary",
       "event-log"
@@ -360,6 +390,7 @@
     "anchor": "config",
     "section": "Reference",
     "title": "Spark Config Quick-Reference",
+    "brief": "Cross-reference of the Spark/PySpark config properties and JVM GC flags discussed elsewhere in the guide, with defaults and version notes.",
     "keywords": [
       "spark-config",
       "gc-flags"

package/vendor-core/docs-content/detection/cstor.md

@@ -2,3 +2,12 @@
 
 A persisted dataset is not fully cached in memory, or is spilling to disk.
 Raise executor memory, or shrink the cached dataset.
+
+The cached-partition counts and sizes come from `SparkListenerBlockUpdated`
+events, which Spark writes only when
+`spark.eventLog.logBlockUpdates.enabled=true`. Since Spark 2.3 the RDD
+storage figures in stage-submission events are always 0, and they are
+used only as a fallback. When a Spark 2.3+ run persists RDDs but its log has
+neither and block-update logging was off, the check reports that the cache could not be
+checked instead of passing it: turn `spark.eventLog.logBlockUpdates.enabled`
+on and rerun to measure eviction and disk spillover.

package/vendor-core/docs-content/detection/fail.md

@@ -1,5 +1,9 @@
 ### `FAIL`: Failed tasks {#fail}
 
 Tasks fail often enough to affect the stage. Failed tasks point to executor
-instability or data-driven errors: check driver logs for the dominant
-failure reason.
+instability or data-driven errors. The finding names the dominant error: the
+exception class, or the executor loss reason (for example "Container killed
+by YARN for exceeding memory limits"). It lists up to five distinct failures,
+each with its message and a short stack excerpt. With redaction on, messages
+and the message text inside excerpts are replaced, since they can carry file
+paths and data values; class names and stack frames stay.

package/vendor-core/docs-site-config.js

@@ -14,3 +14,6 @@ import { typeTag } from './format-utils.js';
 export function findingGuideUrl(type        )         {
   return `docs/user-guide/understanding-findings.html#${typeTag(type).toLowerCase()}`;
 }
+
+// Every other way to get a log (cloud consoles, bastions, copying from storage).
+export const ALTERNATIVE_LOG_RETRIEVAL_URL = 'docs/user-guide/alternative-log-retrieval.html';