sparkforensics-server 0.2.2 → 0.2.4

package/public/docs/assets/{contributor-guide_architecture_worker-protocol.md.IbnfNrV3.js → contributor-guide_architecture_worker-protocol.md.P7d1vtEJ.js}

@@ -1,6 +1,6 @@
-import{_ as o,o as t,c as d,a5 as a}from"./chunks/framework.DSg0KOwT.js";const u=JSON.parse('{"title":"Worker protocol","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/architecture/worker-protocol.md","filePath":"contributor-guide/architecture/worker-protocol.md"}'),c={name:"contributor-guide/architecture/worker-protocol.md"};function s(r,e,i,n,l,p){return t(),d("div",null,[...e[0]||(e[0]=[a(`<h1 id="worker-protocol" tabindex="-1">Worker protocol <a class="header-anchor" href="#worker-protocol" aria-label="Permalink to &quot;Worker protocol&quot;">​</a></h1><h2 id="message-protocol" tabindex="-1">Message protocol <a class="header-anchor" href="#message-protocol" aria-label="Permalink to &quot;Message protocol&quot;">​</a></h2><p>Worker to main:</p><ul><li><code>app</code>, <code>stage</code>, <code>sql</code>, <code>executor</code>, <code>job</code>, <code>progress</code>: progressive, posted as the parser reads.</li><li><code>sqlPlan</code> (<code>{ type: &#39;sqlPlan&#39;, data: { executionId, planTree } }</code>): posted by <code>endSqlExecution</code> once a SQL execution&#39;s plan tree resolves.</li><li><code>runAggregates</code>: one whole-run core-time-series summary (busy-core-ms, peak concurrency, per-stage task-duration sums), emitted just before <code>done</code>.</li><li><code>stageExecutorMetrics</code>: the post-completion re-post described in <a href="./overview.html#streaming">Streaming</a>, also emitted just before <code>done</code>.</li><li><code>done</code>, <code>taskData</code>, <code>error</code>.</li></ul><p>A History Server failure is always the typed, display-safe payload <code>{ type: &#39;error&#39;, source: &#39;shs&#39;, code }</code>, where <code>code</code> is one of <code>local-server-unavailable</code>, <code>upstream-unreachable</code>, <code>application-not-found</code>, <code>access-or-upstream-failure</code>, or <code>invalid-event-log</code>. It never carries an upstream message, URL, status, or response body.</p><p>Evidence availability has no dedicated worker message: the final <code>app</code> message carries a compact <code>evidenceInputs</code> counter summary, and <code>done.skippedLines</code> supplies its parse-integrity input.</p><p>Main to worker: <code>parse</code>, <code>parseFiles(files)</code> (rolling <code>eventlog_v2_*</code> directories, one continuous stream across files), <code>parseFromUrl({ baseUrl, appId, attemptId })</code>, and <code>getTaskData(stageId)</code>. The SHS request object is normalized before it reaches the worker: <code>{ baseUrl: string, appId: string, attemptId: string | null }</code>.</p><p><code>taskData</code> uses structured-clone (<code>.slice()</code>) so the worker retains its own <code>Float64Array</code> for re-renders.</p><p>Post-parse prefetch: after <code>done</code>, main runs <code>analyzer.ts</code> to build the bottleneck catalog, then fires parallel <code>getTaskData</code> for every flagged stage so their widgets render immediately. Unflagged stages are on-demand.</p><h3 id="evidence-availability-worker-input" tabindex="-1">Evidence-availability worker input <a class="header-anchor" href="#evidence-availability-worker-input" aria-label="Permalink to &quot;Evidence-availability worker input&quot;">​</a></h3><p><code>packages/core/src/event-handlers.ts</code>&#39;s <code>createState()</code>/per-event handlers start and increment this fixed <code>evidenceInputs</code> shape on the normalized app model:</p><div class="language-text vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">text</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>environmentUpdates, applicationEnds, stageSubmissions, rddStorageSnapshots,</span></span>
+import{_ as o,o as t,c as d,a5 as a}from"./chunks/framework.DSg0KOwT.js";const u=JSON.parse('{"title":"Worker protocol","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/architecture/worker-protocol.md","filePath":"contributor-guide/architecture/worker-protocol.md"}'),c={name:"contributor-guide/architecture/worker-protocol.md"};function s(r,e,i,n,l,p){return t(),d("div",null,[...e[0]||(e[0]=[a(`<h1 id="worker-protocol" tabindex="-1">Worker protocol <a class="header-anchor" href="#worker-protocol" aria-label="Permalink to &quot;Worker protocol&quot;">​</a></h1><h2 id="message-protocol" tabindex="-1">Message protocol <a class="header-anchor" href="#message-protocol" aria-label="Permalink to &quot;Message protocol&quot;">​</a></h2><p>Worker to main:</p><ul><li><code>app</code>, <code>stage</code>, <code>sql</code>, <code>executor</code>, <code>job</code>, <code>progress</code>: progressive, posted as the parser reads.</li><li><code>sqlPlan</code> (<code>{ type: &#39;sqlPlan&#39;, data: { executionId, planTree } }</code>): posted by <code>endSqlExecution</code> once a SQL execution&#39;s plan tree resolves. A repeated <code>SQLExecutionEnd</code> for that execution posts nothing, so no later <code>sql</code> message replaces the entry holding its <code>planTree</code>.</li><li><code>runAggregates</code>: one whole-run core-time-series summary (busy-core-ms, peak concurrency, per-stage task-duration sums), emitted just before <code>done</code>.</li><li><code>stageExecutorMetrics</code>: the post-completion re-post described in <a href="./overview.html#streaming">Streaming</a>, also emitted just before <code>done</code>.</li><li><code>done</code>, <code>taskData</code>, <code>error</code>.</li></ul><p>A History Server failure is always the typed, display-safe payload <code>{ type: &#39;error&#39;, source: &#39;shs&#39;, code }</code>, where <code>code</code> is one of <code>local-server-unavailable</code>, <code>upstream-unreachable</code>, <code>application-not-found</code>, <code>access-or-upstream-failure</code>, or <code>invalid-event-log</code>. It never carries an upstream message, URL, status, or response body.</p><p>Evidence availability has no dedicated worker message: the final <code>app</code> message carries a compact <code>evidenceInputs</code> counter summary, and <code>done.skippedLines</code> supplies its parse-integrity input.</p><p>Main to worker: <code>parse</code>, <code>parseFiles(files)</code> (rolling <code>eventlog_v2_*</code> directories, one continuous stream across files), <code>parseFromUrl({ baseUrl, appId, attemptId })</code>, and <code>getTaskData(stageId)</code>. The SHS request object is normalized before it reaches the worker: <code>{ baseUrl: string, appId: string, attemptId: string | null }</code>.</p><p><code>taskData</code> uses structured-clone (<code>.slice()</code>) so the worker retains its own <code>Float64Array</code> for re-renders.</p><p>Post-parse prefetch: after <code>done</code>, main runs <code>analyzer.ts</code> to build the bottleneck catalog, then fires parallel <code>getTaskData</code> for every flagged stage so their widgets render immediately. Unflagged stages are on-demand.</p><h3 id="decompress-worker" tabindex="-1">Decompress worker <a class="header-anchor" href="#decompress-worker" aria-label="Permalink to &quot;Decompress worker&quot;">​</a></h3><p>A dropped zstd file (<code>parse</code>, and each zstd file of a <code>parseFiles</code> directory) is decompressed in a second, nested worker, <code>packages/core/src/zstd-worker.ts</code>, so fzstd and the NDJSON parser run at the same time. On the largest real log, fzstd had been about 47% of the parse worker&#39;s time. The parse worker starts it on the first zstd file and reuses it for the rest of the parse. It dies with the parse worker, so the page&#39;s <code>terminate()</code> also cancels it. Other codecs and the SHS path (<code>parseFromUrl</code>, which decodes whole zip entries synchronously) still decompress on the parse worker.</p><p><code>packages/core/src/zstd-worker-client.ts</code> is the parse-worker end: it plugs into <code>streamFile</code> as the <code>zstdDecoder</code> option, like the Node CLI&#39;s native decoder. The messages, all buffers moved by transfer, never copied:</p><ul><li>Parse to decompress: <code>start { id }</code> opens a stream, and replaces any open one. <code>data { id, seq, bytes, final }</code> carries one compressed read slice. <code>cancel { id }</code> drops the stream.</li><li>Decompress to parse: <code>ready</code> once, at startup. <code>chunk { id, bytes, length }</code> carries decoded output in batches of up to 1 MiB. <code>consumed { id, seq }</code> follows the last <code>chunk</code> of slice <code>seq</code>. <code>error { id, message }</code> ends the stream, and its message becomes the usual <code>Could not decompress</code> error.</li></ul><p>Flow control is a window of three input slices: <code>push()</code> resolves while fewer than three slices wait for their <code>consumed</code>. Each <code>chunk</code> is parsed in its message handler, so a slice is acknowledged only after its output was parsed, and at most three slices&#39; output is ever queued. The final <code>push()</code> resolves once every slice is acknowledged. When <code>streamFile</code> gives up mid-stream (a failed file read, a parse exception), it calls <code>cancel()</code>.</p><p>When the nested worker cannot start (<code>new Worker</code> throws or its script fails to load), the parse worker logs a warning and decodes with in-thread fzstd, as it did before, with identical output. A crash after startup fails the stream it was decoding, and later streams fall back the same way. The progress <code>pct</code> is the read position, so it can run up to the window ahead of the slice being parsed. The self-contained <code>file://</code> export never parses in the browser (it opens with its run already analyzed), so it never starts either worker.</p><h3 id="evidence-availability-worker-input" tabindex="-1">Evidence-availability worker input <a class="header-anchor" href="#evidence-availability-worker-input" aria-label="Permalink to &quot;Evidence-availability worker input&quot;">​</a></h3><p><code>packages/core/src/event-handlers.ts</code>&#39;s <code>createState()</code>/per-event handlers start and increment this fixed <code>evidenceInputs</code> shape on the normalized app model:</p><div class="language-text vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">text</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>environmentUpdates, applicationEnds, stageSubmissions, rddStorageSnapshots,</span></span>
 <span class="line"><span>sqlExecutions, resolvedSqlPlans, executorMetricRows, taskRecords</span></span></code></pre></div><p>The counters are a structured-clone-safe summary: they contain counts, not raw events, task records, host names, SQL text, paths, or Spark-property values. The final <code>app</code> message snapshots the counters after all input is processed, immediately before <code>done</code>; a consumer must not add a second worker message just for the ledger. <code>app</code> messages are also emitted mid-parse from <code>SparkListenerApplicationStart</code> and <code>SparkListenerEnvironmentUpdate</code>; those snapshots are partial and non-authoritative. Only the terminal <code>app</code> message emitted by <code>emitParseCompletion</code> (the one immediately before <code>done</code>) should be used for evidence-availability conclusions. On the main thread, <code>useIngest</code> combines the normalized <code>AppModel</code> with <code>done.skippedLines</code>, derives the ledger before calling <code>analyze()</code>, and stores it as <code>appModel.evidenceAvailability</code>.</p><h3 id="evidence-availability-contract-v1" tabindex="-1">Evidence-availability contract (V1) <a class="header-anchor" href="#evidence-availability-contract-v1" aria-label="Permalink to &quot;Evidence-availability contract (V1)&quot;">​</a></h3><p><code>packages/core/src/evidence-availability.ts</code> is the single reusable taxonomy for the browser, future report output, and future headless consumers. Its serialized ledger shape is <code>{ schemaVersion: 1, entries: EvidenceAvailabilityEntry[] }</code>. Entries are ordered by the following fixed eight-key enum:</p><div class="language-text vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">text</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>executorMetrics, rddStorageSnapshots, sqlPlan, sparkConfiguration,</span></span>
 <span class="line"><span>taskCoreTime, infrastructureContext, sourceContext, costContext</span></span></code></pre></div><p>Each entry has the stable fields <code>key</code>, <code>state</code>, <code>reasonCode</code>, <code>summary</code>, and optional <code>evidence</code>. The closed V1 state enum is:</p><div class="language-text vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">text</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>present, disabled, notEmitted, notApplicable, outsideEventLog, unknown</span></span></code></pre></div><p>The closed V1 reason-code enum is:</p><div class="language-text vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">text</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>observed, explicitlyDisabled, noObservedExecutorMetrics,</span></span>
 <span class="line"><span>noObservedStageSubmission, noRddStorageSnapshot, noResolvedSqlPlan,</span></span>
 <span class="line"><span>noSqlExecution, noEnvironmentUpdate, noTaskRecords, noUsableCoreTimeAggregate,</span></span>
-<span class="line"><span>outsideEventLogScope, parseIncomplete</span></span></code></pre></div><p><code>summary</code> is domain-agnostic presentation copy derived from the reason code; the enum values, not labels, are the machine contract. Optional <code>evidence</code> is limited to safe compact provenance of the exact <code>{ eventType, count }</code> shape. <code>eventType</code> values are pinned to the exact <code>evidenceInputs</code> counter keys; they are not shortened or aliased. It must never include raw Spark-property values, host names, paths, SQL text, task records, raw-event identifiers, or other event payloads.</p><table tabindex="0"><thead><tr><th>Key</th><th>Proven <code>present</code> evidence</th><th>Trustworthy absence / boundary result</th></tr></thead><tbody><tr><td><code>executorMetrics</code></td><td><code>executorMetricRows &gt; 0</code></td><td>Explicit observed <code>spark.eventLog.logStageExecutorMetrics=false</code> is <code>disabled</code> / <code>explicitlyDisabled</code>; otherwise <code>notEmitted</code> / <code>noObservedExecutorMetrics</code>. Observed metrics win over the explicit setting.</td></tr><tr><td><code>rddStorageSnapshots</code></td><td><code>rddStorageSnapshots &gt; 0</code> from <code>SparkListenerStageSubmitted</code></td><td>No stage submissions is <code>notEmitted</code> / <code>noObservedStageSubmission</code>; stages submitted without any <code>RDD Info</code> rows is <code>notEmitted</code> / <code>noRddStorageSnapshot</code>. This is RDD snapshot evidence, not block-update telemetry.</td></tr><tr><td><code>sqlPlan</code></td><td><code>resolvedSqlPlans &gt; 0</code></td><td>A SQL execution without a resolved plan is <code>notEmitted</code> / <code>noResolvedSqlPlan</code>; no SQL execution is <code>notApplicable</code> / <code>noSqlExecution</code>.</td></tr><tr><td><code>sparkConfiguration</code></td><td><code>environmentUpdates &gt; 0</code></td><td><code>notEmitted</code> / <code>noEnvironmentUpdate</code>; a normalized empty config object alone is not evidence.</td></tr><tr><td><code>taskCoreTime</code></td><td><code>taskRecords &gt; 0</code> and a usable <code>runAggregates.perStage</code> task count</td><td>No task records is <code>notEmitted</code> / <code>noTaskRecords</code>; task records with no usable aggregate are <code>notEmitted</code> / <code>noUsableCoreTimeAggregate</code>. An empty aggregate object is not usable evidence.</td></tr><tr><td><code>infrastructureContext</code></td><td>none</td><td>Always <code>outsideEventLog</code> / <code>outsideEventLogScope</code>.</td></tr><tr><td><code>sourceContext</code></td><td>none</td><td>Always <code>outsideEventLog</code> / <code>outsideEventLogScope</code>.</td></tr><tr><td><code>costContext</code></td><td>none</td><td>Always <code>outsideEventLog</code> / <code>outsideEventLogScope</code>.</td></tr></tbody></table><p>Parse integrity is fail-closed. A parse is trustworthy only when it skipped no malformed lines and observed an application-end event. For an untrustworthy parse, proof already observed remains <code>present</code>, and an explicitly observed executor-metrics disablement remains <code>disabled</code>; every conclusion based on absence or irrelevance becomes <code>unknown</code> / <code>parseIncomplete</code>. The three outside-event-log product boundaries remain <code>outsideEventLog</code>. In particular, <code>notEmitted</code> and <code>notApplicable</code> are never favorable measurements inferred from an incomplete parse.</p><p>Availability is model metadata, not a detector or finding: it never changes detector thresholds or ordering, impact band, catalog/scorecard counts, no-bottleneck behavior, or detector suppression. <code>session-snapshot.ts</code> captures and restores <code>evidenceAvailability</code> with the normalized model, so a recent-file switch retains the same ledger without reparsing.</p><h3 id="portable-evidence-report-v1" tabindex="-1">Portable evidence report (V1) <a class="header-anchor" href="#portable-evidence-report-v1" aria-label="Permalink to &quot;Portable evidence report (V1)&quot;">​</a></h3><p><code>packages/core/src/evidence-report.ts</code>&#39;s <code>buildEvidenceReport(appModel, { redact })</code> returns <code>{ markdown, json }</code>: a self-contained, byte-stable document that runs the detectors over an <code>appModel</code> and serializes the result for sharing outside the tool. Raw task records are never included; identifier redaction (app id + host names → <code>app-1</code>/<code>host-1</code> pseudonyms via <code>packages/core/src/redact.ts</code>) is opt-in with <code>{ redact: true }</code>. The JSON is pinned by <code>EVIDENCE_SCHEMA_VERSION</code> (currently <code>3</code>, surfaced as <code>json.schemaVersion</code>) and has this fixed top-level key order:</p><div class="language-text vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">text</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>schemaVersion, summary, evidenceAvailability, detectors, findings, recommendations, cleanChecks</span></span></code></pre></div><ul><li><code>summary</code> is the run header: <code>{ app: { id, name, sparkVersion }, stageCount, jobCount, sqlExecutionCount, findingCount, impactBandCounts }</code>.</li><li><code>evidenceAvailability</code> is the ledger above (or <code>null</code> when absent).</li><li><code>detectors</code> is <code>detectorCatalog()</code> output: one <code>{ type, version, scope, thresholds, docAnchor }</code> per detector, in <code>DETECTORS</code> order, so the exact threshold set that produced each finding travels with the evidence.</li><li><code>findings</code> are deterministically sorted rows (impact band → type → stage → id), each with a stable <code>id</code>, <code>tag</code>, core columns, an always-present <code>actionLabel</code>, and an <code>evidence</code> sub-object for non-core fields; <code>confidence</code>/ <code>validationRequired</code>/<code>docAnchor</code> appear only when the detector emitted them.</li><li><code>recommendations</code> is the impact-ranked <code>buildRecommendationRollup</code> output (<code>packages/core/src/recommendation-rollup.ts</code>), and <code>cleanChecks</code> lists every detector type that fired zero findings this run: see the 2026-09-03 update below.</li></ul><p>Determinism holds because detector order, finding sort, and object key order are all fixed, so a given <code>appModel</code> serializes identically across calls.</p><p>Redaction enumerates hosts two ways: by walking the findings tree for every string value under a key literally named <code>host</code> (<code>evidence.host</code>, <code>evidence.failedTaskDetails[].host</code>, <code>evidence.retriedTaskDetails[].host</code>, and any future nested <code>host</code> field, all covered without enumerating paths), and by scanning every string value for EC2-style hostnames / bare IPv4 tokens. So identifiers that surface only in free text (recommendation copy, a <code>stageFailed</code> failure-reason value) are pseudonymized too. Pseudonym numbering uses a numeric-aware sort, so re-redacting an already-redacted report is a no-op even past <code>host-10</code>.</p><p>The Markdown rendering mirrors the JSON&#39;s AC3 field set: each finding block prints its <code>detector version</code>, its sorted <code>evidence</code> entries (byte-magnitude keys humanized), and the report ends with a <code>## Detectors</code> catalog carrying the version + threshold set. A finding&#39;s <code>impactEstimate</code> (when its <code>basis</code> isn&#39;t <code>&#39;informational&#39;</code>) prints as its own <code>- impact: </code> line (<code>Estimated &lt;low&gt;-&lt;high&gt;</code> and/or the raw-waste figure, plus <code>estimateMethod</code>), via <code>renderImpactEstimate</code>/<code>formatWallClockRange</code>/<code>formatRawWaste</code> in <code>packages/core/src/evidence-report.ts</code>. <code>EvidenceExport</code> names downloads <code>evidence-&lt;appId&gt;[-redacted].&lt;md|json&gt;</code>, taking the app id from the (already pseudonymized when redacting) report so a redacted file never leaks the real id and is never name-identical to a raw export.</p><p>Decision 9 (design spec): <code>Finding</code>&#39;s <code>impactEstimate</code> field, plus the <code>utilizationFraction</code>/<code>memorySize</code>/<code>diskSize</code>/<code>numCachedPartitions</code>/<code>numPartitions</code> instrumentation fields the impact estimator reads, were added without bumping <code>EVIDENCE_SCHEMA_VERSION</code> past <code>1</code>. Both additions are purely optional and ride <code>Finding</code>&#39;s existing optional-field-plus-catch-all convention, so an evidence report built before these fields existed still deserializes and compares byte-for-byte against one built after: nothing about the schema&#39;s stability guarantee changed, only its surface grew. A future reader who notices <code>impactEstimate</code> in the JSON without a schema bump is looking at this deliberate call, not an oversight.</p><p>2026-08-30 update: <code>EVIDENCE_SCHEMA_VERSION</code> was bumped to <code>2</code> for the occupancy-weighted attribution redesign (see <a href="./impact-estimation.html#occupancy-weighted-attribution">Occupancy-weighted attribution</a>): <code>ImpactEstimate</code>&#39;s shape changed from <code>{low, high}</code> to <code>{basis, wallClock, estimateMethod, rawWaste?}</code>, a real, non-additive breaking change to a field this same Decision 9 previously shipped without a bump. <code>impactEstimate</code> had zero consumers outside <code>packages/core/src/impact-estimator.ts</code> and its own tests at the time of this bump (confirmed by grep across <code>src/view/*</code> and <code>packages/core/src/evidence-report.ts</code>), so no other code needed migrating alongside it.</p><p><code>FindingRow</code>&#39;s schema was extended to surface <code>impactEstimate</code> as a first-class column without further bumping <code>EVIDENCE_SCHEMA_VERSION</code> past <code>2</code>, consistent with <code>Finding</code>&#39;s existing optional-field-plus-catch-all convention. It appears after the pinned core columns (<code>id</code>, <code>type</code>, <code>impactBand</code>, <code>stageId</code>, <code>metric</code>, <code>value</code>, <code>recommendation</code>, <code>detectorVersion</code>, plus optional but pinned <code>confidence</code>, <code>validationRequired</code>, <code>docAnchor</code>) without displacing any of them; byte-for-byte deserializability of existing reports is preserved. <code>FindingRow.impactEstimate</code> carries the full contract documented in <a href="./impact-estimation.html#occupancy-weighted-attribution">Impact estimation</a> (basis, wallClock, estimateMethod, rawWaste).</p><p>2026-09-03 update: the &quot;Fix These First&quot; dashboard redesign (impact-ranked recommendation rollup, short per-finding action labels, a clean-checks table) was UI-only when it shipped; this update ports the underlying data into <code>buildEvidenceReport()</code> so the CLI (<code>packages/cli/bin/sparkforensics-analyze.mjs</code>) and the MCP tool (<code>diagnoseRun</code> in <code>packages/core/src/mcp-tools.ts</code>) get it too, not just the web markdown/JSON download. Three additions, all purely additive, so this does not bump <code>EVIDENCE_SCHEMA_VERSION</code> past <code>2</code>, the same rationale as the <code>impactEstimate</code> addition immediately above: <code>FindingRow.actionLabel</code> is now always present (a short imperative label like &quot;Reduce shuffle size&quot;), sourced from a new core module, <code>packages/core/src/finding-action-label.ts</code>&#39;s <code>coreFindingActionLabel</code>, extracted from the (type, discriminant) switch statement that used to live only in the view layer (<code>src/view/finding-action-label.ts</code>, which now wraps the core function and layers its own <code>REGISTRY</code> fallback on top). <code>EvidenceReportJson.recommendations</code> is the same impact-ranked <code>buildRecommendationRollup</code> grouping (<code>packages/core/src/recommendation-rollup.ts</code>) that <code>FixTheseFirst.tsx</code> renders, so CLI/MCP/download consumers get the same &quot;what&#39;s the highest-impact fix&quot; ranking the dashboard shows, without changing the existing <code>findings</code> array&#39;s own impact-band-sorted order at all. <code>EvidenceReportJson.cleanChecks</code> lists every detector type that fired zero findings this run, each with <code>getThresholdSummary</code>&#39;s one-line &quot;what would have tripped it&quot; sentence; unlike the dashboard&#39;s <code>Alerts.tsx</code> clean-checks table, it deliberately includes the one &quot;always-mounted&quot; reference type (<code>coreLocality</code>) even when it has no findings, since a flat evidence report has no separate always-visible surface for it to already appear on the way that widget does on the board.</p><h3 id="finding-identity" tabindex="-1">Finding identity <a class="header-anchor" href="#finding-identity" aria-label="Permalink to &quot;Finding identity&quot;">​</a></h3><p>Every finding <code>analyzer.ts</code> emits carries a stable <code>id</code> (<code>push()</code>&#39;s <code>findingId()</code> choke point, FNV-1a hash over <code>type | location | metric | value | discriminators</code>) and a <code>detectorVersion</code> (from the emitting <code>DETECTORS</code> entry). The two are deliberately decoupled: <code>id</code> is derived only from a finding&#39;s evidence tuple, never from <code>detectorVersion</code>, so a threshold or logic tweak that bumps a detector&#39;s <code>version</code> does not change the <code>id</code> of findings it still emits at the same location/metric. A saved reference (dashboard bookmark, exported report row, future URL-restored filter) keeps pointing at the same logical finding across detector revisions; <code>detectorVersion</code> is separate provenance metadata for &quot;which ruleset produced this,&quot; not part of identity. See <code>packages/core/test/analyzer-finding-identity.test.js</code> for the decoupling proof.</p><p><code>id</code> stability holds for equivalent reruns of the same schema/analyzer version on the same input. Changing the id-derivation rule itself (the hash algorithm, or which fields feed the location key/discriminators) is an intentional breaking change and must bump <code>EVIDENCE_SCHEMA_VERSION</code> (<code>packages/core/src/evidence-report.ts</code>); there is no separate id-scheme version.</p><h3 id="headless-analysis-cli-v1" tabindex="-1">Headless analysis CLI (V1) <a class="header-anchor" href="#headless-analysis-cli-v1" aria-label="Permalink to &quot;Headless analysis CLI (V1)&quot;">​</a></h3><p><code>packages/cli/bin/sparkforensics-analyze.mjs</code> runs the same parser + detector contracts outside the browser, for CI. It accepts a single event-log file or a rolling-log directory, drives <code>runParse</code>/<code>runParseFiles</code> (<code>packages/core/src/parser-worker.ts</code>) with a Node-only File-like shim (<code>packages/core/src/cli/collect-run.ts</code>), and writes the exact <code>buildEvidenceReport</code> JSON schema described above: one format, not a second. Alternatively, <code>--shs-base-url &lt;url&gt; --app-id &lt;id&gt; [--attempt-id &lt;id&gt;]</code> fetches the run from a Spark History Server instead (mutually exclusive with the positional file/directory argument), calling <code>resolveFromShs</code> (<code>packages/core/src/shs-load.ts</code>, shared with the MCP server&#39;s SHS source path below) directly; no dependency on the <code>packages/server</code> package. A failed SHS fetch reports its message to <code>stderr</code> and exits <code>2</code>, same as a local file that can&#39;t be parsed.</p><p>Optional CLI-flag budgets (<code>--max-runtime &lt;ms&gt;</code>, <code>--max-spill &lt;gb&gt;</code>, <code>--max-skew &lt;ratio&gt;</code>, <code>--max-failed-task-rate &lt;pct&gt;</code>, <code>--min-efficiency &lt;pct&gt;</code>) are evaluated in <code>packages/core/src/cli/budgets.ts</code> against the existing finding catalog (<code>analyze()</code>) and <code>computeEfficiencyModel</code>; there is no second rule engine. A budget whose required evidence is missing (e.g. the run never emitted <code>ApplicationEnd</code>, or has no usable per-task <code>runAggregates</code>) is reported as inconclusive (<code>stderr</code> warning) rather than silently passing, and gets its own exit code distinct from both pass and violation. Exit codes: <code>0</code> pass, <code>1</code> a configured budget was violated, <code>2</code> the input could not be parsed at all, <code>3</code> no violations but at least one budget was inconclusive. A violation always wins over an inconclusive result in the same run (exit <code>1</code>, not <code>3</code>).</p><h3 id="mcp-server-v1" tabindex="-1">MCP server (V1) <a class="header-anchor" href="#mcp-server-v1" aria-label="Permalink to &quot;MCP server (V1)&quot;">​</a></h3><p><code>packages/core/src/mcp-server-factory.ts</code>&#39;s <code>createMcpServer()</code> registers 6 tools: <code>diagnose_run</code> (thresholded findings + remediation text), <code>get_run_summary</code> (app/stage/job/sql counts and duration, no findings), <code>compare_runs</code> (categorized findings delta + metric deltas between two runs), <code>evaluate_budgets</code> (pass/fail budget thresholds against one run, optionally with a second run for regression/fail-on-introduced budgets: the MCP side of the CLI&#39;s <code>evaluateBudgets()</code> gating), <code>get_finding_evidence</code> (raw evidence bundle for one finding, for drill-down after <code>diagnose_run</code>), and <code>get_finding_documentation</code> (detection/tuning reference docs for one finding type, independent of any run). None re-implement detector logic; all repackage <code>analyze</code>/<code>buildEvidenceReport</code>/<code>compareRuns</code>/<code>captureSnapshot</code>/<code>evaluateBudgets</code> from <code>packages/core/src/mcp-tools.ts</code>, which resolves a <code>source</code> (event-log <code>path</code>, or an SHS <code>shsBaseUrl</code>/<code>appId</code>/<code>attemptId</code> triple) into a cached <code>AppModel</code>. The SHS source path calls <code>resolveFromShs</code> (<code>packages/core/src/shs-load.ts</code>, also used directly by the CLI&#39;s <code>--shs-base-url</code> mode above), which reuses two extractions shared with the browser ingestion flow: <code>fetchShsEventLog</code> (<code>packages/core/src/proxy.js</code>) fetches the zip, and <code>decodeShsArchive</code> (<code>packages/core/src/shs-fetch.ts</code>, re-exported from <code>parser-worker.ts</code>&#39;s barrel) decodes it into a parsed <code>AppModel</code>, the same split <code>runParseFromUrl</code> itself calls into.</p><p>Two transports connect to that one factory: <code>packages/mcp/bin/sparkforensics-mcp.mjs</code> (stdio, for local MCP clients) and <code>packages/server/index.js</code>&#39;s <code>/mcp</code> route (streamable HTTP, for the local server). The run cache (<code>packages/core/src/mcp-tools.ts</code>) is a module-level LRU (cap 8 and 15-minute idle TTL by default, overridable via <code>SPARKFORENSICS_MCP_CACHE_CAP</code> and <code>SPARKFORENSICS_MCP_CACHE_TTL_MS</code>, lazily swept on access) keyed by resolved source (path mtime, or SHS baseUrl+appId+attemptId), so a client mints a <code>runId</code> once via <code>resolveOrCreateRun</code> and reuses it across subsequent tool calls instead of re-parsing.</p><p>Every tool failure comes back as <code>{isError: true, content: [...], structuredContent: {code}}</code>, never an HTTP-status-shaped error; <code>code</code> is one of the 5 existing SHS codes (<code>SHS_ERROR_CODES</code>, <code>packages/core/src/shs-request.js</code>) plus <code>run-not-found</code>, <code>finding-not-found</code>, <code>invalid-event-log</code> (also covers a nonexistent <code>path</code> source), and <code>archive-too-large</code> (SHS archive over the <code>SPARKFORENSICS_MAX_ARCHIVE_BYTES</code> byte cap, default 1 GiB, because the MCP path buffers the whole archive in memory, unlike the streaming <code>/shs-proxy</code> route).</p><p><code>scripts/vendor-core.mjs</code> (shared by <code>packages/cli</code>, <code>packages/mcp</code>, and <code>packages/server</code>&#39;s <code>prepack</code> scripts) vendors <code>packages/core/src/</code> wholesale into each package&#39;s own <code>vendor-core/</code> at pack time, pre-stripping TypeScript to plain <code>.js</code> (Node&#39;s native TS stripping refuses to run on <code>.ts</code> files under <code>node_modules</code>, which is exactly where a published <code>vendor-core/</code> lands). Each package&#39;s bin/entry point resolves its needed module from <code>vendor-core/</code> if present, else falls back to the real <code>packages/core/src/</code> sibling loaded as <code>.ts</code> directly (<code>packages/server/index.js</code> mirrors <code>resolveStaticRoot</code>&#39;s <code>public/</code>-vs-<code>../dist</code> pattern for this same fallback).</p><h2 id="typescript-core-and-runtime-event-validation" tabindex="-1">TypeScript core and runtime event validation <a class="header-anchor" href="#typescript-core-and-runtime-event-validation" aria-label="Permalink to &quot;TypeScript core and runtime event validation&quot;">​</a></h2><p>All of <code>src/</code> (browser SPA) and <code>packages/core/src/</code> (shared analysis logic) is strict TypeScript; the only plain-<code>.js</code> holdouts are the two vendored third-party decompressors, <code>packages/core/src/vendor/fflate.js</code> and <code>packages/core/src/vendor/fzstd.js</code> (left untouched deliberately: vendored code, not project code). Every remaining import of a same-repo module uses a <code>.ts</code> specifier (e.g. <code>import { dispatchLine } from &#39;./event-handlers.ts&#39;</code>), not <code>.js</code>: the CLI and MCP entrypoints (<code>packages/cli/bin/sparkforensics-analyze.mjs</code>, <code>packages/mcp/bin/sparkforensics-mcp.mjs</code>) run under plain Node&#39;s ESM resolver, which cannot remap a <code>.js</code> specifier to a same-named <code>.ts</code> file the way Vite/Vitest&#39;s bundler-style resolver can. <code>tsconfig.json</code> sets <code>&quot;allowImportingTsExtensions&quot;: true</code> to make this legal.</p><p>Event schema validation lives in <code>packages/core/src/event-schemas.ts</code>: one zod schema per <code>SparkListener*</code> event variant the parser understands (15 total: <code>LogStart</code>, <code>ApplicationStart</code>, <code>EnvironmentUpdate</code>, <code>ApplicationEnd</code>, <code>JobStart</code>, <code>JobEnd</code>, <code>StageSubmitted</code>, <code>StageCompleted</code>, <code>StageExecutorMetrics</code>, <code>TaskEnd</code>, the three SQL-execution-UI listener events, <code>ExecutorAdded</code>, <code>ExecutorRemoved</code>), combined into <code>SparkEventSchema = z.discriminatedUnion(&#39;Event&#39;, [...])</code>. <code>processEvent</code>&#39;s switch (<code>event-handlers.ts</code>) consumes the resulting <code>SparkEvent</code> union type directly, so a schema change and a handler&#39;s expectations can&#39;t silently drift apart.</p><p>The recursive <code>sparkPlanInfo.children</code> tree carried on SQL-execution-start events is validated by <code>parseSparkPlanInfoTree</code>, an iterative, explicit heap-allocated-stack parser, deliberately not <code>z.lazy()</code>. A real plan tree&#39;s depth is unbounded, and attacker-uncontrolled input should never hand zod&#39;s own recursive schema resolution an arbitrarily deep structure to walk. The iterative parser shallow-validates one node at a time via <code>z.object(...)</code> and builds the tree itself, capping at <code>MAX_PLAN_DEPTH = 500</code> (throws past that). This mirrors <code>packages/core/src/plan-tree-walk.ts</code>&#39;s <code>walkPlanTree</code>, which uses the same iterative-over-recursive approach for the <em>resolved</em> tree; <code>event-schemas.ts</code> applies it one layer earlier, to the raw JSON before it becomes a tree at all.</p><p>There are two external-data boundaries, the two places this codebase parses data it does not control. Both now run that data through a schema, and both treat a validation failure the same way: a silent skip, not a distinct error.</p><ul><li><code>dispatchLine</code> (<code>event-handlers.ts</code>): after <code>JSON.parse</code> succeeds, a line whose <code>Event</code> value is one of the 15 modeled types but fails that type&#39;s own schema now increments <code>skippedLines</code> (previously: no shape validation existed at all, and a malformed event silently corrupted downstream state with no signal anywhere). An <code>Event</code> value outside the 15 modeled types is still silently ignored without incrementing <code>skippedLines</code>, unchanged from before migration (see the note below on why the broader design was rejected).</li><li><code>shs-fetch.ts</code>: on a non-OK response from the local SHS proxy, the JSON error envelope is validated against <code>ShsProxyErrorBodySchema</code> (<code>packages/core/src/shs-schemas.ts</code>, <code>{ code: string }</code>, <code>.passthrough()</code>). A body that isn&#39;t valid JSON, or is JSON but fails that schema, falls back to the generic <code>access-or-upstream-failure</code> code: the same silent-skip treatment as a malformed log line, collapsing what could have been a separate &quot;malformed SHS response&quot; error code into the existing generic one.</li></ul><p>Neither boundary distinguishes &quot;malformed JSON&quot; from &quot;wrong shape&quot; from &quot;unrecognized variant&quot; in what it reports outward: all three collapse into the same skip/fallback path. That was a deliberate scope decision. The correction below says why the <em>line</em> boundary stops there rather than flagging every unrecognized event type.</p><blockquote><p>Correction made during migration, not part of the original design: an earlier draft counted <em>any</em> <code>Event</code> value outside the 15 modeled types toward <code>skippedLines</code>, not just ones that fail their own schema. Running that design against real Spark event logs (which always contain plenty of ordinary event types this tool has never modeled: <code>TaskStart</code>, <code>BlockManagerAdded</code>, <code>ExecutorMetricsUpdate</code>, and others, always silently ignored pre-migration) pushed <code>skippedLines</code> from 0 to the tens of thousands on completely healthy logs. That would have tripped <code>evidence-availability.ts</code>&#39;s fail-closed <code>trustworthy</code> gate and shown a false &quot;malformed JSON&quot; warning on every real file. Caught in review against real fixtures, not synthetic ones; fixed to the narrower rule described above before merging. <code>evidence-availability.ts</code>&#39;s gate itself (<code>trustworthy = skippedLines === 0 &amp;&amp; applicationEnds &gt; 0</code>) needed no code change once this was fixed: it was already correct, the input feeding it was not.</p></blockquote><p>The exhaustiveness convention is <code>packages/core/src/assert-never.ts</code>. <code>assertNever(x: never): never</code> throws at runtime and, more importantly, fails <code>tsc</code> at compile time if <code>x</code> is not actually <code>never</code>, i.e. if some case of a union type isn&#39;t handled. Used today at the <code>default</code> arm of <code>event-handlers.ts</code>&#39;s <code>processEvent</code> switch (over <code>SparkEvent</code>) and <code>analyzer.ts</code>&#39;s scope-dispatch switch (over a detector&#39;s <code>scope: &#39;stage&#39; | &#39;sql&#39; | &#39;app&#39; | &#39;config&#39;</code>). It is the project&#39;s standard pattern for any future exhaustive switch or dispatch over a closed union: add <code>default: return assertNever(x);</code> (or the closest non-returning equivalent) so that adding a new union member without updating every consumer becomes a compile error instead of a silent runtime gap.</p>`,55)])])}const m=o(c,[["render",s]]);export{u as __pageData,m as default};
+<span class="line"><span>outsideEventLogScope, parseIncomplete</span></span></code></pre></div><p><code>summary</code> is domain-agnostic presentation copy derived from the reason code; the enum values, not labels, are the machine contract. Optional <code>evidence</code> is limited to safe compact provenance of the exact <code>{ eventType, count }</code> shape. <code>eventType</code> values are pinned to the exact <code>evidenceInputs</code> counter keys; they are not shortened or aliased. It must never include raw Spark-property values, host names, paths, SQL text, task records, raw-event identifiers, or other event payloads.</p><table tabindex="0"><thead><tr><th>Key</th><th>Proven <code>present</code> evidence</th><th>Trustworthy absence / boundary result</th></tr></thead><tbody><tr><td><code>executorMetrics</code></td><td><code>executorMetricRows &gt; 0</code></td><td>Explicit observed <code>spark.eventLog.logStageExecutorMetrics=false</code> is <code>disabled</code> / <code>explicitlyDisabled</code>; otherwise <code>notEmitted</code> / <code>noObservedExecutorMetrics</code>. Observed metrics win over the explicit setting.</td></tr><tr><td><code>rddStorageSnapshots</code></td><td><code>rddStorageSnapshots &gt; 0</code> from <code>SparkListenerStageSubmitted</code></td><td>No stage submissions is <code>notEmitted</code> / <code>noObservedStageSubmission</code>; stages submitted without any <code>RDD Info</code> rows is <code>notEmitted</code> / <code>noRddStorageSnapshot</code>. This is RDD snapshot evidence, not block-update telemetry.</td></tr><tr><td><code>sqlPlan</code></td><td><code>resolvedSqlPlans &gt; 0</code></td><td>A SQL execution without a resolved plan is <code>notEmitted</code> / <code>noResolvedSqlPlan</code>; no SQL execution is <code>notApplicable</code> / <code>noSqlExecution</code>.</td></tr><tr><td><code>sparkConfiguration</code></td><td><code>environmentUpdates &gt; 0</code></td><td><code>notEmitted</code> / <code>noEnvironmentUpdate</code>; a normalized empty config object alone is not evidence.</td></tr><tr><td><code>taskCoreTime</code></td><td><code>taskRecords &gt; 0</code> and a usable <code>runAggregates.perStage</code> task count</td><td>No task records is <code>notEmitted</code> / <code>noTaskRecords</code>; task records with no usable aggregate are <code>notEmitted</code> / <code>noUsableCoreTimeAggregate</code>. An empty aggregate object is not usable evidence.</td></tr><tr><td><code>infrastructureContext</code></td><td>none</td><td>Always <code>outsideEventLog</code> / <code>outsideEventLogScope</code>.</td></tr><tr><td><code>sourceContext</code></td><td>none</td><td>Always <code>outsideEventLog</code> / <code>outsideEventLogScope</code>.</td></tr><tr><td><code>costContext</code></td><td>none</td><td>Always <code>outsideEventLog</code> / <code>outsideEventLogScope</code>.</td></tr></tbody></table><p>Parse integrity is fail-closed. A parse is trustworthy only when it skipped no malformed lines and observed an application-end event. For an untrustworthy parse, proof already observed remains <code>present</code>, and an explicitly observed executor-metrics disablement remains <code>disabled</code>; every conclusion based on absence or irrelevance becomes <code>unknown</code> / <code>parseIncomplete</code>. The three outside-event-log product boundaries remain <code>outsideEventLog</code>. In particular, <code>notEmitted</code> and <code>notApplicable</code> are never favorable measurements inferred from an incomplete parse.</p><p>Availability is model metadata, not a detector or finding: it never changes detector thresholds or ordering, impact band, catalog/scorecard counts, no-bottleneck behavior, or detector suppression. <code>session-snapshot.ts</code> captures and restores <code>evidenceAvailability</code> with the normalized model, so a recent-file switch retains the same ledger without reparsing.</p><h3 id="portable-evidence-report-v1" tabindex="-1">Portable evidence report (V1) <a class="header-anchor" href="#portable-evidence-report-v1" aria-label="Permalink to &quot;Portable evidence report (V1)&quot;">​</a></h3><p><code>packages/core/src/evidence-report.ts</code>&#39;s <code>buildEvidenceReport(appModel, { redact })</code> returns <code>{ markdown, json }</code>: a self-contained, byte-stable document that runs the detectors over an <code>appModel</code> and serializes the result for sharing outside the tool. Raw task records are never included; identifier redaction (app id + host names → <code>app-1</code>/<code>host-1</code> pseudonyms via <code>packages/core/src/redact.ts</code>) is opt-in with <code>{ redact: true }</code>. The JSON is pinned by <code>EVIDENCE_SCHEMA_VERSION</code> (currently <code>3</code>, surfaced as <code>json.schemaVersion</code>) and has this fixed top-level key order:</p><div class="language-text vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">text</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span>schemaVersion, summary, evidenceAvailability, detectors, findings, recommendations, cleanChecks</span></span></code></pre></div><ul><li><code>summary</code> is the run header: <code>{ app: { id, name, sparkVersion }, stageCount, jobCount, sqlExecutionCount, findingCount, impactBandCounts }</code>.</li><li><code>evidenceAvailability</code> is the ledger above (or <code>null</code> when absent).</li><li><code>detectors</code> is <code>detectorCatalog()</code> output: one <code>{ type, version, scope, thresholds, docAnchor }</code> per detector, in <code>DETECTORS</code> order, so the exact threshold set that produced each finding travels with the evidence.</li><li><code>findings</code> are deterministically sorted rows (impact band → type → stage → id), each with a stable <code>id</code>, <code>tag</code>, core columns, an always-present <code>actionLabel</code>, and an <code>evidence</code> sub-object for non-core fields; <code>confidence</code>/ <code>validationRequired</code>/<code>docAnchor</code> appear only when the detector emitted them.</li><li><code>recommendations</code> is the impact-ranked <code>buildRecommendationRollup</code> output (<code>packages/core/src/recommendation-rollup.ts</code>), and <code>cleanChecks</code> lists every detector type that fired zero findings this run: see the 2026-09-03 update below.</li></ul><p>Determinism holds because detector order, finding sort, and object key order are all fixed, so a given <code>appModel</code> serializes identically across calls.</p><p>Redaction enumerates hosts two ways: by walking the findings tree for every string value under a key literally named <code>host</code> (<code>evidence.host</code>, <code>evidence.failedTaskDetails[].host</code>, <code>evidence.retriedTaskDetails[].host</code>, and any future nested <code>host</code> field, all covered without enumerating paths), and by scanning every string value for EC2-style hostnames / bare IPv4 tokens. So identifiers that surface only in free text (recommendation copy, a <code>stageFailed</code> failure-reason value) are pseudonymized too. Pseudonym numbering uses a numeric-aware sort, so re-redacting an already-redacted report is a no-op even past <code>host-10</code>.</p><p>The Markdown rendering mirrors the JSON&#39;s AC3 field set: each finding block prints its <code>detector version</code>, its sorted <code>evidence</code> entries (byte-magnitude keys humanized), and the report ends with a <code>## Detectors</code> catalog carrying the version + threshold set. A finding&#39;s <code>impactEstimate</code> (when its <code>basis</code> isn&#39;t <code>&#39;informational&#39;</code>) prints as its own <code>- impact: </code> line (<code>Estimated &lt;low&gt;-&lt;high&gt;</code> and/or the raw-waste figure, plus <code>estimateMethod</code>), via <code>renderImpactEstimate</code>/<code>formatWallClockRange</code>/<code>formatRawWaste</code> in <code>packages/core/src/evidence-report.ts</code>. <code>EvidenceExport</code> names downloads <code>evidence-&lt;appId&gt;[-redacted].&lt;md|json&gt;</code>, taking the app id from the (already pseudonymized when redacting) report so a redacted file never leaks the real id and is never name-identical to a raw export.</p><p>Decision 9 (design spec): <code>Finding</code>&#39;s <code>impactEstimate</code> field, plus the <code>utilizationFraction</code>/<code>memorySize</code>/<code>diskSize</code>/<code>numCachedPartitions</code>/<code>numPartitions</code> instrumentation fields the impact estimator reads, were added without bumping <code>EVIDENCE_SCHEMA_VERSION</code> past <code>1</code>. Both additions are purely optional and ride <code>Finding</code>&#39;s existing optional-field-plus-catch-all convention, so an evidence report built before these fields existed still deserializes and compares byte-for-byte against one built after: nothing about the schema&#39;s stability guarantee changed, only its surface grew. A future reader who notices <code>impactEstimate</code> in the JSON without a schema bump is looking at this deliberate call, not an oversight.</p><p>2026-08-30 update: <code>EVIDENCE_SCHEMA_VERSION</code> was bumped to <code>2</code> for the occupancy-weighted attribution redesign (see <a href="./impact-estimation.html#occupancy-weighted-attribution">Occupancy-weighted attribution</a>): <code>ImpactEstimate</code>&#39;s shape changed from <code>{low, high}</code> to <code>{basis, wallClock, estimateMethod, rawWaste?}</code>, a real, non-additive breaking change to a field this same Decision 9 previously shipped without a bump. <code>impactEstimate</code> had zero consumers outside <code>packages/core/src/impact-estimator.ts</code> and its own tests at the time of this bump (confirmed by grep across <code>src/view/*</code> and <code>packages/core/src/evidence-report.ts</code>), so no other code needed migrating alongside it.</p><p><code>FindingRow</code>&#39;s schema was extended to surface <code>impactEstimate</code> as a first-class column without further bumping <code>EVIDENCE_SCHEMA_VERSION</code> past <code>2</code>, consistent with <code>Finding</code>&#39;s existing optional-field-plus-catch-all convention. It appears after the pinned core columns (<code>id</code>, <code>type</code>, <code>impactBand</code>, <code>stageId</code>, <code>metric</code>, <code>value</code>, <code>recommendation</code>, <code>detectorVersion</code>, plus optional but pinned <code>confidence</code>, <code>validationRequired</code>, <code>docAnchor</code>) without displacing any of them; byte-for-byte deserializability of existing reports is preserved. <code>FindingRow.impactEstimate</code> carries the full contract documented in <a href="./impact-estimation.html#occupancy-weighted-attribution">Impact estimation</a> (basis, wallClock, estimateMethod, rawWaste).</p><p>2026-09-03 update: the &quot;Fix These First&quot; dashboard redesign (impact-ranked recommendation rollup, short per-finding action labels, a clean-checks table) was UI-only when it shipped; this update ports the underlying data into <code>buildEvidenceReport()</code> so the CLI (<code>packages/cli/bin/sparkforensics-analyze.mjs</code>) and the MCP tool (<code>diagnoseRun</code> in <code>packages/core/src/mcp-tools.ts</code>) get it too, not just the web markdown/JSON download. Three additions, all purely additive, so this does not bump <code>EVIDENCE_SCHEMA_VERSION</code> past <code>2</code>, the same rationale as the <code>impactEstimate</code> addition immediately above: <code>FindingRow.actionLabel</code> is now always present (a short imperative label like &quot;Reduce shuffle size&quot;), sourced from a new core module, <code>packages/core/src/finding-action-label.ts</code>&#39;s <code>coreFindingActionLabel</code>, extracted from the (type, discriminant) switch statement that used to live only in the view layer (<code>src/view/finding-action-label.ts</code>, which now wraps the core function and layers its own <code>REGISTRY</code> fallback on top). <code>EvidenceReportJson.recommendations</code> is the same impact-ranked <code>buildRecommendationRollup</code> grouping (<code>packages/core/src/recommendation-rollup.ts</code>) that <code>FixTheseFirst.tsx</code> renders, so CLI/MCP/download consumers get the same &quot;what&#39;s the highest-impact fix&quot; ranking the dashboard shows, without changing the existing <code>findings</code> array&#39;s own impact-band-sorted order at all. <code>EvidenceReportJson.cleanChecks</code> lists every detector type that fired zero findings this run, each with <code>getThresholdSummary</code>&#39;s one-line &quot;what would have tripped it&quot; sentence; unlike the dashboard&#39;s <code>Alerts.tsx</code> clean-checks table, it deliberately includes the one &quot;always-mounted&quot; reference type (<code>coreLocality</code>) even when it has no findings, since a flat evidence report has no separate always-visible surface for it to already appear on the way that widget does on the board.</p><h3 id="finding-identity" tabindex="-1">Finding identity <a class="header-anchor" href="#finding-identity" aria-label="Permalink to &quot;Finding identity&quot;">​</a></h3><p>Every finding <code>analyzer.ts</code> emits carries a stable <code>id</code> (<code>push()</code>&#39;s <code>findingId()</code> choke point, FNV-1a hash over <code>type | location | metric | value | discriminators</code>) and a <code>detectorVersion</code> (from the emitting <code>DETECTORS</code> entry). The two are deliberately decoupled: <code>id</code> is derived only from a finding&#39;s evidence tuple, never from <code>detectorVersion</code>, so a threshold or logic tweak that bumps a detector&#39;s <code>version</code> does not change the <code>id</code> of findings it still emits at the same location/metric. A saved reference (dashboard bookmark, exported report row, future URL-restored filter) keeps pointing at the same logical finding across detector revisions; <code>detectorVersion</code> is separate provenance metadata for &quot;which ruleset produced this,&quot; not part of identity. See <code>packages/core/test/analyzer-finding-identity.test.js</code> for the decoupling proof.</p><p><code>id</code> stability holds for equivalent reruns of the same schema/analyzer version on the same input. Changing the id-derivation rule itself (the hash algorithm, or which fields feed the location key/discriminators) is an intentional breaking change and must bump <code>EVIDENCE_SCHEMA_VERSION</code> (<code>packages/core/src/evidence-report.ts</code>); there is no separate id-scheme version.</p><h3 id="headless-analysis-cli-v1" tabindex="-1">Headless analysis CLI (V1) <a class="header-anchor" href="#headless-analysis-cli-v1" aria-label="Permalink to &quot;Headless analysis CLI (V1)&quot;">​</a></h3><p><code>packages/cli/bin/sparkforensics-analyze.mjs</code> runs the same parser + detector contracts outside the browser, for CI. It accepts a single event-log file or a rolling-log directory, drives <code>runParse</code>/<code>runParseFiles</code> (<code>packages/core/src/parser-worker.ts</code>) with a Node-only File-like shim (<code>packages/core/src/cli/collect-run.ts</code>), and writes the exact <code>buildEvidenceReport</code> JSON schema described above: one format, not a second. Alternatively, <code>--shs-base-url &lt;url&gt; --app-id &lt;id&gt; [--attempt-id &lt;id&gt;]</code> fetches the run from a Spark History Server instead (mutually exclusive with the positional file/directory argument), calling <code>resolveFromShs</code> (<code>packages/core/src/shs-load.ts</code>, shared with the MCP server&#39;s SHS source path below) directly; no dependency on the <code>packages/server</code> package. A failed SHS fetch reports its message to <code>stderr</code> and exits <code>2</code>, same as a local file that can&#39;t be parsed.</p><p>Optional CLI-flag budgets (<code>--max-runtime &lt;ms&gt;</code>, <code>--max-spill &lt;gb&gt;</code>, <code>--max-skew &lt;ratio&gt;</code>, <code>--max-failed-task-rate &lt;pct&gt;</code>, <code>--min-efficiency &lt;pct&gt;</code>) are evaluated in <code>packages/core/src/cli/budgets.ts</code> against the existing finding catalog (<code>analyze()</code>) and <code>computeEfficiencyModel</code>; there is no second rule engine. A budget whose required evidence is missing (e.g. the run never emitted <code>ApplicationEnd</code>, or has no usable per-task <code>runAggregates</code>) is reported as inconclusive (<code>stderr</code> warning) rather than silently passing, and gets its own exit code distinct from both pass and violation. Exit codes: <code>0</code> pass, <code>1</code> a configured budget was violated, <code>2</code> the input could not be parsed at all, <code>3</code> no violations but at least one budget was inconclusive. A violation always wins over an inconclusive result in the same run (exit <code>1</code>, not <code>3</code>).</p><h3 id="mcp-server-v1" tabindex="-1">MCP server (V1) <a class="header-anchor" href="#mcp-server-v1" aria-label="Permalink to &quot;MCP server (V1)&quot;">​</a></h3><p><code>packages/core/src/mcp-server-factory.ts</code>&#39;s <code>createMcpServer()</code> registers 6 tools: <code>diagnose_run</code> (thresholded findings + remediation text), <code>get_run_summary</code> (app/stage/job/sql counts and duration, no findings), <code>compare_runs</code> (categorized findings delta + metric deltas between two runs), <code>evaluate_budgets</code> (pass/fail budget thresholds against one run, optionally with a second run for regression/fail-on-introduced budgets: the MCP side of the CLI&#39;s <code>evaluateBudgets()</code> gating), <code>get_finding_evidence</code> (raw evidence bundle for one finding, for drill-down after <code>diagnose_run</code>), and <code>get_finding_documentation</code> (detection/tuning reference docs for one finding type, independent of any run). None re-implement detector logic; all repackage <code>analyze</code>/<code>buildEvidenceReport</code>/<code>compareRuns</code>/<code>captureSnapshot</code>/<code>evaluateBudgets</code> from <code>packages/core/src/mcp-tools.ts</code>, which resolves a <code>source</code> (event-log <code>path</code>, or an SHS <code>shsBaseUrl</code>/<code>appId</code>/<code>attemptId</code> triple) into a cached <code>AppModel</code>. The SHS source path calls <code>resolveFromShs</code> (<code>packages/core/src/shs-load.ts</code>, also used directly by the CLI&#39;s <code>--shs-base-url</code> mode above), which reuses two extractions shared with the browser ingestion flow: <code>fetchShsEventLog</code> (<code>packages/core/src/proxy.js</code>) fetches the zip, and <code>decodeShsArchive</code> (<code>packages/core/src/shs-fetch.ts</code>, re-exported from <code>parser-worker.ts</code>&#39;s barrel) decodes it into a parsed <code>AppModel</code>, the same split <code>runParseFromUrl</code> itself calls into.</p><p>Two transports connect to that one factory: <code>packages/mcp/bin/sparkforensics-mcp.mjs</code> (stdio, for local MCP clients) and <code>packages/server/index.js</code>&#39;s <code>/mcp</code> route (streamable HTTP, for the local server). The run cache (<code>packages/core/src/mcp-tools.ts</code>) is a module-level LRU (cap 8 and 15-minute idle TTL by default, overridable via <code>SPARKFORENSICS_MCP_CACHE_CAP</code> and <code>SPARKFORENSICS_MCP_CACHE_TTL_MS</code>, lazily swept on access) keyed by resolved source (path mtime, or SHS baseUrl+appId+attemptId), so a client mints a <code>runId</code> once via <code>resolveOrCreateRun</code> and reuses it across subsequent tool calls instead of re-parsing.</p><p>Every tool failure comes back as <code>{isError: true, content: [...], structuredContent: {code}}</code>, never an HTTP-status-shaped error; <code>code</code> is one of the 5 existing SHS codes (<code>SHS_ERROR_CODES</code>, <code>packages/core/src/shs-request.js</code>) plus <code>run-not-found</code>, <code>finding-not-found</code>, <code>invalid-event-log</code> (also covers a nonexistent <code>path</code> source), and <code>archive-too-large</code> (SHS archive over the <code>SPARKFORENSICS_MAX_ARCHIVE_BYTES</code> byte cap, default 1 GiB, because the MCP path buffers the whole archive in memory, unlike the streaming <code>/shs-proxy</code> route).</p><p><code>scripts/vendor-core.mjs</code> (shared by <code>packages/cli</code>, <code>packages/mcp</code>, and <code>packages/server</code>&#39;s <code>prepack</code> scripts) vendors <code>packages/core/src/</code> wholesale into each package&#39;s own <code>vendor-core/</code> at pack time, pre-stripping TypeScript to plain <code>.js</code> (Node&#39;s native TS stripping refuses to run on <code>.ts</code> files under <code>node_modules</code>, which is exactly where a published <code>vendor-core/</code> lands). Each package&#39;s bin/entry point resolves its needed module from <code>vendor-core/</code> if present, else falls back to the real <code>packages/core/src/</code> sibling loaded as <code>.ts</code> directly (<code>packages/server/index.js</code> mirrors <code>resolveStaticRoot</code>&#39;s <code>public/</code>-vs-<code>../dist</code> pattern for this same fallback).</p><h2 id="typescript-core-and-runtime-event-validation" tabindex="-1">TypeScript core and runtime event validation <a class="header-anchor" href="#typescript-core-and-runtime-event-validation" aria-label="Permalink to &quot;TypeScript core and runtime event validation&quot;">​</a></h2><p>All of <code>src/</code> (browser SPA) and <code>packages/core/src/</code> (shared analysis logic) is strict TypeScript; the only plain-<code>.js</code> holdouts are the two vendored third-party decompressors, <code>packages/core/src/vendor/fflate.js</code> and <code>packages/core/src/vendor/fzstd.js</code> (left untouched deliberately: vendored code, not project code). Every remaining import of a same-repo module uses a <code>.ts</code> specifier (e.g. <code>import { dispatchLine } from &#39;./event-handlers.ts&#39;</code>), not <code>.js</code>: the CLI and MCP entrypoints (<code>packages/cli/bin/sparkforensics-analyze.mjs</code>, <code>packages/mcp/bin/sparkforensics-mcp.mjs</code>) run under plain Node&#39;s ESM resolver, which cannot remap a <code>.js</code> specifier to a same-named <code>.ts</code> file the way Vite/Vitest&#39;s bundler-style resolver can. <code>tsconfig.json</code> sets <code>&quot;allowImportingTsExtensions&quot;: true</code> to make this legal.</p><p>Event schema validation lives in <code>packages/core/src/event-schemas.ts</code>: one zod schema per <code>SparkListener*</code> event variant the parser understands (15 total: <code>LogStart</code>, <code>ApplicationStart</code>, <code>EnvironmentUpdate</code>, <code>ApplicationEnd</code>, <code>JobStart</code>, <code>JobEnd</code>, <code>StageSubmitted</code>, <code>StageCompleted</code>, <code>StageExecutorMetrics</code>, <code>TaskEnd</code>, the three SQL-execution-UI listener events, <code>ExecutorAdded</code>, <code>ExecutorRemoved</code>), combined into <code>SparkEventSchema = z.discriminatedUnion(&#39;Event&#39;, [...])</code>. <code>processEvent</code>&#39;s switch (<code>event-handlers.ts</code>) consumes the resulting <code>SparkEvent</code> union type directly, so a schema change and a handler&#39;s expectations can&#39;t silently drift apart.</p><p>The recursive <code>sparkPlanInfo.children</code> tree carried on SQL-execution-start events is validated by <code>parseSparkPlanInfoTree</code>, an iterative, explicit heap-allocated-stack parser, deliberately not <code>z.lazy()</code>. A real plan tree&#39;s depth is unbounded, and attacker-uncontrolled input should never hand zod&#39;s own recursive schema resolution an arbitrarily deep structure to walk. The iterative parser shallow-validates one node at a time via <code>z.object(...)</code> and builds the tree itself, capping at <code>MAX_PLAN_DEPTH = 500</code> (throws past that). This mirrors <code>packages/core/src/plan-tree-walk.ts</code>&#39;s <code>walkPlanTree</code>, which uses the same iterative-over-recursive approach for the <em>resolved</em> tree; <code>event-schemas.ts</code> applies it one layer earlier, to the raw JSON before it becomes a tree at all.</p><p>There are two external-data boundaries, the two places this codebase parses data it does not control. Both now run that data through a schema, and both treat a validation failure the same way: a silent skip, not a distinct error.</p><ul><li><code>dispatchLine</code> (<code>event-handlers.ts</code>): after <code>JSON.parse</code> succeeds, a line whose <code>Event</code> value is one of the 15 modeled types but fails that type&#39;s own schema now increments <code>skippedLines</code> (previously: no shape validation existed at all, and a malformed event silently corrupted downstream state with no signal anywhere). An <code>Event</code> value outside the 15 modeled types is still silently ignored without incrementing <code>skippedLines</code>, unchanged from before migration (see the note below on why the broader design was rejected). One exception: an AQE update that a later update for the same open execution supersedes is never parsed (<code>deferAdaptiveUpdate</code>, see <a href="./detector-contract.html">the detector contract</a>), so a malformed superseded update is not counted.</li><li><code>shs-fetch.ts</code>: on a non-OK response from the local SHS proxy, the JSON error envelope is validated against <code>ShsProxyErrorBodySchema</code> (<code>packages/core/src/shs-schemas.ts</code>, <code>{ code: string }</code>, <code>.passthrough()</code>). A body that isn&#39;t valid JSON, or is JSON but fails that schema, falls back to the generic <code>access-or-upstream-failure</code> code: the same silent-skip treatment as a malformed log line, collapsing what could have been a separate &quot;malformed SHS response&quot; error code into the existing generic one.</li></ul><p>Neither boundary distinguishes &quot;malformed JSON&quot; from &quot;wrong shape&quot; from &quot;unrecognized variant&quot; in what it reports outward: all three collapse into the same skip/fallback path. That was a deliberate scope decision. The correction below says why the <em>line</em> boundary stops there rather than flagging every unrecognized event type.</p><blockquote><p>Correction made during migration, not part of the original design: an earlier draft counted <em>any</em> <code>Event</code> value outside the 15 modeled types toward <code>skippedLines</code>, not just ones that fail their own schema. Running that design against real Spark event logs (which always contain plenty of ordinary event types this tool has never modeled: <code>TaskStart</code>, <code>BlockManagerAdded</code>, <code>ExecutorMetricsUpdate</code>, and others, always silently ignored pre-migration) pushed <code>skippedLines</code> from 0 to the tens of thousands on completely healthy logs. That would have tripped <code>evidence-availability.ts</code>&#39;s fail-closed <code>trustworthy</code> gate and shown a false &quot;malformed JSON&quot; warning on every real file. Caught in review against real fixtures, not synthetic ones; fixed to the narrower rule described above before merging. <code>evidence-availability.ts</code>&#39;s gate itself (<code>trustworthy = skippedLines === 0 &amp;&amp; applicationEnds &gt; 0</code>) needed no code change once this was fixed: it was already correct, the input feeding it was not.</p></blockquote><p>The exhaustiveness convention is <code>packages/core/src/assert-never.ts</code>. <code>assertNever(x: never): never</code> throws at runtime and, more importantly, fails <code>tsc</code> at compile time if <code>x</code> is not actually <code>never</code>, i.e. if some case of a union type isn&#39;t handled. Used today at the <code>default</code> arm of <code>event-handlers.ts</code>&#39;s <code>processEvent</code> switch (over <code>SparkEvent</code>) and <code>analyzer.ts</code>&#39;s scope-dispatch switch (over a detector&#39;s <code>scope: &#39;stage&#39; | &#39;sql&#39; | &#39;app&#39; | &#39;config&#39;</code>). It is the project&#39;s standard pattern for any future exhaustive switch or dispatch over a closed union: add <code>default: return assertNever(x);</code> (or the closest non-returning equivalent) so that adding a new union member without updating every consumer becomes a compile error instead of a silent runtime gap.</p>`,61)])])}const m=o(c,[["render",s]]);export{u as __pageData,m as default};

package/public/docs/assets/{contributor-guide_architecture_worker-protocol.md.IbnfNrV3.lean.js → contributor-guide_architecture_worker-protocol.md.P7d1vtEJ.lean.js}

@@ -1 +1 @@
-import{_ as o,o as t,c as d,a5 as a}from"./chunks/framework.DSg0KOwT.js";const u=JSON.parse('{"title":"Worker protocol","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/architecture/worker-protocol.md","filePath":"contributor-guide/architecture/worker-protocol.md"}'),c={name:"contributor-guide/architecture/worker-protocol.md"};function s(r,e,i,n,l,p){return t(),d("div",null,[...e[0]||(e[0]=[a("",55)])])}const m=o(c,[["render",s]]);export{u as __pageData,m as default};
+import{_ as o,o as t,c as d,a5 as a}from"./chunks/framework.DSg0KOwT.js";const u=JSON.parse('{"title":"Worker protocol","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/architecture/worker-protocol.md","filePath":"contributor-guide/architecture/worker-protocol.md"}'),c={name:"contributor-guide/architecture/worker-protocol.md"};function s(r,e,i,n,l,p){return t(),d("div",null,[...e[0]||(e[0]=[a("",61)])])}const m=o(c,[["render",s]]);export{u as __pageData,m as default};

package/public/docs/assets/{contributor-guide_development-setup.md.DvAN_9mK.js → contributor-guide_development-setup.md.DvMV1xhQ.js}

@@ -1,4 +1,4 @@
-import{_ as i,o as a,c as e,a5 as t}from"./chunks/framework.DSg0KOwT.js";const c=JSON.parse('{"title":"Development setup","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/development-setup.md","filePath":"contributor-guide/development-setup.md"}'),n={name:"contributor-guide/development-setup.md"};function p(h,s,l,k,r,d){return a(),e("div",null,[...s[0]||(s[0]=[t(`<h1 id="development-setup" tabindex="-1">Development setup <a class="header-anchor" href="#development-setup" aria-label="Permalink to &quot;Development setup&quot;">​</a></h1><p>Clone the repo, then:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">git</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> submodule</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> update</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --init</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> dev/log-corpus</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">   # or: git clone --recurse-submodules</span></span>
+import{_ as i,o as e,c as t,a5 as a}from"./chunks/framework.DSg0KOwT.js";const k=JSON.parse('{"title":"Development setup","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/development-setup.md","filePath":"contributor-guide/development-setup.md"}'),n={name:"contributor-guide/development-setup.md"};function p(h,s,l,o,d,r){return e(),t("div",null,[...s[0]||(s[0]=[a(`<h1 id="development-setup" tabindex="-1">Development setup <a class="header-anchor" href="#development-setup" aria-label="Permalink to &quot;Development setup&quot;">​</a></h1><p>Clone the repo, then:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">git</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> submodule</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> update</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --init</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> dev/log-corpus</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">   # or: git clone --recurse-submodules</span></span>
 <span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npm</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> install</span></span>
 <span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npm</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> test</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">              # vitest run: the full unit/component suite</span></span>
 <span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npm</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> run</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> test:watch</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">    # vitest, watch mode</span></span>
@@ -7,6 +7,6 @@ import{_ as i,o as a,c as e,a5 as t}from"./chunks/framework.DSg0KOwT.js";const c
 <span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npm</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> run</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> preview</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">       # serves dist/ locally</span></span>
 <span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npx</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> tsc</span><span style="--shiki-light:#005CC5;--shiki-dark:#79B8FF;"> --noEmit</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">      # typecheck (strict TypeScript across src/)</span></span>
 <span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npm</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> run</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> lint</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">          # eslint over the repo</span></span>
-<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">node</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> packages/cli/bin/sparkforensics-analyze.mjs</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> &lt;</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">file</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">|</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">dir</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">&gt;   </span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># run the CLI analyzer against a log, outside the browser</span></span></code></pre></div><p><code>npm install</code> also runs the <code>prepare</code> script, which points git at <code>.githooks/</code>. After that, a pre-commit hook lints staged JS files on every commit. Fix what it reports, or skip it with <code>git commit --no-verify</code>.</p><p><code>dev/log-corpus</code> is a git submodule pointing at the private <code>spark-event-corpus-data</code> repo. If you don&#39;t have access to it, skip the <code>git submodule update</code> step above: <code>packages/server/test/shs-proxy-fixture.test.js</code> will report as skipped rather than failed, which is expected, not a bug.</p><p>The <code>server/</code> package (the optional local-server deploy mode) keeps its own dependencies and test suite. Run them separately:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npm</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> run</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> test:server</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">   # runs server/&#39;s own suite, not part of \`npm test\`</span></span></code></pre></div><p>Build the docs site before packing or releasing <code>server/</code>: run <code>npm run docs:build</code> from the repo root, or the packed artifact ships without <code>/docs/</code>.</p><p>This docs site has its own dev loop:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npm</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> run</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> docs:dev</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">       # VitePress dev server for docs-site/</span></span>
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">node</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> packages/cli/bin/sparkforensics-analyze.mjs</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;"> &lt;</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">file</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">|</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">dir</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">&gt;   </span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># run the CLI analyzer against a log, outside the browser</span></span></code></pre></div><p><code>npm install</code> also runs the <code>prepare</code> script, which points git at <code>.githooks/</code>. After that, a pre-commit hook lints staged JS files on every commit. Fix what it reports, or skip it with <code>git commit --no-verify</code>.</p><p>The tuning reference under <code>packages/core/src/docs-content/{chapters,tuning,diagrams}</code> isn&#39;t committed. It&#39;s generated from the <code>shuffle-works/spark-tuning-reference</code> commit pinned in <code>packages/core/src/docs-content/upstream.json</code>: the first <code>npm test</code>, <code>npm run docs:dev</code> or <code>npm run build</code> fetches it over HTTPS (about 2s, no credentials) and later runs reuse the gitignored copy while it matches the pin. <code>npm run docs:fetch</code> does the same step on its own. Never edit those folders: fix the content upstream, then <code>npm run docs:bump</code> to move the pin. Running the MCP or CLI bins from source (<code>node packages/*/bin/...</code>) doesn&#39;t fetch it, so run <code>npm run docs:fetch</code> first: without it the MCP&#39;s tuning-reference lookups come back empty.</p><p>Offline, a cached copy keeps working. If the pin has moved since, <code>npm test</code> and <code>docs:dev</code> warn and use the older copy, while <code>docs:build</code>, <code>npm pack</code> and anything with <code>CI</code> set fail. To build from a local upstream checkout instead (offline, or to preview upstream edits in this docs site before they merge), set <code>SPARK_TUNING_REFERENCE_DIR</code>:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">SPARK_TUNING_REFERENCE_DIR</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">=</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;">../spark-tuning-reference</span><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;"> npm</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> run</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> docs:dev</span></span></code></pre></div><p>The same checks apply to that checkout. <code>npm pack</code> and CI accept it only when it sits at the pinned commit with no uncommitted <code>content/</code> changes.</p><p><code>dev/log-corpus</code> is a git submodule pointing at the public <code>spark-event-corpus-data</code> repo. It&#39;s optional: if you skip the <code>git submodule update</code> step above, <code>packages/server/test/shs-proxy-fixture.test.js</code> will report as skipped rather than failed, which is expected, not a bug.</p><p>The <code>server/</code> package (the optional local-server deploy mode) keeps its own dependencies and test suite. Run them separately:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npm</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> run</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> test:server</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">   # runs server/&#39;s own suite, not part of \`npm test\`</span></span></code></pre></div><p>Packing or releasing <code>server/</code> builds the docs site for you: its <code>prepack</code> runs <code>scripts/copy-frontend.js</code>, which runs the root <code>npm run build</code> (<code>docs:build</code> included), so the packed artifact always ships <code>/docs/</code>.</p><p>This docs site has its own dev loop:</p><div class="language-bash vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">bash</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npm</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> run</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> docs:dev</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">       # VitePress dev server for docs-site/</span></span>
 <span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npm</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> run</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> docs:build</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">     # production build; also checks internal links/anchors</span></span>
-<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npm</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> run</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> docs:preview</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">   # serves the built docs site locally</span></span></code></pre></div>`,10)])])}const F=i(n,[["render",p]]);export{c as __pageData,F as default};
+<span class="line"><span style="--shiki-light:#6F42C1;--shiki-dark:#B392F0;">npm</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> run</span><span style="--shiki-light:#032F62;--shiki-dark:#9ECBFF;"> docs:preview</span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;">   # serves the built docs site locally</span></span></code></pre></div>`,14)])])}const g=i(n,[["render",p]]);export{k as __pageData,g as default};

package/public/docs/assets/contributor-guide_development-setup.md.DvMV1xhQ.lean.js

@@ -0,0 +1 @@
+import{_ as i,o as e,c as t,a5 as a}from"./chunks/framework.DSg0KOwT.js";const k=JSON.parse('{"title":"Development setup","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/development-setup.md","filePath":"contributor-guide/development-setup.md"}'),n={name:"contributor-guide/development-setup.md"};function p(h,s,l,o,d,r){return e(),t("div",null,[...s[0]||(s[0]=[a("",14)])])}const g=i(n,[["render",p]]);export{k as __pageData,g as default};

package/public/docs/assets/contributor-guide_testing.md.BU6vyA-9.js

@@ -0,0 +1 @@
+import{_ as t,o,c as s,a5 as d}from"./chunks/framework.DSg0KOwT.js";const u=JSON.parse('{"title":"Testing & verification","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/testing.md","filePath":"contributor-guide/testing.md"}'),a={name:"contributor-guide/testing.md"};function r(i,e,c,n,l,p){return o(),s("div",null,[...e[0]||(e[0]=[d('<h1 id="testing-verification" tabindex="-1">Testing &amp; verification <a class="header-anchor" href="#testing-verification" aria-label="Permalink to &quot;Testing &amp; verification&quot;">​</a></h1><p>Before opening a PR:</p><ol><li>Run the full suite: <code>npm test</code> (root) and, if you touched <code>server/</code>, <code>npm run test:server</code> too: it&#39;s a separate package with its own dependencies, not covered by the root suite.</li><li>Typecheck: <code>npx tsc --noEmit</code>.</li><li>Lint: <code>npm run lint</code>. This also runs automatically on every commit via <code>.githooks/pre-commit</code>, but run it yourself so you catch failures before committing.</li><li>For any UI change, verify it in a real browser (Playwright or headless Chrome). Unit and jsdom tests check code correctness; they don&#39;t prove a feature works end-to-end. Drive the app, exercise the golden path and the edge cases, and look for regressions elsewhere on the dashboard.</li><li>Report the passing test count when you&#39;re done, not &quot;tests pass.&quot;</li></ol><p>Sample Spark event logs for manual testing live outside this repo. Keep them in a sibling <code>../spark-log-examples/</code> directory: real <code>.zstd</code> event logs plus baseline/candidate pairs for run comparison.</p><h2 id="testing-layout" tabindex="-1">Testing layout <a class="header-anchor" href="#testing-layout" aria-label="Permalink to &quot;Testing layout&quot;">​</a></h2><p>Vitest throughout, two environments in one run. Core logic tests (<code>tests/*.test.js</code>, covering parser, analyzer, detectors, format-utils, etc.) run under Node. View tests (<code>tests/view/*.test.tsx</code>) declare <code>// @vitest-environment jsdom</code> and use React Testing Library (<code>render</code>/<code>screen</code>/<code>userEvent</code>). There are no hand-rolled DOM-mounting jsdom tests left: the old <code>tests/widgets/*.test.js</code> suite called <code>src/widgets/*.js</code>&#39;s <code>renderX(mount, ...)</code> functions directly, and every behavior it covered now has an RTL equivalent.</p><p><code>server/</code> is a separate npm package with its own Vitest config and <code>npm test</code>, not run by the root suite. One test there, <code>server/test/bin.test.js</code>, packs and installs the real published tarball and spawns the real <code>.bin</code> entry point: the only test that exercises the production build end-to-end.</p><p>Two contract tests guard invariants that a future edit could silently break. The &quot;detector contract&quot; suite in <code>tests/analyzer.test.js</code> asserts <code>stageSlowness</code> stays array-index-after <code>slowHost</code> in <code>DETECTORS</code> (see <a href="./architecture/detector-contract.html#detector-contract">Detector contract</a>). <code>tests/view/detector-registry.test.tsx</code> asserts <code>REGISTRY</code> completeness against every <code>DETECTORS</code> type, including the <code>underBroadcast</code>/<code>overBroadcast</code> correction: <code>broadcastSizing</code> itself is never an emitted <code>finding.type</code>, as the doc comment above <code>REGISTRY</code> in <code>src/view/detector-registry.tsx</code> explains.</p><h2 id="test-fixtures" tabindex="-1">Test fixtures <a class="header-anchor" href="#test-fixtures" aria-label="Permalink to &quot;Test fixtures&quot;">​</a></h2><p>Real Spark event logs from private workloads live under <code>examples/</code>, gitignored: they&#39;re large, and their file names and contents identify the jobs that produced them, so they never enter this public repo. Tests, docs and scripts refer to them only by the neutral labels below. A test that reads one skips when its file is missing, so the suite passes without any of them. To run those tests, copy or symlink your own logs into <code>examples/</code> under these names:</p><table tabindex="0"><thead><tr><th>Label</th><th>Log it stands for</th><th>Used by</th></tr></thead><tbody><tr><td><code>private-log-01.zstd</code></td><td>~12 MB zstd run with Plan Advisor findings</td><td><code>impact-estimator-real-log.test.js</code>, <code>plan-node-stage-mapping-real-log.test.js</code>, spot-checks in <a href="./architecture/impact-estimation.html">Impact estimation</a></td></tr><tr><td><code>private-log-02.zstd</code></td><td>~2 MB zstd run with a heavy shuffle-and-spill stage</td><td><code>plan-node-stage-mapping-real-log.test.js</code>, spot-checks in <a href="./architecture/impact-estimation.html">Impact estimation</a></td></tr><tr><td><code>private-log-03.zstd</code></td><td>~28 MB zstd run with about 1,200 stages</td><td>timing spot-check in <a href="./architecture/impact-estimation.html">Impact estimation</a></td></tr><tr><td><code>private-log-04.zstd</code></td><td>~9 MB zstd run whose stage 394 plan became <code>plan-graph-stage-394.json</code></td><td><code>scripts/extract-plan-fixture.mjs</code></td></tr><tr><td><code>private-log-05</code></td><td>~76 MB uncompressed NDJSON run, 62 stages</td><td><code>parser-worker.test.js</code>; also a fast end-to-end smoke test</td></tr><tr><td><code>private-log-06</code></td><td>~1.3 GB uncompressed NDJSON run</td><td><code>detectors-plan.test.js</code>; also a stress test for the Web Worker streaming path</td></tr><tr><td><code>private-log-07</code></td><td>~120 MB log from a Spark History Server download</td><td>none: check the <code>LZ4Block</code> codec decoder by hand; <code>tests/lz4-block.test.js</code> uses a small hardcoded byte array instead</td></tr></tbody></table><p><code>scripts/extract-plan-fixture.mjs</code> anonymizes every table, column, path and literal before it writes the committed fixture; keep it that way if you extract another one.</p><p>Of the finding types moved onto the Gold Standard row/expand contract in the widget-gold-standard plan, only <code>retryWaste</code> and <code>autoscalingChurn</code> fire on any of the 15 real logs in <code>../spark-log-examples/</code>. <code>failures</code>, <code>stageFailed</code>, and <code>jobFailureRate</code> fire on none of them; they were verified with unit tests only (<code>tests/view/failures.test.tsx</code>, <code>tests/view/job-failures.test.tsx</code>), not against a real log.</p><p><code>dev/log-corpus/</code> is a git submodule pointing at the public <code>spark-event-corpus-data</code> repo, pinned to a tag. It backs <code>packages/server/test/shs-proxy-fixture.test.js</code>, which loads a real <code>*-parquet-baseline.ndjson</code> fixture and asserts the <code>/shs-proxy</code> route streams it back unmodified. Populate it locally with <code>git submodule update --init dev/log-corpus</code>; without it, the test suite skips (see <a href="./development-setup.html">Development setup</a>).</p>',14)])])}const m=t(a,[["render",r]]);export{u as __pageData,m as default};

package/public/docs/assets/contributor-guide_testing.md.BU6vyA-9.lean.js

@@ -0,0 +1 @@
+import{_ as t,o,c as s,a5 as d}from"./chunks/framework.DSg0KOwT.js";const u=JSON.parse('{"title":"Testing & verification","description":"","frontmatter":{},"headers":[],"relativePath":"contributor-guide/testing.md","filePath":"contributor-guide/testing.md"}'),a={name:"contributor-guide/testing.md"};function r(i,e,c,n,l,p){return o(),s("div",null,[...e[0]||(e[0]=[d("",14)])])}const m=t(a,[["render",r]]);export{u as __pageData,m as default};

package/public/docs/assets/{tuning-reference_bottleneck-failures.md.4z5BXGJ2.js → tuning-reference_bottleneck-failures.md.CAsFf3RR.js}

@@ -1,5 +1,5 @@
 import{_ as a,a as t}from"./chunks/retry-escalation-ladder.dark.DHipdJgZ.js";import{_ as o,o as s,c as r,a5 as i}from"./chunks/framework.DSg0KOwT.js";const k=JSON.parse('{"title":"Task Failures","description":"","frontmatter":{"title":"Task Failures"},"headers":[],"relativePath":"tuning-reference/bottleneck-failures.md","filePath":"tuning-reference/bottleneck-failures.md"}'),n={name:"tuning-reference/bottleneck-failures.md"};function l(c,e,d,f,h,u){return s(),r("div",null,[...e[0]||(e[0]=[i(`<h1 id="bottleneck-failures" tabindex="-1">Task Failures <a class="header-anchor" href="#bottleneck-failures" aria-label="Permalink to &quot;Task Failures {#bottleneck-failures}&quot;">​</a></h1><p><span class="tag">FAIL</span></p><h2 id="what-it-is" tabindex="-1">What it is <a class="header-anchor" href="#what-it-is" aria-label="Permalink to &quot;What it is&quot;">​</a></h2><p>A task failure is any task end whose <code>Reason</code> field is something other than success. Spark&#39;s event log serializes this as the formatted class name of whichever <code>TaskEndReason</code> instance was assigned to that task attempt<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>, and the canonical set of reasons is <code>Success</code>, <code>FetchFailed</code>, <code>ExceptionFailure</code>, <code>TaskResultLost</code>, <code>TaskKilled</code>, <code>TaskCommitDenied</code>, <code>ExecutorLostFailure</code>, <code>Resubmitted</code>, and <code>UnknownReason</code><sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>. Each carries its own extra detail: a <code>FetchFailed</code> records a reduce ID and message; an <code>ExceptionFailure</code> carries the exception&#39;s class name, description, and stack trace, plus accumulator updates (falling back to reading them out of the task&#39;s metrics object for logs written by Spark 1.x); a <code>TaskKilled</code> records a kill reason; a <code>TaskCommitDenied</code> records the job ID, partition ID, and attempt number; an <code>ExecutorLostFailure</code> records whether the loss was caused by the application, the executor ID, and a loss reason; <code>UnknownReason</code> carries no extra fields<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup>.</p><h2 id="how-it-s-detected" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h2><table tabindex="0"><thead><tr><th>failed tasks (share)</th><th>Level</th></tr></thead><tbody><tr><td>&gt; 5%</td><td>Warning</td></tr><tr><td>&gt; 20%</td><td>Critical</td></tr></tbody></table><p>The task-metrics object attached to a <code>SparkListenerTaskEnd</code> event is optional in the schema, so it isn&#39;t guaranteed to be present for every failure. It tends to be populated for reasons where the attempt actually ran far enough to produce metrics (an <code>ExceptionFailure</code> or <code>TaskKilled</code>, for instance) but can be absent for a <code>Resubmitted</code> attempt that never completed on its executor at all<sup class="footnote-ref"><a href="#fn1" id="fnref1:3">[1:3]</a></sup>. Detection has to key off the <code>Reason</code> field itself rather than assume metrics will always be there to inspect.</p><h2 id="why-it-matters" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h2><p>Some reasons point at something worse than a one-off hiccup. An <code>ExecutorLostFailure</code> can be the knock-on effect of a container or pod killed for exceeding its memory grant: enabling off-heap memory without also raising <code>spark.executor.memoryOverhead</code> (or setting <code>spark.executor.pyspark.memory</code> explicitly) lets real usage exceed what YARN or Kubernetes allocated, and the executor is killed with a generic container-memory error rather than a Spark-level exception<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>. Losing an executor mid-shuffle can also cascade into failures elsewhere: if dynamic allocation removes an executor before a shuffle it wrote data for has completed, that shuffle output is gone, and downstream tasks reading it fail and force an unnecessary recompute<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>.</p><h2 id="how-to-fix-it" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h2><ul><li>Read the <code>Reason</code> field before treating all failures the same: an <code>ExceptionFailure</code> points at application code, and an <code>ExecutorLostFailure</code> usually points at memory sizing or infrastructure.</li><li>For memory-driven <code>ExecutorLostFailure</code>s, size <code>spark.executor.memoryOverhead</code> (which defaults to <code>max(384 MB, 10% of executor memory)</code>) to actually cover off-heap and PySpark memory, since neither is folded into that default budget automatically<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup>.</li><li>For failures caused by dynamic allocation reclaiming an executor mid-shuffle, enable the external <a href="./shuffle.html">shuffle service</a> or <code>spark.dynamicAllocation.shuffleTracking.enabled</code> so shuffle output can outlive the executor that wrote it, instead of being lost and recomputed<sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup>.</li></ul><p>Config-only levers for memory-driven <code>ExecutorLostFailure</code>s and mid-shuffle executor loss:</p><div class="language-properties vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">properties</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Cover off-heap + PySpark memory the default overhead budget does NOT include.</span></span>
-<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Default is max(384m, 10% of executor memory); 2g is an example, size to real off-heap/PySpark use.</span></span>
+<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Default is max(384m, 10% of executor memory); 2g is an example — size to real off-heap/PySpark use.</span></span>
 <span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.executor.memoryOverhead</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=2g</span></span>
 <span class="line"></span>
 <span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Keep shuffle output alive when dynamic allocation reclaims an executor mid-shuffle</span></span>

package/public/docs/assets/{tuning-reference_bottleneck-gc.md.DSxzZRK7.js → tuning-reference_bottleneck-gc.md.B4T3kJYV.js}

@@ -1,6 +1,6 @@
-import{_ as a,o as t,c as o,a5 as s}from"./chunks/framework.DSg0KOwT.js";const p=JSON.parse('{"title":"GC Pressure","description":"","frontmatter":{"title":"GC Pressure"},"headers":[],"relativePath":"tuning-reference/bottleneck-gc.md","filePath":"tuning-reference/bottleneck-gc.md"}'),r={name:"tuning-reference/bottleneck-gc.md"};function i(n,e,l,c,h,f){return t(),o("div",null,[...e[0]||(e[0]=[s(`<h1 id="bottleneck-gc" tabindex="-1">GC Pressure <a class="header-anchor" href="#bottleneck-gc" aria-label="Permalink to &quot;GC Pressure {#bottleneck-gc}&quot;">​</a></h1><p><span class="tag">GC</span></p><h2 id="what-it-is" tabindex="-1">What it is <a class="header-anchor" href="#what-it-is" aria-label="Permalink to &quot;What it is&quot;">​</a></h2><p>GC pressure describes how much of an executor&#39;s time goes to JVM garbage collection instead of running task code. Spark tracks this directly in its per-task metrics: <code>jvmGCTime</code> sits alongside <code>executorRunTime</code>, the elapsed wall-clock time the executor spent running the task (as opposed to <code>executorCpuTime</code>, which measures CPU time specifically).<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup> <code>jvmGCTime</code> is not additive on top of that duration: it&#39;s defined as the elapsed time the JVM spent in garbage collection <em>while executing the task</em>, so GC pauses fall inside the same wall-clock window <code>executorRunTime</code> already measures, not outside it. Summing the two would double-count the GC pauses and overstate how long the task actually ran.<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup></p><h2 id="how-it-s-detected" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h2><p>Because <code>jvmGCTime</code> sits inside <code>executorRunTime</code> rather than alongside it, the ratio between the two gives a bounded read on how much of a task&#39;s wall-clock time went to garbage collection:</p><table tabindex="0"><thead><tr><th>gcPct = jvmGCTime / executorRunTime</th><th>Level</th></tr></thead><tbody><tr><td>&gt; 10%</td><td>Warning</td></tr><tr><td>&gt; 20%</td><td>Critical</td></tr></tbody></table><h2 id="why-it-matters" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h2><p>As gcPct climbs past these thresholds, a growing share of every task&#39;s wall-clock time is consumed by garbage collection instead of actual computation, so the executor&#39;s useful throughput drops even while it appears busy. Two causes show up repeatedly: oversized executors, and an on-heap memory split pushed too far toward execution/storage.</p><p>Sizing an executor with too many cores relative to its heap is one documented trigger: a fat-executor configuration using all 16 cores of a node was observed to not only hurt HDFS throughput but also &quot;result in excessive garbage [collection].&quot;<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup></p><p>Pushing <code>spark.memory.fraction</code> too far toward its upper end has a similar effect through a different path. Its complement, <code>1 − spark.memory.fraction</code>, is untracked &quot;User Memory&quot; reserved for UDFs, Python/Arrow glue, and native library buffers; Spark doesn&#39;t manage this region at all, and starving it (which is what raising <code>spark.memory.fraction</code> toward something like 0.9 does) leads to &quot;GC pressure or random OOMs,&quot; with no warning before the failure.<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup></p><h2 id="how-to-fix-it" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h2><ul><li>Keep cores per executor down rather than packing an entire node&#39;s cores onto one JVM (the common ~5-cores-per-executor guideline). That avoids the excessive-garbage-collection failure mode documented for fat-executor configurations.<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup></li><li>Don&#39;t raise <code>spark.memory.fraction</code> past a comfortable range chasing more execution/storage memory: doing so starves the untracked user-memory region and risks the same GC-pressure/OOM failure mode.<sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup></li><li>Move Tungsten&#39;s execution and storage buffers off the JVM heap with <code>spark.memory.offHeap.enabled=true</code> (and a positive <code>spark.memory.offHeap.size</code>, which is required whenever off-heap is enabled<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup>). Off-heap buffers are invisible to the garbage collector, so fewer and smaller live objects need to be tracked, scanned, and copied on the heap, reducing both the frequency and duration of GC pauses.<sup class="footnote-ref"><a href="#fn3" id="fnref3:2">[3:2]</a></sup></li><li>Tune or switch the GC collector on large heaps. G1GC is Spark&#39;s default collector since 4.0 (which defaults to JDK 17), and large executor heaps may need <code>-XX:G1HeapRegionSize</code> raised as well.<sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup> G1&#39;s stop-the-world evacuation and full-GC pauses can still spike under heavy old-generation pressure on large heaps: one documented 88GB-heap benchmark saw a job&#39;s pause spike to nearly 100 seconds under default G1 settings, and only tuning <code>InitiatingHeapOccupancyPercent</code>, <code>ConcGCThreads</code>, and RSet-update parameters brought G1 back ahead of Parallel/CMS on both throughput and latency.<sup class="footnote-ref"><a href="#fn6" id="fnref6">[6]</a></sup> ZGC is a documented alternative built for pause-time control: it performs its expensive work concurrently, without stopping application threads for more than a millisecond, with pause times designed to stay independent of heap size from a few hundred megabytes up to 16TB.<sup class="footnote-ref"><a href="#fn7" id="fnref7">[7]</a></sup></li></ul><p>Move buffers off-heap and give G1 room on large heaps (sizes are examples; set to your workload):</p><div class="language-properties vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">properties</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Move Tungsten execution/storage buffers off the JVM heap so the GC has fewer objects to scan</span></span>
+import{_ as a,o as t,c as o,a5 as s}from"./chunks/framework.DSg0KOwT.js";const p=JSON.parse('{"title":"GC Pressure","description":"","frontmatter":{"title":"GC Pressure"},"headers":[],"relativePath":"tuning-reference/bottleneck-gc.md","filePath":"tuning-reference/bottleneck-gc.md"}'),r={name:"tuning-reference/bottleneck-gc.md"};function i(n,e,l,c,h,d){return t(),o("div",null,[...e[0]||(e[0]=[s(`<h1 id="bottleneck-gc" tabindex="-1">GC Pressure <a class="header-anchor" href="#bottleneck-gc" aria-label="Permalink to &quot;GC Pressure {#bottleneck-gc}&quot;">​</a></h1><p><span class="tag">GC</span></p><h2 id="what-it-is" tabindex="-1">What it is <a class="header-anchor" href="#what-it-is" aria-label="Permalink to &quot;What it is&quot;">​</a></h2><p>GC pressure describes how much of an executor&#39;s time goes to JVM garbage collection instead of running task code. Spark tracks this directly in its per-task metrics: <code>jvmGCTime</code> sits alongside <code>executorRunTime</code>, the elapsed wall-clock time the executor spent running the task (as opposed to <code>executorCpuTime</code>, which measures CPU time specifically).<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup> <code>jvmGCTime</code> is not additive on top of that duration: it&#39;s defined as the elapsed time the JVM spent in garbage collection <em>while executing the task</em>, so GC pauses fall inside the same wall-clock window <code>executorRunTime</code> already measures, not outside it. Summing the two would double-count the GC pauses and overstate how long the task actually ran.<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup></p><h2 id="how-it-s-detected" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h2><p>Because <code>jvmGCTime</code> sits inside <code>executorRunTime</code> rather than alongside it, the ratio between the two gives a bounded read on how much of a task&#39;s wall-clock time went to garbage collection. Both directions below apply once a stage&#39;s <code>executorRunTime</code> reaches 10 seconds, a floor that keeps short stages from reading as noise:</p><table tabindex="0"><thead><tr><th>Signal (gcPct = jvmGCTime / executorRunTime)</th><th>Fires when</th></tr></thead><tbody><tr><td>High GC</td><td>&gt; 10%</td></tr><tr><td>Low GC (cost signal)</td><td>&lt; 5%</td></tr></tbody></table><p>A stage lands in at most one of these two opposite-direction bands at a time. Severity for both tracks the estimated recoverable time as a share of the app&#39;s total runtime: ≥2% is critical, ≥0.5% is warning, anything smaller is info. Low GC signals a possible memory over-provisioning cost rather than time lost to garbage collection.</p><h2 id="why-it-matters" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h2><p>As gcPct climbs past these thresholds, a growing share of every task&#39;s wall-clock time is consumed by garbage collection instead of actual computation, so the executor&#39;s useful throughput drops even while it appears busy. Two causes show up repeatedly: oversized executors, and an on-heap memory split pushed too far toward execution/storage.</p><p>Sizing an executor with too many cores relative to its heap is one documented trigger: a fat-executor configuration using all 16 cores of a node was observed to not only hurt HDFS throughput but also &quot;result in excessive garbage [collection].&quot;<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup></p><p>Pushing <code>spark.memory.fraction</code> too far toward its upper end has a similar effect through a different path. Its complement, <code>1 − spark.memory.fraction</code>, is untracked &quot;User Memory&quot; reserved for UDFs, Python/Arrow glue, and native library buffers; Spark doesn&#39;t manage this region at all, and starving it (which is what raising <code>spark.memory.fraction</code> toward something like 0.9 does) leads to &quot;GC pressure or random OOMs,&quot; with no warning before the failure.<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup></p><h2 id="how-to-fix-it" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h2><ul><li>Keep cores per executor down rather than packing an entire node&#39;s cores onto one JVM (the common ~5-cores-per-executor guideline). That avoids the excessive-garbage-collection failure mode documented for fat-executor configurations.<sup class="footnote-ref"><a href="#fn2" id="fnref2:1">[2:1]</a></sup></li><li>Don&#39;t raise <code>spark.memory.fraction</code> past a comfortable range chasing more execution/storage memory: doing so starves the untracked user-memory region and risks the same GC-pressure/OOM failure mode.<sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup></li><li>Move Tungsten&#39;s execution and storage buffers off the JVM heap with <code>spark.memory.offHeap.enabled=true</code> (and a positive <code>spark.memory.offHeap.size</code>, which is required whenever off-heap is enabled<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup>). Off-heap buffers are invisible to the garbage collector, so fewer and smaller live objects need to be tracked, scanned, and copied on the heap, reducing both the frequency and duration of GC pauses.<sup class="footnote-ref"><a href="#fn3" id="fnref3:2">[3:2]</a></sup></li><li>Tune or switch the GC collector on large heaps. G1GC is Spark&#39;s default collector since 4.0 (which defaults to JDK 17), and large executor heaps may need <code>-XX:G1HeapRegionSize</code> raised as well.<sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup> G1&#39;s stop-the-world evacuation and full-GC pauses can still spike under heavy old-generation pressure on large heaps: one documented 88GB-heap benchmark saw a job&#39;s pause spike to nearly 100 seconds under default G1 settings, and only tuning <code>InitiatingHeapOccupancyPercent</code>, <code>ConcGCThreads</code>, and RSet-update parameters brought G1 back ahead of Parallel/CMS on both throughput and latency.<sup class="footnote-ref"><a href="#fn6" id="fnref6">[6]</a></sup> ZGC is a documented alternative built for pause-time control: it performs its expensive work concurrently, without stopping application threads for more than a millisecond, with pause times designed to stay independent of heap size from a few hundred megabytes up to 16TB.<sup class="footnote-ref"><a href="#fn7" id="fnref7">[7]</a></sup></li></ul><p>Move buffers off-heap and give G1 room on large heaps (sizes are examples; set to your workload):</p><div class="language-properties vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">properties</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Move Tungsten execution/storage buffers off the JVM heap so the GC has fewer objects to scan</span></span>
 <span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.memory.offHeap.enabled</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=true</span></span>
 <span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.memory.offHeap.size</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=2g            </span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># must be &gt; 0 whenever off-heap is enabled; 2g is an example</span></span>
 <span class="line"></span>
 <span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># G1 is the default collector since Spark 4.0 (JDK 17); raise the region size on large heaps</span></span>
-<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.executor.extraJavaOptions</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=-XX:+UseG1GC -XX:</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">G1HeapRegionSize</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=16m   </span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># 16m is an example</span></span></code></pre></div><h2 id="confidence" tabindex="-1">Confidence <a class="header-anchor" href="#confidence" aria-label="Permalink to &quot;Confidence&quot;">​</a></h2><p>The warning and critical thresholds on gcPct are validated: they read a bounded ratio of <code>jvmGCTime</code> to <code>executorRunTime</code>, both first-class per-task metrics Spark records directly,<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup> and crossing 10% or 20% marks a real, measurable share of wall-clock time lost to garbage collection.</p><p><span class="tag">EXPERIMENTAL</span> A separate signal treats suspiciously <em>low</em> GC time as a possible cost-model or noise-floor indicator, and that one is an unsourced heuristic with no external basis, so treat it as exploratory rather than a confirmed diagnostic.</p><h2 id="limitations-false-positive-risk" tabindex="-1">Limitations / false-positive risk <a class="header-anchor" href="#limitations-false-positive-risk" aria-label="Permalink to &quot;Limitations / false-positive risk&quot;">​</a></h2><p>A high gcPct can reflect a transient memory spike (a single skewed task, a brief burst of large allocations) rather than chronic pressure, so a threshold crossing on one stage does not by itself prove a sustained sizing problem; corroborate against the executor&#39;s behavior over the whole run. The low-GC signal has no external basis at all, and a near-zero reading is at least as likely to mean the workload simply is not allocation-heavy as to mean anything is wrong.</p><h2 id="related" tabindex="-1">Related <a class="header-anchor" href="#related" aria-label="Permalink to &quot;Related&quot;">​</a></h2><ul><li><strong>Why it happens:</strong> <a href="./memory-model.html">Memory Management</a></li><li><strong>Executor sizing:</strong> <a href="./cluster-config.html">Cluster Tuning</a></li></ul><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/monitoring.html#spark-history-server" target="_blank" rel="noreferrer">Monitoring and Instrumentation</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://raw.githubusercontent.com/spoddutur/spark-notes/master/distribution_of_executors_cores_and_memory_for_spark_application.md" target="_blank" rel="noreferrer">Distribution of Executors, Cores, and Memory for a Spark Application</a> <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://luminousmen.com/post/dive-into-spark-memory" target="_blank" rel="noreferrer">Diving into Spark Memory Management</a> <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a> <a href="#fnref3:2" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration (Spark)</a> <a href="#fnref4" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/tuning.html" target="_blank" rel="noreferrer">Spark Tuning Guide</a> <a href="#fnref5" class="footnote-backref">↩︎</a></p></li><li id="fn6" class="footnote-item"><p><a href="https://www.databricks.com/blog/2015/05/28/tuning-java-garbage-collection-for-spark-applications.html" target="_blank" rel="noreferrer">Tuning Java Garbage Collection for Apache Spark Applications</a> <a href="#fnref6" class="footnote-backref">↩︎</a></p></li><li id="fn7" class="footnote-item"><p><a href="https://wiki.openjdk.org/display/zgc" target="_blank" rel="noreferrer">The Z Garbage Collector (ZGC)</a> <a href="#fnref7" class="footnote-backref">↩︎</a></p></li></ol></section>`,24)])])}const u=a(r,[["render",i]]);export{p as __pageData,u as default};
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.executor.extraJavaOptions</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=-XX:+UseG1GC -XX:</span><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">G1HeapRegionSize</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=16m   </span><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># 16m is an example</span></span></code></pre></div><h2 id="confidence" tabindex="-1">Confidence <a class="header-anchor" href="#confidence" aria-label="Permalink to &quot;Confidence&quot;">​</a></h2><p>The warning and critical thresholds on gcPct are validated: they read a bounded ratio of <code>jvmGCTime</code> to <code>executorRunTime</code>, both first-class per-task metrics Spark records directly,<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup> and crossing 10% or 20% marks a real, measurable share of wall-clock time lost to garbage collection.</p><p><span class="tag">EXPERIMENTAL</span> A separate signal treats suspiciously <em>low</em> GC time as a possible cost-model or noise-floor indicator, and that one is an unsourced heuristic with no external basis, so treat it as exploratory rather than a confirmed diagnostic.</p><h2 id="limitations-false-positive-risk" tabindex="-1">Limitations / false-positive risk <a class="header-anchor" href="#limitations-false-positive-risk" aria-label="Permalink to &quot;Limitations / false-positive risk&quot;">​</a></h2><p>A high gcPct can reflect a transient memory spike (a single skewed task, a brief burst of large allocations) rather than chronic pressure, so a threshold crossing on one stage does not by itself prove a sustained sizing problem; corroborate against the executor&#39;s behavior over the whole run. The low-GC signal has no external basis at all, and a near-zero reading is at least as likely to mean the workload simply is not allocation-heavy as to mean anything is wrong.</p><h2 id="related" tabindex="-1">Related <a class="header-anchor" href="#related" aria-label="Permalink to &quot;Related&quot;">​</a></h2><ul><li><strong>Why it happens:</strong> <a href="./memory-model.html">Memory Management</a></li><li><strong>Executor sizing:</strong> <a href="./cluster-config.html">Cluster Tuning</a></li></ul><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/monitoring.html#spark-history-server" target="_blank" rel="noreferrer">Monitoring and Instrumentation</a> <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://raw.githubusercontent.com/spoddutur/spark-notes/master/distribution_of_executors_cores_and_memory_for_spark_application.md" target="_blank" rel="noreferrer">Distribution of Executors, Cores, and Memory for a Spark Application</a> <a href="#fnref2" class="footnote-backref">↩︎</a> <a href="#fnref2:1" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://luminousmen.com/post/dive-into-spark-memory" target="_blank" rel="noreferrer">Diving into Spark Memory Management</a> <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a> <a href="#fnref3:2" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration (Spark)</a> <a href="#fnref4" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/tuning.html" target="_blank" rel="noreferrer">Spark Tuning Guide</a> <a href="#fnref5" class="footnote-backref">↩︎</a></p></li><li id="fn6" class="footnote-item"><p><a href="https://www.databricks.com/blog/2015/05/28/tuning-java-garbage-collection-for-spark-applications.html" target="_blank" rel="noreferrer">Tuning Java Garbage Collection for Apache Spark Applications</a> <a href="#fnref6" class="footnote-backref">↩︎</a></p></li><li id="fn7" class="footnote-item"><p><a href="https://wiki.openjdk.org/display/zgc" target="_blank" rel="noreferrer">The Z Garbage Collector (ZGC)</a> <a href="#fnref7" class="footnote-backref">↩︎</a></p></li></ol></section>`,25)])])}const u=a(r,[["render",i]]);export{p as __pageData,u as default};

package/public/docs/assets/{tuning-reference_bottleneck-gc.md.DSxzZRK7.lean.js → tuning-reference_bottleneck-gc.md.B4T3kJYV.lean.js}

@@ -1 +1 @@
-import{_ as a,o as t,c as o,a5 as s}from"./chunks/framework.DSg0KOwT.js";const p=JSON.parse('{"title":"GC Pressure","description":"","frontmatter":{"title":"GC Pressure"},"headers":[],"relativePath":"tuning-reference/bottleneck-gc.md","filePath":"tuning-reference/bottleneck-gc.md"}'),r={name:"tuning-reference/bottleneck-gc.md"};function i(n,e,l,c,h,f){return t(),o("div",null,[...e[0]||(e[0]=[s("",24)])])}const u=a(r,[["render",i]]);export{p as __pageData,u as default};
+import{_ as a,o as t,c as o,a5 as s}from"./chunks/framework.DSg0KOwT.js";const p=JSON.parse('{"title":"GC Pressure","description":"","frontmatter":{"title":"GC Pressure"},"headers":[],"relativePath":"tuning-reference/bottleneck-gc.md","filePath":"tuning-reference/bottleneck-gc.md"}'),r={name:"tuning-reference/bottleneck-gc.md"};function i(n,e,l,c,h,d){return t(),o("div",null,[...e[0]||(e[0]=[s("",25)])])}const u=a(r,[["render",i]]);export{p as __pageData,u as default};

package/public/docs/assets/tuning-reference_bottleneck-shuffle.md.DDMQVOW6.js

@@ -0,0 +1,12 @@
+import{_ as a,o as t,c as s,a5 as o}from"./chunks/framework.DSg0KOwT.js";const p=JSON.parse('{"title":"Shuffle I/O","description":"","frontmatter":{"title":"Shuffle I/O"},"headers":[],"relativePath":"tuning-reference/bottleneck-shuffle.md","filePath":"tuning-reference/bottleneck-shuffle.md"}'),r={name:"tuning-reference/bottleneck-shuffle.md"};function i(n,e,f,l,c,d){return t(),s("div",null,[...e[0]||(e[0]=[o(`<h1 id="bottleneck-shuffle" tabindex="-1">Shuffle I/O <a class="header-anchor" href="#bottleneck-shuffle" aria-label="Permalink to &quot;Shuffle I/O {#bottleneck-shuffle}&quot;">​</a></h1><p><span class="tag">SHFL</span></p><h2 id="what-it-is" tabindex="-1">What it is <a class="header-anchor" href="#what-it-is" aria-label="Permalink to &quot;What it is&quot;">​</a></h2><p>Shuffle I/O is the data movement Spark performs between stages whenever an operation needs to redistribute data across the cluster so that related rows land on the same partition. At the DataFrame/SQL level, <code>groupBy()</code>, <code>join()</code>, <code>agg()</code>, <code>sortBy()</code>, and <code>reduceByKey()</code>-style aggregations are the classic wide transformations that force this exchange<sup class="footnote-ref"><a href="#fn1" id="fnref1">[1]</a></sup>. Non-broadcast join strategies (shuffle hash join, shuffle sort-merge join, and shuffle-and-replicated nested loop / Cartesian product join) all shuffle data across executors the same way<sup class="footnote-ref"><a href="#fn1" id="fnref1:1">[1:1]</a></sup>. At the RDD layer, the operations that can cause a shuffle are repartitioning (<code>repartition</code>, <code>coalesce</code>), <code>&#39;ByKey</code> operations other than counting (<code>groupByKey</code>, <code>reduceByKey</code>), and join-family operations (<code>cogroup</code>, <code>join</code>)<sup class="footnote-ref"><a href="#fn2" id="fnref2">[2]</a></sup>.</p><p>Spark can skip the shuffle when it already knows the data layout: Storage Partition Join avoids it entirely when Spark can use partitioning already reported by a compatible V2 data source<sup class="footnote-ref"><a href="#fn3" id="fnref3">[3]</a></sup>, and classic Hive-style bucketing has the same effect: once both sides of a join are bucketed and sorted the same way, the physical plan shows no <code>Exchange</code> operator<sup class="footnote-ref"><a href="#fn4" id="fnref4">[4]</a></sup>. <code>DataFrameWriter.partitionBy</code>, by contrast, does not trigger a shuffle by itself on write<sup class="footnote-ref"><a href="#fn5" id="fnref5">[5]</a></sup>.</p><p>At the metrics level, shuffle reads split cross-node traffic from same-host traffic: <code>remoteBytesRead</code> counts bytes read from a remote executor, <code>localBytesRead</code> counts bytes read from local disk, and <code>totalBytesRead</code> is their sum<sup class="footnote-ref"><a href="#fn6" id="fnref6">[6]</a></sup>.</p><h2 id="how-it-s-detected" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h2><table tabindex="0"><thead><tr><th>Signal</th><th>Fires when</th></tr></thead><tbody><tr><td>Shuffle read bytes in a stage</td><td>&gt; 50 MB</td></tr></tbody></table><p>50 MB marks a stage as shuffle-heavy enough to flag. Severity then tracks the estimated recoverable time as a share of the app&#39;s total runtime: ≥2% is critical, ≥0.5% is warning, anything smaller is info.</p><p>Beyond raw byte volume, the executor-side wait is captured by <code>fetchWaitTime</code>: time a task spends blocked on a remote shuffle block it needs next, not counting time spent prefetching other blocks in the background<sup class="footnote-ref"><a href="#fn6" id="fnref6:1">[6:1]</a></sup>. That wait time counts toward the task&#39;s overall <code>executorRunTime</code>, since Spark&#39;s wall-clock task-time metric explicitly includes time fetching shuffle data<sup class="footnote-ref"><a href="#fn6" id="fnref6:2">[6:2]</a></sup>.</p><h2 id="why-it-matters" tabindex="-1">Why it matters <a class="header-anchor" href="#why-it-matters" aria-label="Permalink to &quot;Why it matters&quot;">​</a></h2><p>As shuffle volume grows, the underlying pull-based mechanism gets less efficient: the number of shuffle blocks grows quadratically with mapper × reducer count while individual block sizes shrink to only tens of KB, which is inefficient for disk-backed random reads<sup class="footnote-ref"><a href="#fn7" id="fnref7">[7]</a></sup>. Because fetch-wait time is part of a task&#39;s measured execution window, that inefficiency shows up directly as added task time rather than as separate, hidden overhead<sup class="footnote-ref"><a href="#fn6" id="fnref6:3">[6:3]</a></sup>.</p><h2 id="how-to-fix-it" tabindex="-1">How to fix it <a class="header-anchor" href="#how-to-fix-it" aria-label="Permalink to &quot;How to fix it&quot;">​</a></h2><ul><li>Let <a href="./aqe.html">Adaptive Query Execution</a> coalesce small post-shuffle partitions automatically at runtime (<code>spark.sql.adaptive.coalescePartitions.enabled</code>, default <code>true</code>) instead of only hand-tuning a fixed partition count<sup class="footnote-ref"><a href="#fn3" id="fnref3:1">[3:1]</a></sup>.</li><li>Review <code>spark.sql.shuffle.partitions</code> (default <code>200</code>), which applies uniformly to every <code>join()</code>, <code>groupBy()</code>, and aggregation regardless of how much data is actually moving<sup class="footnote-ref"><a href="#fn8" id="fnref8">[8]</a></sup>. There&#39;s no fixed formula for the right value: pull it down toward the executor core count for small or streaming workloads<sup class="footnote-ref"><a href="#fn1" id="fnref1:2">[1:2]</a></sup>, and otherwise favor over- to under-provisioning, since Spark&#39;s low per-task overhead makes it safer to have too many tasks than too few<sup class="footnote-ref"><a href="#fn9" id="fnref9">[9]</a></sup>.</li><li>Where possible, avoid the shuffle altogether: bucket both sides of a join identically, or rely on Storage Partition Join for compatible sources, so the physical plan drops the <code>Exchange</code> node<sup class="footnote-ref"><a href="#fn3" id="fnref3:2">[3:2]</a></sup><sup class="footnote-ref"><a href="#fn4" id="fnref4:1">[4:1]</a></sup>.</li><li>If map tasks are I/O-bound writing many shuffle files, increase <code>spark.shuffle.file.buffer</code> (default <code>32k</code>) to reduce disk seeks and system calls on the write side<sup class="footnote-ref"><a href="#fn8" id="fnref8:1">[8:1]</a></sup>; Learning Spark&#39;s tuning table recommends bumping it to 1 MB for large jobs<sup class="footnote-ref"><a href="#fn1" id="fnref1:3">[1:3]</a></sup>.</li><li>If executors have memory to spare, increase <code>spark.reducer.maxSizeInFlight</code> (default <code>48m</code>) so reducers can pull more map output concurrently and need fewer fetch rounds<sup class="footnote-ref"><a href="#fn8" id="fnref8:2">[8:2]</a></sup>.</li><li>At larger scale, enable the external shuffle service (effectively required for <a href="./cluster-config.html">dynamic allocation</a>, since it lets shuffle files be served after an executor is removed<sup class="footnote-ref"><a href="#fn10" id="fnref10">[10]</a></sup>) and consider push-based (Magnet) shuffle, which converts many small random reads into large sequential reads of pre-merged chunks<sup class="footnote-ref"><a href="#fn7" id="fnref7:1">[7:1]</a></sup>.</li></ul><blockquote><p><strong>PySpark:</strong> adjust the shuffle partition count directly from a running session with <code>spark.conf.set(&quot;spark.sql.shuffle.partitions&quot;, 100)</code>.</p></blockquote><p>A shuffle-tuning starting point: defaults shown, comments say which way to move:</p><div class="language-properties vp-adaptive-theme"><button title="Copy Code" class="copy"></button><span class="lang">properties</span><pre class="shiki shiki-themes github-light github-dark vp-code" tabindex="0"><code><span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Let AQE coalesce small post-shuffle partitions at runtime</span></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.sql.adaptive.enabled</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=true</span></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.sql.adaptive.coalescePartitions.enabled</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=true</span></span>
+<span class="line"></span>
+<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Baseline shuffle partition count (default 200); pull toward executor-core count for small jobs</span></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.sql.shuffle.partitions</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=200</span></span>
+<span class="line"></span>
+<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Reduce write-side disk seeks on large shuffles (default 32k; Learning Spark suggests 1m)</span></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.shuffle.file.buffer</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=1m</span></span>
+<span class="line"></span>
+<span class="line"><span style="--shiki-light:#6A737D;--shiki-dark:#6A737D;"># Let reducers pull more map output per fetch round when executors have spare memory (default 48m)</span></span>
+<span class="line"><span style="--shiki-light:#D73A49;--shiki-dark:#F97583;">spark.reducer.maxSizeInFlight</span><span style="--shiki-light:#24292E;--shiki-dark:#E1E4E8;">=48m</span></span></code></pre></div><h2 id="bottleneck-partition-sizing" tabindex="-1">Partition sizing <a class="header-anchor" href="#bottleneck-partition-sizing" aria-label="Permalink to &quot;Partition sizing {#bottleneck-partition-sizing}&quot;">​</a></h2><p><span class="tag">PART</span></p><h3 id="how-it-s-detected-1" tabindex="-1">How it&#39;s detected <a class="header-anchor" href="#how-it-s-detected-1" aria-label="Permalink to &quot;How it&#39;s detected&quot;">​</a></h3><p>A stage&#39;s shuffle-read partition sizes surface three distinct problems:</p><table tabindex="0"><thead><tr><th>Signal</th><th>Fires when</th><th>Level</th></tr></thead><tbody><tr><td>Largest partition vs. median</td><td>&gt; 5× the median <strong>and</strong> &gt; 256 MB</td><td>Warning</td></tr><tr><td>Low parallelism</td><td>≥ 1 GB of shuffle read spread across ≤ 7 tasks</td><td>Warning</td></tr><tr><td>Oversized partition</td><td>Largest partition ≥ 5 GB</td><td>Critical</td></tr></tbody></table><p>Skew and low-parallelism severity track the estimated recoverable time as a share of the app&#39;s total runtime, the same wall-clock model used across this reference. An oversized partition is a fixed safety signal instead: it reports critical purely on its own size, because a partition past 5 GB is an OOM/crash risk regardless of how much wall-clock time fixing it would recover, so it stays critical even on a stage that barely dents the run.</p><p>Adaptive Query Execution re-optimizes the plan while the query runs: as each shuffle stage materializes, it reads the real shuffle-file sizes and resizes partitions before launching the stages downstream<sup class="footnote-ref"><a href="#fn11" id="fnref11">[11]</a></sup>. That runtime feedback is what corrects a partition grid that static estimates would get wrong.</p><p>The coarse starting grid is <code>spark.sql.shuffle.partitions</code> (default <code>200</code>), the fixed partition count Spark uses when shuffling for joins or aggregations<sup class="footnote-ref"><a href="#fn12" id="fnref12">[12]</a></sup>. A single number cannot fit every stage. Spread a few megabytes across 200 partitions and most cores sit idle on a handful of rows each; push hundreds of gigabytes through the same 200 and every executor is overloaded, often into memory errors. For small or streaming workloads the 200 default is usually too high, and pulling it toward the executor-core count thins out the flood of tiny partitions crossing the network<sup class="footnote-ref"><a href="#fn1" id="fnref1:4">[1:4]</a></sup>. The pattern AQE encourages is the opposite of hand-tuning that number: leave the initial count large and let runtime coalescing combine adjacent small partitions instead<sup class="footnote-ref"><a href="#fn11" id="fnref11:1">[11:1]</a></sup>.</p><p><strong>Partitions too small (low parallelism payoff).</strong> <code>spark.sql.adaptive.coalescePartitions.enabled</code> (default <code>true</code>) merges contiguous shuffle partitions up to a target size so a stage does not end up with many tiny tasks<sup class="footnote-ref"><a href="#fn13" id="fnref13">[13]</a></sup>. The target is <code>spark.sql.adaptive.advisoryPartitionSizeInBytes</code>, the advisory shuffle-partition size AQE steers toward during adaptive optimization, default <code>64 MB</code><sup class="footnote-ref"><a href="#fn13" id="fnref13:1">[13:1]</a></sup>.</p><p><strong>Partitions too big or skewed.</strong> <code>spark.sql.adaptive.skewJoin.enabled</code> (default <code>true</code>) handles skew in shuffled joins by splitting the oversized partitions and replicating the matching side where needed<sup class="footnote-ref"><a href="#fn14" id="fnref14">[14]</a></sup>. A partition only counts as skewed when it clears both bars: larger than <code>spark.sql.adaptive.skewJoin.skewedPartitionFactor</code> (default <code>5.0</code>) times the median partition size, and larger than <code>spark.sql.adaptive.skewJoin.skewedPartitionThresholdInBytes</code> (default <code>256 MB</code>)<sup class="footnote-ref"><a href="#fn14" id="fnref14:1">[14:1]</a></sup>. Once it qualifies, the partition is split back down toward the advisory size.</p><p>So the two knobs divide the labor: <code>spark.sql.shuffle.partitions</code> lays down the coarse initial grid, and <code>advisoryPartitionSizeInBytes</code> (64 MB) is the size AQE aims each partition at from both directions. Partitions below it get coalesced; skewed partitions past the 256 MB / 5.0-factor bar get split toward it.</p><h2 id="limitations-false-positive-risk" tabindex="-1">Limitations / false-positive risk <a class="header-anchor" href="#limitations-false-positive-risk" aria-label="Permalink to &quot;Limitations / false-positive risk&quot;">​</a></h2><p>A high shuffle-byte count is not automatically a defect. A wide transformation such as a large <code>join()</code> or <code>groupBy()</code> legitimately has to move that data, so the volume can be an inherent property of the query rather than something worth fixing. The byte thresholds that raise Info, Warning, and Critical levels are heuristic cutoffs, not measured limits for a given cluster, so treat them as a prompt to look rather than a verdict.</p><h2 id="related" tabindex="-1">Related <a class="header-anchor" href="#related" aria-label="Permalink to &quot;Related&quot;">​</a></h2><ul><li><strong>The mechanism:</strong> <a href="./shuffle.html">Shuffle</a></li><li><strong>Tuning parallelism:</strong> <a href="./partitioning.html">Partitioning</a></li></ul><hr class="footnotes-sep"><section class="footnotes"><ol class="footnotes-list"><li id="fn1" class="footnote-item"><p><em>Learning Spark, 2nd Edition</em>, Damji, Wenig, Das, Lee, ch. 7 <a href="#fnref1" class="footnote-backref">↩︎</a> <a href="#fnref1:1" class="footnote-backref">↩︎</a> <a href="#fnref1:2" class="footnote-backref">↩︎</a> <a href="#fnref1:3" class="footnote-backref">↩︎</a> <a href="#fnref1:4" class="footnote-backref">↩︎</a></p></li><li id="fn2" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/rdd-programming-guide.html" target="_blank" rel="noreferrer">RDD Programming Guide</a> <a href="#fnref2" class="footnote-backref">↩︎</a></p></li><li id="fn3" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/sql-performance-tuning.html" target="_blank" rel="noreferrer">Performance Tuning (Spark SQL, DataFrames and Datasets Guide)</a> <a href="#fnref3" class="footnote-backref">↩︎</a> <a href="#fnref3:1" class="footnote-backref">↩︎</a> <a href="#fnref3:2" class="footnote-backref">↩︎</a></p></li><li id="fn4" class="footnote-item"><p><a href="https://books.japila.pl/spark-sql-internals/bucketing/" target="_blank" rel="noreferrer">Bucketing (The Internals of Spark SQL)</a> <a href="#fnref4" class="footnote-backref">↩︎</a> <a href="#fnref4:1" class="footnote-backref">↩︎</a></p></li><li id="fn5" class="footnote-item"><p><a href="https://luminousmen.com/post/spark-tips-partition-tuning" target="_blank" rel="noreferrer">Spark Tips: Partition Tuning</a> <a href="#fnref5" class="footnote-backref">↩︎</a></p></li><li id="fn6" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/monitoring.html#spark-history-server" target="_blank" rel="noreferrer">Monitoring and Instrumentation</a> <a href="#fnref6" class="footnote-backref">↩︎</a> <a href="#fnref6:1" class="footnote-backref">↩︎</a> <a href="#fnref6:2" class="footnote-backref">↩︎</a> <a href="#fnref6:3" class="footnote-backref">↩︎</a></p></li><li id="fn7" class="footnote-item"><p><a href="https://issues.apache.org/jira/browse/SPARK-30602" target="_blank" rel="noreferrer">SPARK-30602: Support push-based shuffle to improve shuffle efficiency</a> <a href="#fnref7" class="footnote-backref">↩︎</a> <a href="#fnref7:1" class="footnote-backref">↩︎</a></p></li><li id="fn8" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration (Spark)</a> <a href="#fnref8" class="footnote-backref">↩︎</a> <a href="#fnref8:1" class="footnote-backref">↩︎</a> <a href="#fnref8:2" class="footnote-backref">↩︎</a></p></li><li id="fn9" class="footnote-item"><p><a href="https://blog.cloudera.com/how-to-tune-your-apache-spark-jobs-part-2/" target="_blank" rel="noreferrer">How to Tune Your Apache Spark Jobs (Part 2): Cloudera Engineering Blog</a> <a href="#fnref9" class="footnote-backref">↩︎</a></p></li><li id="fn10" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/job-scheduling.html" target="_blank" rel="noreferrer">Job Scheduling: Dynamic Resource Allocation</a> <a href="#fnref10" class="footnote-backref">↩︎</a></p></li><li id="fn11" class="footnote-item"><p><a href="https://www.databricks.com/blog/2020/05/29/adaptive-query-execution-speeding-up-spark-sql-at-runtime.html" target="_blank" rel="noreferrer">Adaptive Query Execution: Speeding Up Spark SQL at Runtime (Databricks)</a> <a href="#fnref11" class="footnote-backref">↩︎</a> <a href="#fnref11:1" class="footnote-backref">↩︎</a></p></li><li id="fn12" class="footnote-item"><p><a href="https://raw.githubusercontent.com/apache/spark/v3.5.0/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala" target="_blank" rel="noreferrer">SQLConf: shuffle-partition defaults (Spark source)</a> <a href="#fnref12" class="footnote-backref">↩︎</a></p></li><li id="fn13" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/sql-performance-tuning.html" target="_blank" rel="noreferrer">Performance Tuning: Coalescing Post Shuffle Partitions</a> <a href="#fnref13" class="footnote-backref">↩︎</a> <a href="#fnref13:1" class="footnote-backref">↩︎</a></p></li><li id="fn14" class="footnote-item"><p><a href="https://spark.apache.org/docs/latest/configuration.html" target="_blank" rel="noreferrer">Configuration (Spark)</a> <a href="#fnref14" class="footnote-backref">↩︎</a> <a href="#fnref14:1" class="footnote-backref">↩︎</a></p></li></ol></section>`,34)])])}const u=a(r,[["render",i]]);export{p as __pageData,u as default};

package/public/docs/assets/{tuning-reference_bottleneck-shuffle.md.CM-nTmIH.lean.js → tuning-reference_bottleneck-shuffle.md.DDMQVOW6.lean.js}

@@ -1 +1 @@
-import{_ as a,o as t,c as s,a5 as o}from"./chunks/framework.DSg0KOwT.js";const p=JSON.parse('{"title":"Shuffle I/O","description":"","frontmatter":{"title":"Shuffle I/O"},"headers":[],"relativePath":"tuning-reference/bottleneck-shuffle.md","filePath":"tuning-reference/bottleneck-shuffle.md"}'),r={name:"tuning-reference/bottleneck-shuffle.md"};function i(n,e,f,l,c,d){return t(),s("div",null,[...e[0]||(e[0]=[o("",28)])])}const u=a(r,[["render",i]]);export{p as __pageData,u as default};
+import{_ as a,o as t,c as s,a5 as o}from"./chunks/framework.DSg0KOwT.js";const p=JSON.parse('{"title":"Shuffle I/O","description":"","frontmatter":{"title":"Shuffle I/O"},"headers":[],"relativePath":"tuning-reference/bottleneck-shuffle.md","filePath":"tuning-reference/bottleneck-shuffle.md"}'),r={name:"tuning-reference/bottleneck-shuffle.md"};function i(n,e,f,l,c,d){return t(),s("div",null,[...e[0]||(e[0]=[o("",34)])])}const u=a(r,[["render",i]]);export{p as __pageData,u as default};