sparkforensics-server 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (167) hide show
  1. package/index.js +147 -0
  2. package/lib/static-files.js +50 -0
  3. package/package.json +28 -0
  4. package/public/assets/AutoscalingChurn-CP5OjN6v.js +1 -0
  5. package/public/assets/CacheUtilization-BjyawXL6.js +1 -0
  6. package/public/assets/CachingOpportunity-DV2ABG2G.js +1 -0
  7. package/public/assets/ConfigAudit-DMiBhAoG.js +1 -0
  8. package/public/assets/CoreUsageArea-CGYIgPnt.js +1 -0
  9. package/public/assets/Dashboard-D1_IrWRs.js +12 -0
  10. package/public/assets/ExecutorTimeline-DkaAGqQT.js +1 -0
  11. package/public/assets/Failures-Crngoicu.js +1 -0
  12. package/public/assets/GcPressure-CFXm4BnA.js +1 -0
  13. package/public/assets/IncompleteRun-BWEJTb13.js +1 -0
  14. package/public/assets/JobFailures-BmjxCA-h.js +1 -0
  15. package/public/assets/MemoryUtilization-Bu2uQJbd.js +1 -0
  16. package/public/assets/PlanExplorer-DKUwF0uz.js +1 -0
  17. package/public/assets/PlanGraphRoute-BjPSugIG.js +1 -0
  18. package/public/assets/ShuffleIO--Rur3neu.js +1 -0
  19. package/public/assets/Spill-DIMpt8c2.js +1 -0
  20. package/public/assets/TaskSkew-BDr1DbeI.js +1 -0
  21. package/public/assets/charts-vendor-NMqKV0H4.js +59 -0
  22. package/public/assets/index-4id52G-A.js +9 -0
  23. package/public/assets/index-D9kwVYIO.css +1 -0
  24. package/public/assets/parser-worker-00O9WEQC.js +65 -0
  25. package/public/assets/plan-graph-vendor-BnuhLJ6X.css +1 -0
  26. package/public/assets/plan-graph-vendor-vZQHG2EJ.js +7 -0
  27. package/public/assets/react-vendor-u4pyvC5Y.js +9 -0
  28. package/public/assets/useRecentFiles-46iBU9zJ.js +5 -0
  29. package/public/docs/404.html +26 -0
  30. package/public/docs/assets/app.Bxy4_HVA.js +1 -0
  31. package/public/docs/assets/chunks/@localSearchIndexroot.DclgKtlT.js +1 -0
  32. package/public/docs/assets/chunks/VPLocalSearchBox.L5jNOIR3.js +9 -0
  33. package/public/docs/assets/chunks/framework.CNDaxUod.js +20 -0
  34. package/public/docs/assets/chunks/theme.CoT6y5bf.js +2 -0
  35. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.BtjoE_1k.js +1 -0
  36. package/public/docs/assets/contributor-guide_architecture_board-widgets.md.BtjoE_1k.lean.js +1 -0
  37. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.DQyCHQNM.js +1 -0
  38. package/public/docs/assets/contributor-guide_architecture_detector-contract.md.DQyCHQNM.lean.js +1 -0
  39. package/public/docs/assets/contributor-guide_architecture_drill-down.md.BKfyDQck.js +1 -0
  40. package/public/docs/assets/contributor-guide_architecture_drill-down.md.BKfyDQck.lean.js +1 -0
  41. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.CzlXOnCq.js +1 -0
  42. package/public/docs/assets/contributor-guide_architecture_impact-estimation.md.CzlXOnCq.lean.js +1 -0
  43. package/public/docs/assets/contributor-guide_architecture_index.md.94lNbF-4.js +1 -0
  44. package/public/docs/assets/contributor-guide_architecture_index.md.94lNbF-4.lean.js +1 -0
  45. package/public/docs/assets/contributor-guide_architecture_overview.md.mQFlN66o.js +1 -0
  46. package/public/docs/assets/contributor-guide_architecture_overview.md.mQFlN66o.lean.js +1 -0
  47. package/public/docs/assets/contributor-guide_architecture_state-and-history.md.DaqttKgQ.js +1 -0
  48. package/public/docs/assets/contributor-guide_architecture_state-and-history.md.DaqttKgQ.lean.js +1 -0
  49. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.Bk6Iyj9S.js +1 -0
  50. package/public/docs/assets/contributor-guide_architecture_widget-rendering.md.Bk6Iyj9S.lean.js +1 -0
  51. package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.BaS_L10g.js +6 -0
  52. package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.BaS_L10g.lean.js +1 -0
  53. package/public/docs/assets/contributor-guide_contributing.md.C-lKs70I.js +1 -0
  54. package/public/docs/assets/contributor-guide_contributing.md.C-lKs70I.lean.js +1 -0
  55. package/public/docs/assets/contributor-guide_development-setup.md.CpUMVtIK.js +12 -0
  56. package/public/docs/assets/contributor-guide_development-setup.md.CpUMVtIK.lean.js +1 -0
  57. package/public/docs/assets/contributor-guide_testing.md.C0q-BJfl.js +1 -0
  58. package/public/docs/assets/contributor-guide_testing.md.C0q-BJfl.lean.js +1 -0
  59. package/public/docs/assets/index.md.CETnDZzR.js +1 -0
  60. package/public/docs/assets/index.md.CETnDZzR.lean.js +1 -0
  61. package/public/docs/assets/inter-italic-cyrillic-ext.r48I6akx.woff2 +0 -0
  62. package/public/docs/assets/inter-italic-cyrillic.By2_1cv3.woff2 +0 -0
  63. package/public/docs/assets/inter-italic-greek-ext.1u6EdAuj.woff2 +0 -0
  64. package/public/docs/assets/inter-italic-greek.DJ8dCoTZ.woff2 +0 -0
  65. package/public/docs/assets/inter-italic-latin-ext.CN1xVJS-.woff2 +0 -0
  66. package/public/docs/assets/inter-italic-latin.C2AdPX0b.woff2 +0 -0
  67. package/public/docs/assets/inter-italic-vietnamese.BSbpV94h.woff2 +0 -0
  68. package/public/docs/assets/inter-roman-cyrillic-ext.BBPuwvHQ.woff2 +0 -0
  69. package/public/docs/assets/inter-roman-cyrillic.C5lxZ8CY.woff2 +0 -0
  70. package/public/docs/assets/inter-roman-greek-ext.CqjqNYQ-.woff2 +0 -0
  71. package/public/docs/assets/inter-roman-greek.BBVDIX6e.woff2 +0 -0
  72. package/public/docs/assets/inter-roman-latin-ext.4ZJIpNVo.woff2 +0 -0
  73. package/public/docs/assets/inter-roman-latin.Di8DUHzh.woff2 +0 -0
  74. package/public/docs/assets/inter-roman-vietnamese.BjW4sHH5.woff2 +0 -0
  75. package/public/docs/assets/style.D2mY0HUs.css +1 -0
  76. package/public/docs/assets/user-guide_getting-started.md.BJdD7a7_.js +3 -0
  77. package/public/docs/assets/user-guide_getting-started.md.BJdD7a7_.lean.js +1 -0
  78. package/public/docs/assets/user-guide_mcp-tools.md.DwYI1d6C.js +95 -0
  79. package/public/docs/assets/user-guide_mcp-tools.md.DwYI1d6C.lean.js +1 -0
  80. package/public/docs/assets/user-guide_run-comparison.md.5-oaOfTu.js +1 -0
  81. package/public/docs/assets/user-guide_run-comparison.md.5-oaOfTu.lean.js +1 -0
  82. package/public/docs/assets/user-guide_understanding-findings.md.CK2WZ0Tf.js +1 -0
  83. package/public/docs/assets/user-guide_understanding-findings.md.CK2WZ0Tf.lean.js +1 -0
  84. package/public/docs/contributor-guide/architecture/board-widgets.html +29 -0
  85. package/public/docs/contributor-guide/architecture/detector-contract.html +29 -0
  86. package/public/docs/contributor-guide/architecture/drill-down.html +29 -0
  87. package/public/docs/contributor-guide/architecture/impact-estimation.html +29 -0
  88. package/public/docs/contributor-guide/architecture/index.html +29 -0
  89. package/public/docs/contributor-guide/architecture/overview.html +29 -0
  90. package/public/docs/contributor-guide/architecture/state-and-history.html +29 -0
  91. package/public/docs/contributor-guide/architecture/widget-rendering.html +29 -0
  92. package/public/docs/contributor-guide/architecture/worker-protocol.html +34 -0
  93. package/public/docs/contributor-guide/contributing.html +29 -0
  94. package/public/docs/contributor-guide/development-setup.html +40 -0
  95. package/public/docs/contributor-guide/testing.html +29 -0
  96. package/public/docs/favicon.svg +4 -0
  97. package/public/docs/hashmap.json +1 -0
  98. package/public/docs/index.html +29 -0
  99. package/public/docs/user-guide/getting-started.html +31 -0
  100. package/public/docs/user-guide/mcp-tools.html +123 -0
  101. package/public/docs/user-guide/run-comparison.html +29 -0
  102. package/public/docs/user-guide/understanding-findings.html +29 -0
  103. package/public/docs/vp-icons.css +0 -0
  104. package/public/favicon.svg +4 -0
  105. package/public/index.html +36 -0
  106. package/public/vendor/spark-doc/anchors.json +205 -0
  107. package/public/vendor/spark-doc/index.html +4202 -0
  108. package/public/vendor/spark-doc/landing.html +418 -0
  109. package/public/vendor/spark-doc/meta.html +970 -0
  110. package/vendor-core/analyzer.js +167 -0
  111. package/vendor-core/assert-never.js +3 -0
  112. package/vendor-core/cli/budgets.js +203 -0
  113. package/vendor-core/cli/collect-run.js +107 -0
  114. package/vendor-core/core-count.js +68 -0
  115. package/vendor-core/core-locality-ratio.js +54 -0
  116. package/vendor-core/core-time-series.js +92 -0
  117. package/vendor-core/core-usage-locality.js +70 -0
  118. package/vendor-core/detectors.js +1989 -0
  119. package/vendor-core/docs-config.js +72 -0
  120. package/vendor-core/docs-site-config.js +23 -0
  121. package/vendor-core/efficiency-model.js +62 -0
  122. package/vendor-core/etl-phases.js +28 -0
  123. package/vendor-core/event-handlers.js +906 -0
  124. package/vendor-core/event-schemas.js +405 -0
  125. package/vendor-core/evidence-availability.js +121 -0
  126. package/vendor-core/evidence-report.js +459 -0
  127. package/vendor-core/finding-action-label.js +97 -0
  128. package/vendor-core/finding-filter-predicate.js +38 -0
  129. package/vendor-core/format-utils.js +167 -0
  130. package/vendor-core/impact-band.js +50 -0
  131. package/vendor-core/impact-estimator.js +428 -0
  132. package/vendor-core/ingest.js +139 -0
  133. package/vendor-core/job-groups.js +30 -0
  134. package/vendor-core/load-vendored.js +24 -0
  135. package/vendor-core/lz4-block.js +135 -0
  136. package/vendor-core/mcp-error.js +3 -0
  137. package/vendor-core/mcp-server-factory.js +115 -0
  138. package/vendor-core/mcp-tools.js +331 -0
  139. package/vendor-core/model-assembler.js +76 -0
  140. package/vendor-core/occupancy.js +202 -0
  141. package/vendor-core/parser-worker.js +249 -0
  142. package/vendor-core/plan-dot.js +25 -0
  143. package/vendor-core/plan-duration-attribution.js +185 -0
  144. package/vendor-core/plan-graph-model.js +171 -0
  145. package/vendor-core/plan-node-detail.js +159 -0
  146. package/vendor-core/plan-summary.js +233 -0
  147. package/vendor-core/plan-tree-walk.js +29 -0
  148. package/vendor-core/proxy.js +157 -0
  149. package/vendor-core/recommendation-rollup.js +197 -0
  150. package/vendor-core/redact.js +175 -0
  151. package/vendor-core/rolling-log-reassembly.js +52 -0
  152. package/vendor-core/run-aggregates.js +44 -0
  153. package/vendor-core/run-comparison.js +458 -0
  154. package/vendor-core/scaling-sim.js +73 -0
  155. package/vendor-core/session-snapshot.js +79 -0
  156. package/vendor-core/shs-fetch.js +196 -0
  157. package/vendor-core/shs-load.js +121 -0
  158. package/vendor-core/shs-request.js +101 -0
  159. package/vendor-core/shs-schemas.js +13 -0
  160. package/vendor-core/snappy-block.js +140 -0
  161. package/vendor-core/stage-quantiles.js +199 -0
  162. package/vendor-core/threshold-summary.js +35 -0
  163. package/vendor-core/types.js +286 -0
  164. package/vendor-core/vendor/fflate.js +2695 -0
  165. package/vendor-core/vendor/fzstd.js +768 -0
  166. package/vendor-core/wall-clock.js +36 -0
  167. package/vendor-core/wasted-core-hours.js +68 -0
package/public/vendor/spark-doc/landing.html ADDED
@@ -0,0 +1,418 @@
1
+ <!doctype html>
2
+ <html lang="en">
3
+ <head>
4
+ <meta charset="utf-8">
5
+ <meta name="viewport" content="width=device-width, initial-scale=1">
6
+ <meta name="description" content="A practical Spark tuning reference: trace job symptoms to the mechanics and settings that change outcomes.">
7
+ <title>Spark Tuning Reference</title>
8
+ <script>
9
+ (function () {
10
+ var storageKey = 'shuffle-works-theme';
11
+ var storedTheme = null;
12
+ try { storedTheme = localStorage.getItem(storageKey); } catch (error) {}
13
+ var theme;
14
+ if (storedTheme === 'light' || storedTheme === 'dark') {
15
+ theme = storedTheme;
16
+ } else {
17
+ theme = window.matchMedia && window.matchMedia('(prefers-color-scheme: light)').matches
18
+ ? 'light'
19
+ : 'dark';
20
+ }
21
+ document.documentElement.dataset.theme = theme;
22
+ }());
23
+ </script>
24
+ <style>
25
+ @import url('https://fonts.googleapis.com/css2?family=Recursive:wght@400;500;600;700&family=JetBrains+Mono:wght@400;500;600&display=swap');
26
+ :root {
27
+ color-scheme: dark;
28
+ --color-bg: #080b0d;
29
+ --color-surface: #11171a;
30
+ --color-text: #f4f4f5;
31
+ --color-text-muted: #a9b5b4;
32
+ --color-border: #233035;
33
+ --color-accent: #ff6b2c;
34
+ --color-accent-hover: #ff8a4f;
35
+ --color-signal: #39d6b5;
36
+ --radius: 0.75rem;
37
+ --shadow-soft: 0 1.5rem 3rem -1rem rgba(0, 0, 0, 0.45);
38
+ --font-sans: 'Recursive', -apple-system, BlinkMacSystemFont, sans-serif;
39
+ --font-mono: 'JetBrains Mono', ui-monospace, monospace;
40
+
41
+ --bg: var(--color-bg);
42
+ --surface: var(--color-surface);
43
+ --surface-strong: #172024;
44
+ --text: var(--color-text);
45
+ --muted: var(--color-text-muted);
46
+ --line: var(--color-border);
47
+ --orange: var(--color-accent);
48
+ --orange-hover: var(--color-accent-hover);
49
+ --teal: var(--color-signal);
50
+ --teal-soft: rgba(57, 214, 181, 0.13);
51
+ --orange-soft: rgba(255, 107, 44, 0.13);
52
+ --on-orange: #17110e;
53
+ --sans: var(--font-sans);
54
+ --mono: var(--font-mono);
55
+ }
56
+
57
+ :root[data-theme="light"] {
58
+ color-scheme: light;
59
+ --color-bg: #f7f8f4;
60
+ --color-surface: #ffffff;
61
+ --color-text: #151b1e;
62
+ --color-text-muted: #4f5f60;
63
+ --color-border: #dbe2df;
64
+ --color-accent: #bf4413;
65
+ --color-accent-hover: #aa3510;
66
+ --color-signal: #007e6b;
67
+ --shadow-soft: 0 1.5rem 3rem -1rem rgba(21, 27, 30, 0.16);
68
+
69
+ --bg: var(--color-bg);
70
+ --surface: var(--color-surface);
71
+ --surface-strong: #edf1ed;
72
+ --text: var(--color-text);
73
+ --muted: var(--color-text-muted);
74
+ --line: var(--color-border);
75
+ --orange: var(--color-accent);
76
+ --orange-hover: var(--color-accent-hover);
77
+ --teal: var(--color-signal);
78
+ --teal-soft: rgba(0, 126, 107, 0.1);
79
+ --orange-soft: rgba(191, 68, 19, 0.1);
80
+ --on-orange: #ffffff;
81
+ }
82
+
83
+ * { box-sizing: border-box; }
84
+ html { scroll-behavior: smooth; }
85
+ body {
86
+ min-width: 20rem;
87
+ margin: 0;
88
+ background: var(--bg);
89
+ color: var(--text);
90
+ font-family: var(--sans);
91
+ font-size: 16px;
92
+ line-height: 1.55;
93
+ -webkit-font-smoothing: antialiased;
94
+ }
95
+ a { color: inherit; text-decoration: none; }
96
+ button { font: inherit; }
97
+ a:focus-visible, button:focus-visible {
98
+ outline: 3px solid var(--teal);
99
+ outline-offset: 3px;
100
+ }
101
+ .skip-link {
102
+ position: absolute;
103
+ z-index: 100;
104
+ top: -5rem;
105
+ left: 1rem;
106
+ padding: .65rem .85rem;
107
+ background: var(--teal);
108
+ color: #06221d;
109
+ font-weight: 800;
110
+ }
111
+ .skip-link:focus { top: 1rem; }
112
+ .container { width: min(100% - 2.5rem, 72rem); margin-inline: auto; }
113
+ .mono { font-family: var(--mono); }
114
+ .eyebrow {
115
+ margin: 0;
116
+ color: var(--teal);
117
+ font-family: var(--mono);
118
+ font-size: .76rem;
119
+ font-weight: 700;
120
+ letter-spacing: .075em;
121
+ text-transform: uppercase;
122
+ }
123
+
124
+ .site-header {
125
+ border-bottom: 1px solid var(--line);
126
+ background: var(--bg);
127
+ }
128
+ .header-inner {
129
+ display: flex;
130
+ min-height: 4.25rem;
131
+ align-items: center;
132
+ justify-content: space-between;
133
+ gap: 1rem;
134
+ }
135
+ .brand { display: inline-flex; align-items: center; gap: .6rem; font-weight: 500; letter-spacing: -.01em; color: var(--muted); font-size: .92rem; }
136
+ .brand-mark { font-family: var(--mono); font-weight: 800; color: var(--text); }
137
+ .header-nav { display: flex; align-items: center; gap: .35rem; }
138
+ .header-link, .theme-toggle {
139
+ display: inline-flex;
140
+ min-height: 2.75rem;
141
+ align-items: center;
142
+ justify-content: center;
143
+ gap: .35rem;
144
+ padding: .35rem .65rem;
145
+ border: 1px solid transparent;
146
+ color: var(--muted);
147
+ background: transparent;
148
+ font-size: .88rem;
149
+ }
150
+ .header-link:hover { color: var(--text); }
151
+ .theme-toggle {
152
+ width: 2.75rem;
153
+ padding: 0;
154
+ border-color: var(--line);
155
+ color: var(--text);
156
+ cursor: pointer;
157
+ border-radius: var(--radius);
158
+ }
159
+ .theme-toggle:hover { border-color: var(--teal); color: var(--teal); }
160
+ .theme-icon { width: 1.15rem; height: 1.15rem; fill: none; stroke: currentColor; stroke-width: 1.8; stroke-linecap: round; }
161
+ .theme-icon.moon { display: none; }
162
+ :root[data-theme="light"] .theme-icon.sun { display: none; }
163
+ :root[data-theme="light"] .theme-icon.moon { display: block; }
164
+
165
+ .hero { padding: clamp(4.5rem, 10vw, 8rem) 0 clamp(4rem, 8vw, 6.5rem); }
166
+ .hero-grid { display: grid; gap: 3rem; align-items: center; }
167
+ .hero-copy { max-width: 44rem; }
168
+ h1, h2, h3 { text-wrap: balance; }
169
+ h1 {
170
+ max-width: 11ch;
171
+ margin: .65rem 0 1.15rem;
172
+ font-size: clamp(3.15rem, 8vw, 6.75rem);
173
+ font-weight: 850;
174
+ letter-spacing: -.03em;
175
+ line-height: .9;
176
+ }
177
+ .hero-summary { max-width: 40rem; margin: 0; color: var(--muted); font-size: clamp(1.08rem, 2.1vw, 1.3rem); }
178
+ .hero-actions { display: flex; flex-wrap: wrap; gap: .8rem; margin-top: 2rem; }
179
+ .known-term { max-width: 40rem; margin-top: 1.5rem; }
180
+ .known-term label { display: block; margin-bottom: .45rem; color: var(--text); font-family: var(--mono); font-size: .82rem; font-weight: 700; }
181
+ .known-term-row { display: flex; flex-wrap: wrap; gap: .55rem; }
182
+ .known-term input { min-height: 3rem; min-width: min(100%, 19rem); flex: 1 1 14rem; padding: .65rem .75rem; border: 1px solid var(--line); border-radius: var(--radius); background: var(--surface); color: var(--text); font: .88rem var(--mono); }
183
+ .known-term input:focus-visible { outline: 3px solid var(--teal); outline-offset: 2px; border-color: var(--teal); }
184
+ .known-term small { display: block; margin-top: .5rem; color: var(--muted); font-size: .82rem; }
185
+ .button {
186
+ display: inline-flex;
187
+ min-height: 3rem;
188
+ align-items: center;
189
+ justify-content: center;
190
+ gap: .6rem;
191
+ padding: .65rem 1.05rem;
192
+ border: 1px solid var(--line);
193
+ font-family: var(--mono);
194
+ font-size: .88rem;
195
+ font-weight: 700;
196
+ border-radius: var(--radius);
197
+ }
198
+ .button-primary { border-color: var(--orange); background: var(--orange); color: var(--on-orange); }
199
+ .button-primary:hover { background: var(--orange-hover); border-color: var(--orange-hover); }
200
+ .button-secondary { background: transparent; color: var(--text); }
201
+ .button-secondary:hover { border-color: var(--teal); color: var(--teal); }
202
+ .arrow { font-size: 1.1em; }
203
+
204
+ .signal-board {
205
+ display: grid;
206
+ gap: 0;
207
+ border: 1px solid var(--line);
208
+ background: var(--surface);
209
+ box-shadow: var(--shadow-soft);
210
+ border-radius: var(--radius);
211
+ overflow: hidden;
212
+ }
213
+ .board-label { padding: .8rem 1rem; border-bottom: 1px solid var(--line); color: var(--muted); font-family: var(--mono); font-size: .74rem; }
214
+ .signal-list { display: grid; margin: 0; padding: 0; list-style: none; }
215
+ .signal-list li { border-bottom: 1px solid var(--line); }
216
+ .signal-list li:last-child { border: 0; }
217
+ .signal-link { display: grid; grid-template-columns: auto 1fr auto; gap: .75rem; align-items: center; padding: .95rem 1rem; }
218
+ .signal-link:hover { background: var(--surface-strong); }
219
+ .signal-marker { width: .6rem; height: .6rem; border-radius: 999px; background: var(--teal); box-shadow: 0 0 0 .22rem var(--teal-soft); }
220
+ .signal-list li:nth-child(2) .signal-marker { background: var(--orange); box-shadow: 0 0 0 .22rem var(--orange-soft); }
221
+ .signal-title { font-weight: 750; }
222
+ .signal-destination { color: var(--muted); font-family: var(--mono); font-size: .75rem; }
223
+
224
+ .section { padding: clamp(4rem, 8vw, 6.5rem) 0; }
225
+ .section-heading { max-width: 44rem; margin-bottom: 2rem; }
226
+ .section-heading h2 { margin: 0 0 .8rem; font-size: clamp(2rem, 4vw, 3.25rem); letter-spacing: -.03em; line-height: 1.04; }
227
+ .section-heading p { margin: 0; color: var(--muted); font-size: 1.05rem; }
228
+ .decision-note { max-width: 52rem; margin: -1rem 0 2rem; padding: 1rem 1.1rem; border: 1px solid var(--line); background: var(--surface); color: var(--muted); border-radius: var(--radius); }
229
+ .decision-note strong { color: var(--text); }
230
+ .symptom-grid { display: grid; border-top: 1px solid var(--line); border-left: 1px solid var(--line); border-radius: var(--radius); overflow: hidden; }
231
+ .symptom { display: flex; min-height: 12.5rem; flex-direction: column; justify-content: space-between; padding: 1.25rem; border-right: 1px solid var(--line); border-bottom: 1px solid var(--line); background: var(--bg); }
232
+ .symptom:hover { background: var(--surface); }
233
+ .symptom h3 { margin: 0 0 .45rem; font-size: 1.2rem; letter-spacing: -.025em; }
234
+ .symptom p { margin: 0; color: var(--muted); font-size: .94rem; }
235
+ .symptom a { display: inline-flex; width: fit-content; margin-top: 1.25rem; color: var(--teal); font-family: var(--mono); font-size: .82rem; font-weight: 700; }
236
+ .symptom a:hover { color: var(--orange); }
237
+
238
+ .workflow { background: var(--surface); border-block: 1px solid var(--line); }
239
+ .workflow-grid { display: grid; gap: 2rem; }
240
+ .workflow-copy { max-width: 32rem; }
241
+ .workflow-copy h2 { margin: 0 0 .9rem; font-size: clamp(2rem, 4vw, 3.25rem); letter-spacing: -.03em; line-height: 1.04; }
242
+ .workflow-copy p { color: var(--muted); }
243
+ .steps { display: grid; margin: 0; padding: 0; list-style: none; border-top: 1px solid var(--line); }
244
+ .steps li { display: grid; grid-template-columns: 5.5rem 1fr; gap: 1rem; padding: 1rem 0; border-bottom: 1px solid var(--line); }
245
+ .step-label { color: var(--orange); font-family: var(--mono); font-size: .72rem; font-weight: 700; letter-spacing: .04em; text-transform: uppercase; }
246
+ .steps strong { display: block; margin-bottom: .15rem; }
247
+ .steps span:not(.step-label) { color: var(--muted); font-size: .93rem; }
248
+
249
+ @media (min-width: 46rem) {
250
+ .hero-grid { grid-template-columns: minmax(0, 1.25fr) minmax(19rem, .75fr); }
251
+ .symptom-grid { grid-template-columns: repeat(2, 1fr); }
252
+ .workflow-grid { grid-template-columns: minmax(0, .88fr) minmax(24rem, 1.12fr); align-items: center; }
253
+ }
254
+ @media (min-width: 70rem) { .symptom-grid { grid-template-columns: repeat(5, 1fr); } }
255
+ @media (max-width: 34rem) {
256
+ .container { width: min(100% - 1.5rem, 72rem); }
257
+ .brand { gap: .4rem; font-size: .86rem; white-space: nowrap; }
258
+ .header-link.github { display: none; }
259
+ h1 { font-size: clamp(2.85rem, 16vw, 4.25rem); }
260
+ .signal-destination { display: none; }
261
+ .header-inner { min-height: 3.25rem; }
262
+ }
263
+ @media (max-width: 24rem) {
264
+ .header-link:not(.github) { display: none; }
265
+ .brand-name { display: none; }
266
+ }
267
+ @media (prefers-reduced-motion: reduce) {
268
+ *, *::before, *::after { scroll-behavior: auto !important; transition-duration: .01ms !important; animation-duration: .01ms !important; animation-iteration-count: 1 !important; }
269
+ }
270
+ </style>
271
+ </head>
272
+ <body>
273
+ <a class="skip-link" href="#main-content">Skip to content</a>
274
+ <header class="site-header">
275
+ <div class="container header-inner">
276
+ <a class="brand" href="landing.html" aria-label="Spark Tuning Reference home" aria-current="page">
277
+ <span class="brand-mark" aria-hidden="true">S/</span>
278
+ <span class="brand-name">Spark Tuning Reference</span>
279
+ </a>
280
+ <nav class="header-nav" aria-label="Primary navigation">
281
+ <a class="header-link" href="index.html">Reference</a>
282
+ <a class="header-link github" href="https://github.com/shuffle-works/spark-tuning-reference" target="_blank" rel="noopener">GitHub <span aria-hidden="true">↗</span></a>
283
+ <button id="theme-toggle" class="theme-toggle" type="button" aria-label="Switch to light theme" aria-pressed="false">
284
+ <svg class="theme-icon sun" viewBox="0 0 24 24" aria-hidden="true"><circle cx="12" cy="12" r="3.5"></circle><path d="M12 2v2M12 20v2M4.93 4.93l1.41 1.41M17.66 17.66l1.41 1.41M2 12h2M20 12h2M4.93 19.07l1.41-1.41M17.66 6.34l1.41-1.41"></path></svg>
285
+ <svg class="theme-icon moon" viewBox="0 0 24 24" aria-hidden="true"><path d="M20.5 14.2A8.7 8.7 0 0 1 9.8 3.5 8.7 8.7 0 1 0 20.5 14.2Z"></path></svg>
286
+ </button>
287
+ </nav>
288
+ </div>
289
+ </header>
290
+
291
+ <main id="main-content">
292
+ <section class="hero" aria-labelledby="page-title">
293
+ <div class="container hero-grid">
294
+ <div class="hero-copy">
295
+ <p class="eyebrow">Practical mechanics, stable references</p>
296
+ <h1 id="page-title">Spark Tuning Reference</h1>
297
+ <p class="hero-summary">Move from a job symptom to the Spark mechanics and settings that can change its outcome without treating a tuning guess as evidence.</p>
298
+ <div class="hero-actions">
299
+ <a class="button button-primary" href="index.html">Browse the full reference <span class="arrow" aria-hidden="true">→</span></a>
300
+ <a class="button button-secondary" href="#symptoms">Choose a symptom</a>
301
+ <form class="known-term" id="known-term-form" action="index.html#config" method="get">
302
+ <label for="known-term-input">Know the term or config key?</label>
303
+ <div class="known-term-row">
304
+ <input id="known-term-input" name="term" type="search" list="known-term-options" placeholder="e.g. spark.sql.shuffle.partitions" autocomplete="off">
305
+ <button class="button button-secondary" type="submit">Jump to topic</button>
306
+ </div>
307
+ <small>Known routes: memory, shuffle, joins, AQE, and config. Other terms open the configuration reference.</small>
308
+ <datalist id="known-term-options">
309
+ <option value="memory"></option><option value="shuffle"></option><option value="joins"></option><option value="AQE"></option><option value="config"></option>
310
+ <option value="spark.memory.fraction"></option><option value="spark.sql.shuffle.partitions"></option><option value="spark.sql.adaptive.enabled"></option><option value="spark.sql.autoBroadcastJoinThreshold"></option>
311
+ </datalist>
312
+ </form>
313
+ </div>
314
+ </div>
315
+ <aside class="signal-board" aria-label="Example symptom paths">
316
+ <div class="board-label">evidence → mechanic → lever</div>
317
+ <ul class="signal-list">
318
+ <li><a class="signal-link" href="index.html#memory-model"><span class="signal-marker" aria-hidden="true"></span><span class="signal-title">Executor memory pressure</span><span class="signal-destination">#memory-model</span></a></li>
319
+ <li><a class="signal-link" href="index.html#shuffle"><span class="signal-marker" aria-hidden="true"></span><span class="signal-title">Slow, wide stages</span><span class="signal-destination">#shuffle</span></a></li>
320
+ <li><a class="signal-link" href="index.html#joins"><span class="signal-marker" aria-hidden="true"></span><span class="signal-title">Join strategy drift</span><span class="signal-destination">#joins</span></a></li>
321
+ </ul>
322
+ </aside>
323
+ </div>
324
+ </section>
325
+
326
+ <section class="section" id="symptoms" aria-labelledby="symptoms-title">
327
+ <div class="container">
328
+ <div class="section-heading">
329
+ <h2 id="symptoms-title">Symptoms, mapped to the right lever</h2>
330
+ <p>Each path stays on the canonical reference document, so existing SparkForensics links and saved anchors continue to work.</p>
331
+ <p class="decision-note"><strong>How to use this:</strong> SparkForensics is the companion that reads your Spark event log and surfaces evidence; this reference explains the mechanics and settings behind it. Diagnose the symptom, change one informed lever, then validate against the next run’s evidence. Confirm behavior and defaults against your deployed Spark version and distribution.</p>
332
+ </div>
333
+ <div class="symptom-grid">
334
+ <article class="symptom">
335
+ <div><h3>Spill or GC pressure</h3><p>Trace executor memory, object lifetime, and the trade-offs behind changing memory settings.</p></div>
336
+ <a href="index.html#memory-model">Open memory model <span aria-hidden="true">→</span></a>
337
+ </article>
338
+ <article class="symptom">
339
+ <div><h3>Shuffle-heavy stages</h3><p>Understand write/read costs, partition size, and why moving data becomes the bottleneck.</p></div>
340
+ <a href="index.html#shuffle">Open shuffle guide <span aria-hidden="true">→</span></a>
341
+ </article>
342
+ <article class="symptom">
343
+ <div><h3>Slow or skewed joins</h3><p>Compare join mechanics before changing broadcast thresholds or reshaping a plan.</p></div>
344
+ <a href="index.html#joins">Open join optimization <span aria-hidden="true">→</span></a>
345
+ </article>
346
+ <article class="symptom">
347
+ <div><h3>Plan adapts poorly</h3><p>Use AQE’s runtime choices as an inspectable mechanism, not a checkbox to enable blindly.</p></div>
348
+ <a href="index.html#aqe">Open AQE guide <span aria-hidden="true">→</span></a>
349
+ </article>
350
+ <article class="symptom">
351
+ <div><h3>Need the exact setting</h3><p>Check a setting’s scope, default, and interaction before applying it to a cluster.</p></div>
352
+ <a href="index.html#config">Open config reference <span aria-hidden="true">→</span></a>
353
+ </article>
354
+ </div>
355
+ </div>
356
+ </section>
357
+
358
+ <section class="section workflow" aria-labelledby="workflow-title">
359
+ <div class="container workflow-grid">
360
+ <div class="workflow-copy">
361
+ <h2 id="workflow-title">Let SparkForensics surface the evidence. Use this reference to reason about it.</h2>
362
+ <p>The two tools stay deliberately separate: SparkForensics reads the event log; this static reference explains the mechanics and settings behind what you find.</p>
363
+ </div>
364
+ <ol class="steps">
365
+ <li><span class="step-label">Observe</span><div><strong>Read the stage or executor behavior</strong><span>Start with the event log, metrics, and plan details Spark already produces.</span></div></li>
366
+ <li><span class="step-label">Explain</span><div><strong>Name the mechanism at work</strong><span>Use the stable reference anchors to connect symptoms to memory, shuffle, joins, AQE, or configuration.</span></div></li>
367
+ <li><span class="step-label">Test</span><div><strong>Change one informed lever</strong><span>Make a measured change, then compare the evidence again rather than carrying a tuning superstition forward.</span></div></li>
368
+ </ol>
369
+ </div>
370
+ </section>
371
+ </main>
372
+
373
+ <script>
374
+ (function () {
375
+ var storageKey = 'shuffle-works-theme';
376
+ var root = document.documentElement;
377
+ var button = document.getElementById('theme-toggle');
378
+
379
+ function updateControl() {
380
+ var isLight = root.dataset.theme === 'light';
381
+ button.setAttribute('aria-pressed', String(isLight));
382
+ button.setAttribute('aria-label', isLight ? 'Switch to dark theme' : 'Switch to light theme');
383
+ }
384
+
385
+ button.addEventListener('click', function () {
386
+ var theme = root.dataset.theme === 'light' ? 'dark' : 'light';
387
+ root.dataset.theme = theme;
388
+ try { localStorage.setItem(storageKey, theme); } catch (error) {}
389
+ updateControl();
390
+ });
391
+ updateControl();
392
+
393
+ var knownTermForm = document.getElementById('known-term-form');
394
+ var knownTermInput = document.getElementById('known-term-input');
395
+ var knownTermRoutes = {
396
+ memory: 'memory-model',
397
+ 'memory-model': 'memory-model',
398
+ spill: 'memory-model',
399
+ 'spark.memory.fraction': 'memory-model',
400
+ shuffle: 'shuffle',
401
+ 'spark.sql.shuffle.partitions': 'shuffle',
402
+ joins: 'joins',
403
+ join: 'joins',
404
+ 'spark.sql.autobroadcastjointhreshold': 'joins',
405
+ aqe: 'aqe',
406
+ 'spark.sql.adaptive.enabled': 'aqe',
407
+ config: 'config',
408
+ };
409
+ knownTermForm.addEventListener('submit', function (event) {
410
+ event.preventDefault();
411
+ var term = knownTermInput.value.trim().toLowerCase();
412
+ var route = knownTermRoutes[term] || 'config';
413
+ window.location.href = 'index.html#' + route;
414
+ });
415
+ }());
416
+ </script>
417
+ </body>
418
+ </html>