sparkforensics-mcp 0.1.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (146) hide show
  1. package/README.md +6 -0
  2. package/bin/sparkforensics-mcp.mjs +19 -6
  3. package/package.json +18 -4
  4. package/vendor-core/analyzer.js +74 -74
  5. package/vendor-core/cli/budgets.js +13 -27
  6. package/vendor-core/cli/collect-run.js +43 -19
  7. package/vendor-core/core-count.js +25 -27
  8. package/vendor-core/core-locality-ratio.js +4 -11
  9. package/vendor-core/core-time-series.js +6 -12
  10. package/vendor-core/core-usage-locality.js +3 -4
  11. package/vendor-core/detectors.js +395 -389
  12. package/vendor-core/docs-config.js +69 -21
  13. package/vendor-core/docs-content/chapters/01-intro.md +32 -0
  14. package/vendor-core/docs-content/chapters/02-spark-architecture.md +76 -0
  15. package/vendor-core/docs-content/chapters/03-memory-model.md +73 -0
  16. package/vendor-core/docs-content/chapters/04-partitioning.md +65 -0
  17. package/vendor-core/docs-content/chapters/05-joins.md +62 -0
  18. package/vendor-core/docs-content/chapters/06-shuffle.md +59 -0
  19. package/vendor-core/docs-content/chapters/07-data-formats.md +81 -0
  20. package/vendor-core/docs-content/chapters/07b-table-formats.md +56 -0
  21. package/vendor-core/docs-content/chapters/08-caching.md +58 -0
  22. package/vendor-core/docs-content/chapters/09-pyspark.md +78 -0
  23. package/vendor-core/docs-content/chapters/10-aqe.md +167 -0
  24. package/vendor-core/docs-content/chapters/11-cluster-config.md +170 -0
  25. package/vendor-core/docs-content/chapters/12-anti-patterns.md +171 -0
  26. package/vendor-core/docs-content/chapters/14-metrics.md +87 -0
  27. package/vendor-core/docs-content/chapters/15-config.md +93 -0
  28. package/vendor-core/docs-content/chapters/nav-index.json +370 -0
  29. package/vendor-core/docs-content/detection/cache.md +7 -0
  30. package/vendor-core/docs-content/detection/cfg.md +15 -0
  31. package/vendor-core/docs-content/detection/chrn.md +9 -0
  32. package/vendor-core/docs-content/detection/cold.md +4 -0
  33. package/vendor-core/docs-content/detection/cstor.md +4 -0
  34. package/vendor-core/docs-content/detection/fail.md +5 -0
  35. package/vendor-core/docs-content/detection/gc.md +4 -0
  36. package/vendor-core/docs-content/detection/host.md +5 -0
  37. package/vendor-core/docs-content/detection/incmp.md +6 -0
  38. package/vendor-core/docs-content/detection/jobs.md +4 -0
  39. package/vendor-core/docs-content/detection/local.md +6 -0
  40. package/vendor-core/docs-content/detection/mem.md +10 -0
  41. package/vendor-core/docs-content/detection/part.md +5 -0
  42. package/vendor-core/docs-content/detection/plan.md +14 -0
  43. package/vendor-core/docs-content/detection/retry.md +4 -0
  44. package/vendor-core/docs-content/detection/sfail.md +5 -0
  45. package/vendor-core/docs-content/detection/shape.md +5 -0
  46. package/vendor-core/docs-content/detection/shfl.md +4 -0
  47. package/vendor-core/docs-content/detection/skew.md +6 -0
  48. package/vendor-core/docs-content/detection/slow.md +6 -0
  49. package/vendor-core/docs-content/detection/spec.md +8 -0
  50. package/vendor-core/docs-content/detection/spill.md +7 -0
  51. package/vendor-core/docs-content/detection/strag.md +5 -0
  52. package/vendor-core/docs-content/detection/tiny.md +4 -0
  53. package/vendor-core/docs-content/detection/util.md +4 -0
  54. package/vendor-core/docs-content/diagrams/aqe-loop.dark.svg +1 -0
  55. package/vendor-core/docs-content/diagrams/aqe-loop.svg +1 -0
  56. package/vendor-core/docs-content/diagrams/broadcast-vs-shuffle.dark.svg +1 -0
  57. package/vendor-core/docs-content/diagrams/broadcast-vs-shuffle.svg +1 -0
  58. package/vendor-core/docs-content/diagrams/cache-lifecycle.dark.svg +1 -0
  59. package/vendor-core/docs-content/diagrams/cache-lifecycle.svg +1 -0
  60. package/vendor-core/docs-content/diagrams/cold-start-timeline.dark.svg +1 -0
  61. package/vendor-core/docs-content/diagrams/cold-start-timeline.svg +1 -0
  62. package/vendor-core/docs-content/diagrams/columnar-layout.dark.svg +1 -0
  63. package/vendor-core/docs-content/diagrams/columnar-layout.svg +1 -0
  64. package/vendor-core/docs-content/diagrams/container-memory.dark.svg +1 -0
  65. package/vendor-core/docs-content/diagrams/container-memory.svg +1 -0
  66. package/vendor-core/docs-content/diagrams/dag-stages.dark.svg +1 -0
  67. package/vendor-core/docs-content/diagrams/dag-stages.svg +1 -0
  68. package/vendor-core/docs-content/diagrams/driver-executor.dark.svg +1 -0
  69. package/vendor-core/docs-content/diagrams/driver-executor.svg +1 -0
  70. package/vendor-core/docs-content/diagrams/duplicate-plan-subtree.dark.svg +1 -0
  71. package/vendor-core/docs-content/diagrams/duplicate-plan-subtree.svg +1 -0
  72. package/vendor-core/docs-content/diagrams/join-strategy.dark.svg +1 -0
  73. package/vendor-core/docs-content/diagrams/join-strategy.svg +1 -0
  74. package/vendor-core/docs-content/diagrams/memory-borrowing.dark.svg +1 -0
  75. package/vendor-core/docs-content/diagrams/memory-borrowing.svg +1 -0
  76. package/vendor-core/docs-content/diagrams/memory-regions.dark.svg +1 -0
  77. package/vendor-core/docs-content/diagrams/memory-regions.svg +1 -0
  78. package/vendor-core/docs-content/diagrams/repartition-vs-coalesce.dark.svg +1 -0
  79. package/vendor-core/docs-content/diagrams/repartition-vs-coalesce.svg +1 -0
  80. package/vendor-core/docs-content/diagrams/retry-escalation-ladder.dark.svg +1 -0
  81. package/vendor-core/docs-content/diagrams/retry-escalation-ladder.svg +1 -0
  82. package/vendor-core/docs-content/diagrams/shuffle-map-reduce.dark.svg +1 -0
  83. package/vendor-core/docs-content/diagrams/shuffle-map-reduce.svg +1 -0
  84. package/vendor-core/docs-content/diagrams/spill-classification.dark.svg +1 -0
  85. package/vendor-core/docs-content/diagrams/spill-classification.svg +1 -0
  86. package/vendor-core/docs-content/diagrams/udf-execution-models.dark.svg +1 -0
  87. package/vendor-core/docs-content/diagrams/udf-execution-models.svg +1 -0
  88. package/vendor-core/docs-content/tuning/broadcast-sizing.md +78 -0
  89. package/vendor-core/docs-content/tuning/cold-start.md +81 -0
  90. package/vendor-core/docs-content/tuning/duplicate-plan-subtree.md +45 -0
  91. package/vendor-core/docs-content/tuning/failures.md +124 -0
  92. package/vendor-core/docs-content/tuning/gc.md +110 -0
  93. package/vendor-core/docs-content/tuning/job-failure-rate.md +101 -0
  94. package/vendor-core/docs-content/tuning/memory-utilization.md +58 -0
  95. package/vendor-core/docs-content/tuning/retry-waste.md +90 -0
  96. package/vendor-core/docs-content/tuning/shuffle.md +154 -0
  97. package/vendor-core/docs-content/tuning/skew.md +123 -0
  98. package/vendor-core/docs-content/tuning/slow-host.md +117 -0
  99. package/vendor-core/docs-content/tuning/small-files.md +99 -0
  100. package/vendor-core/docs-content/tuning/spill.md +114 -0
  101. package/vendor-core/docs-content/tuning/straggler.md +103 -0
  102. package/vendor-core/docs-content/tuning/tiny-tasks.md +94 -0
  103. package/vendor-core/docs-content/tuning/utilization.md +90 -0
  104. package/vendor-core/docs-site-config.js +10 -17
  105. package/vendor-core/efficiency-model.js +7 -13
  106. package/vendor-core/etl-phases.js +3 -5
  107. package/vendor-core/event-handlers.js +232 -134
  108. package/vendor-core/event-schemas.js +48 -114
  109. package/vendor-core/evidence-availability.js +5 -10
  110. package/vendor-core/evidence-report.js +73 -123
  111. package/vendor-core/export-data.js +48 -0
  112. package/vendor-core/finding-action-label.js +4 -10
  113. package/vendor-core/finding-filter-predicate.js +3 -7
  114. package/vendor-core/finding-generic-recommendation.js +112 -0
  115. package/vendor-core/finding-names.js +51 -0
  116. package/vendor-core/format-utils.js +112 -38
  117. package/vendor-core/impact-band.js +18 -24
  118. package/vendor-core/impact-estimator.js +38 -74
  119. package/vendor-core/ingest.js +7 -13
  120. package/vendor-core/job-groups.js +3 -6
  121. package/vendor-core/list-runs.js +278 -0
  122. package/vendor-core/load-vendored.js +6 -12
  123. package/vendor-core/log-header-peek.js +81 -0
  124. package/vendor-core/lz4-block.js +4 -6
  125. package/vendor-core/mcp-server-factory.js +38 -8
  126. package/vendor-core/mcp-tools.js +105 -76
  127. package/vendor-core/model-assembler.js +8 -16
  128. package/vendor-core/occupancy.js +5 -9
  129. package/vendor-core/parser-worker.js +20 -29
  130. package/vendor-core/plan-dot.js +2 -5
  131. package/vendor-core/plan-duration-attribution.js +78 -29
  132. package/vendor-core/plan-graph-model.js +126 -69
  133. package/vendor-core/plan-node-detail.js +31 -17
  134. package/vendor-core/plan-summary.js +19 -8
  135. package/vendor-core/recommendation-rollup.js +35 -39
  136. package/vendor-core/redact.js +72 -16
  137. package/vendor-core/rolling-log-reassembly.js +4 -6
  138. package/vendor-core/run-comparison.js +86 -72
  139. package/vendor-core/scaling-sim.js +5 -7
  140. package/vendor-core/session-snapshot.js +1 -1
  141. package/vendor-core/shs-fetch.js +4 -6
  142. package/vendor-core/shs-load.js +9 -13
  143. package/vendor-core/shs-request.js +1 -1
  144. package/vendor-core/stage-quantiles.js +14 -0
  145. package/vendor-core/types.js +78 -18
  146. package/vendor-core/wasted-core-hours.js +7 -12
@@ -88,15 +88,24 @@
88
88
 
89
89
 
90
90
 
91
+
92
+
93
+
94
+
95
+
91
96
 
92
97
 
93
98
 
94
99
 
95
100
 
101
+
102
+
103
+
104
+
96
105
 
97
106
 
98
107
 
99
-
108
+
100
109
 
101
110
 
102
111
 
@@ -111,14 +120,63 @@
111
120
 
112
121
 
113
122
 
123
+
124
+
125
+
126
+
127
+
128
+
129
+
130
+
131
+
132
+
133
+
134
+
135
+
136
+
137
+
138
+
139
+
140
+
141
+
142
+
143
+
144
+
145
+
146
+
147
+
148
+
149
+
150
+
151
+
152
+
114
153
 
154
+
155
+
156
+
157
+
158
+
159
+
115
160
 
161
+
162
+
163
+
164
+
165
+
166
+
167
+
116
168
 
117
169
 
118
170
 
119
171
 
120
172
 
121
173
 
174
+
175
+
176
+
177
+
178
+
179
+
122
180
 
123
181
 
124
182
 
@@ -142,6 +200,11 @@
142
200
 
143
201
 
144
202
 
203
+ /** Whether the plan-graph heat bar shows a node's own wall-time share
204
+ * ('exclusive') or that share plus every descendant's, within the same
205
+ * Exchange-bounded segment ('inclusive'). */
206
+
207
+
145
208
 
146
209
 
147
210
 
@@ -215,29 +278,21 @@
215
278
 
216
279
 
217
280
 
218
-
219
-
220
-
221
-
222
281
 
223
-
224
-
225
-
226
-
227
-
228
-
282
+
283
+
284
+
285
+
286
+
229
287
 
230
288
 
231
289
 
232
290
 
233
-
291
+
234
292
 
235
-
236
-
237
-
238
-
239
-
240
-
293
+
294
+
295
+
241
296
 
242
297
 
243
298
 
@@ -254,6 +309,11 @@
254
309
 
255
310
 
256
311
 
312
+
313
+
314
+
315
+
316
+
257
317
 
258
318
 
259
319
 
@@ -1,7 +1,6 @@
1
- // Wasted core-hours summary (ELV-023). Pure, worker-agnostic reducer over the
2
- // same inputs the memoryUtilization detector's idle-cores math uses
3
- // (src/detectors.js ~608-619): capacity core-time vs. core-time that actually
4
- // ran tasks. No new detector, no new tag: this only feeds a report widget.
1
+ // Wasted core-hours summary. Pure, worker-agnostic reducer over the same inputs
2
+ // the memoryUtilization detector's idle-cores math uses: capacity core-time vs.
3
+ // core-time that actually ran tasks. Feeds a report widget only, no detector.
5
4
 
6
5
  import { computeTotalCores } from './core-count.js';
7
6
 
@@ -29,14 +28,10 @@ const EMPTY = {
29
28
 
30
29
 
31
30
 
32
- // `app` and `runAggregates` are typed nullable (beyond the brief's bare
33
- // signature) because real callers do pass null here (see
34
- // tests/wasted-core-hours.test.js and the WastedCoreHours widget, where
35
- // appModel.app is SparkAppInfo | null before parsing completes); the
36
- // optional-chaining/`!runAggregates` guard below already tolerated this at
37
- // runtime. `resources` is likewise added to app's shape (not in the brief's
38
- // signature) so the pass-through to computeTotalCores type-checks; real app
39
- // objects always carry it alongside startTime/endTime.
31
+ // `app`/`runAggregates` are nullable because real callers pass null (app is
32
+ // SparkAppInfo | null before parsing completes), which the guard below already
33
+ // tolerates. `resources` is on app's shape so the computeTotalCores pass-through
34
+ // type-checks; real app objects always carry it.
40
35
  export function computeWastedCoreHours(
41
36
  app ,
42
37
  executorsAdded = [],