sparkforensics-mcp 0.1.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (146) hide show
  1. package/README.md +6 -0
  2. package/bin/sparkforensics-mcp.mjs +19 -6
  3. package/package.json +18 -4
  4. package/vendor-core/analyzer.js +74 -74
  5. package/vendor-core/cli/budgets.js +13 -27
  6. package/vendor-core/cli/collect-run.js +43 -19
  7. package/vendor-core/core-count.js +25 -27
  8. package/vendor-core/core-locality-ratio.js +4 -11
  9. package/vendor-core/core-time-series.js +6 -12
  10. package/vendor-core/core-usage-locality.js +3 -4
  11. package/vendor-core/detectors.js +395 -389
  12. package/vendor-core/docs-config.js +69 -21
  13. package/vendor-core/docs-content/chapters/01-intro.md +32 -0
  14. package/vendor-core/docs-content/chapters/02-spark-architecture.md +76 -0
  15. package/vendor-core/docs-content/chapters/03-memory-model.md +73 -0
  16. package/vendor-core/docs-content/chapters/04-partitioning.md +65 -0
  17. package/vendor-core/docs-content/chapters/05-joins.md +62 -0
  18. package/vendor-core/docs-content/chapters/06-shuffle.md +59 -0
  19. package/vendor-core/docs-content/chapters/07-data-formats.md +81 -0
  20. package/vendor-core/docs-content/chapters/07b-table-formats.md +56 -0
  21. package/vendor-core/docs-content/chapters/08-caching.md +58 -0
  22. package/vendor-core/docs-content/chapters/09-pyspark.md +78 -0
  23. package/vendor-core/docs-content/chapters/10-aqe.md +167 -0
  24. package/vendor-core/docs-content/chapters/11-cluster-config.md +170 -0
  25. package/vendor-core/docs-content/chapters/12-anti-patterns.md +171 -0
  26. package/vendor-core/docs-content/chapters/14-metrics.md +87 -0
  27. package/vendor-core/docs-content/chapters/15-config.md +93 -0
  28. package/vendor-core/docs-content/chapters/nav-index.json +370 -0
  29. package/vendor-core/docs-content/detection/cache.md +7 -0
  30. package/vendor-core/docs-content/detection/cfg.md +15 -0
  31. package/vendor-core/docs-content/detection/chrn.md +9 -0
  32. package/vendor-core/docs-content/detection/cold.md +4 -0
  33. package/vendor-core/docs-content/detection/cstor.md +4 -0
  34. package/vendor-core/docs-content/detection/fail.md +5 -0
  35. package/vendor-core/docs-content/detection/gc.md +4 -0
  36. package/vendor-core/docs-content/detection/host.md +5 -0
  37. package/vendor-core/docs-content/detection/incmp.md +6 -0
  38. package/vendor-core/docs-content/detection/jobs.md +4 -0
  39. package/vendor-core/docs-content/detection/local.md +6 -0
  40. package/vendor-core/docs-content/detection/mem.md +10 -0
  41. package/vendor-core/docs-content/detection/part.md +5 -0
  42. package/vendor-core/docs-content/detection/plan.md +14 -0
  43. package/vendor-core/docs-content/detection/retry.md +4 -0
  44. package/vendor-core/docs-content/detection/sfail.md +5 -0
  45. package/vendor-core/docs-content/detection/shape.md +5 -0
  46. package/vendor-core/docs-content/detection/shfl.md +4 -0
  47. package/vendor-core/docs-content/detection/skew.md +6 -0
  48. package/vendor-core/docs-content/detection/slow.md +6 -0
  49. package/vendor-core/docs-content/detection/spec.md +8 -0
  50. package/vendor-core/docs-content/detection/spill.md +7 -0
  51. package/vendor-core/docs-content/detection/strag.md +5 -0
  52. package/vendor-core/docs-content/detection/tiny.md +4 -0
  53. package/vendor-core/docs-content/detection/util.md +4 -0
  54. package/vendor-core/docs-content/diagrams/aqe-loop.dark.svg +1 -0
  55. package/vendor-core/docs-content/diagrams/aqe-loop.svg +1 -0
  56. package/vendor-core/docs-content/diagrams/broadcast-vs-shuffle.dark.svg +1 -0
  57. package/vendor-core/docs-content/diagrams/broadcast-vs-shuffle.svg +1 -0
  58. package/vendor-core/docs-content/diagrams/cache-lifecycle.dark.svg +1 -0
  59. package/vendor-core/docs-content/diagrams/cache-lifecycle.svg +1 -0
  60. package/vendor-core/docs-content/diagrams/cold-start-timeline.dark.svg +1 -0
  61. package/vendor-core/docs-content/diagrams/cold-start-timeline.svg +1 -0
  62. package/vendor-core/docs-content/diagrams/columnar-layout.dark.svg +1 -0
  63. package/vendor-core/docs-content/diagrams/columnar-layout.svg +1 -0
  64. package/vendor-core/docs-content/diagrams/container-memory.dark.svg +1 -0
  65. package/vendor-core/docs-content/diagrams/container-memory.svg +1 -0
  66. package/vendor-core/docs-content/diagrams/dag-stages.dark.svg +1 -0
  67. package/vendor-core/docs-content/diagrams/dag-stages.svg +1 -0
  68. package/vendor-core/docs-content/diagrams/driver-executor.dark.svg +1 -0
  69. package/vendor-core/docs-content/diagrams/driver-executor.svg +1 -0
  70. package/vendor-core/docs-content/diagrams/duplicate-plan-subtree.dark.svg +1 -0
  71. package/vendor-core/docs-content/diagrams/duplicate-plan-subtree.svg +1 -0
  72. package/vendor-core/docs-content/diagrams/join-strategy.dark.svg +1 -0
  73. package/vendor-core/docs-content/diagrams/join-strategy.svg +1 -0
  74. package/vendor-core/docs-content/diagrams/memory-borrowing.dark.svg +1 -0
  75. package/vendor-core/docs-content/diagrams/memory-borrowing.svg +1 -0
  76. package/vendor-core/docs-content/diagrams/memory-regions.dark.svg +1 -0
  77. package/vendor-core/docs-content/diagrams/memory-regions.svg +1 -0
  78. package/vendor-core/docs-content/diagrams/repartition-vs-coalesce.dark.svg +1 -0
  79. package/vendor-core/docs-content/diagrams/repartition-vs-coalesce.svg +1 -0
  80. package/vendor-core/docs-content/diagrams/retry-escalation-ladder.dark.svg +1 -0
  81. package/vendor-core/docs-content/diagrams/retry-escalation-ladder.svg +1 -0
  82. package/vendor-core/docs-content/diagrams/shuffle-map-reduce.dark.svg +1 -0
  83. package/vendor-core/docs-content/diagrams/shuffle-map-reduce.svg +1 -0
  84. package/vendor-core/docs-content/diagrams/spill-classification.dark.svg +1 -0
  85. package/vendor-core/docs-content/diagrams/spill-classification.svg +1 -0
  86. package/vendor-core/docs-content/diagrams/udf-execution-models.dark.svg +1 -0
  87. package/vendor-core/docs-content/diagrams/udf-execution-models.svg +1 -0
  88. package/vendor-core/docs-content/tuning/broadcast-sizing.md +78 -0
  89. package/vendor-core/docs-content/tuning/cold-start.md +81 -0
  90. package/vendor-core/docs-content/tuning/duplicate-plan-subtree.md +45 -0
  91. package/vendor-core/docs-content/tuning/failures.md +124 -0
  92. package/vendor-core/docs-content/tuning/gc.md +110 -0
  93. package/vendor-core/docs-content/tuning/job-failure-rate.md +101 -0
  94. package/vendor-core/docs-content/tuning/memory-utilization.md +58 -0
  95. package/vendor-core/docs-content/tuning/retry-waste.md +90 -0
  96. package/vendor-core/docs-content/tuning/shuffle.md +154 -0
  97. package/vendor-core/docs-content/tuning/skew.md +123 -0
  98. package/vendor-core/docs-content/tuning/slow-host.md +117 -0
  99. package/vendor-core/docs-content/tuning/small-files.md +99 -0
  100. package/vendor-core/docs-content/tuning/spill.md +114 -0
  101. package/vendor-core/docs-content/tuning/straggler.md +103 -0
  102. package/vendor-core/docs-content/tuning/tiny-tasks.md +94 -0
  103. package/vendor-core/docs-content/tuning/utilization.md +90 -0
  104. package/vendor-core/docs-site-config.js +10 -17
  105. package/vendor-core/efficiency-model.js +7 -13
  106. package/vendor-core/etl-phases.js +3 -5
  107. package/vendor-core/event-handlers.js +232 -134
  108. package/vendor-core/event-schemas.js +48 -114
  109. package/vendor-core/evidence-availability.js +5 -10
  110. package/vendor-core/evidence-report.js +73 -123
  111. package/vendor-core/export-data.js +48 -0
  112. package/vendor-core/finding-action-label.js +4 -10
  113. package/vendor-core/finding-filter-predicate.js +3 -7
  114. package/vendor-core/finding-generic-recommendation.js +112 -0
  115. package/vendor-core/finding-names.js +51 -0
  116. package/vendor-core/format-utils.js +112 -38
  117. package/vendor-core/impact-band.js +18 -24
  118. package/vendor-core/impact-estimator.js +38 -74
  119. package/vendor-core/ingest.js +7 -13
  120. package/vendor-core/job-groups.js +3 -6
  121. package/vendor-core/list-runs.js +278 -0
  122. package/vendor-core/load-vendored.js +6 -12
  123. package/vendor-core/log-header-peek.js +81 -0
  124. package/vendor-core/lz4-block.js +4 -6
  125. package/vendor-core/mcp-server-factory.js +38 -8
  126. package/vendor-core/mcp-tools.js +105 -76
  127. package/vendor-core/model-assembler.js +8 -16
  128. package/vendor-core/occupancy.js +5 -9
  129. package/vendor-core/parser-worker.js +20 -29
  130. package/vendor-core/plan-dot.js +2 -5
  131. package/vendor-core/plan-duration-attribution.js +78 -29
  132. package/vendor-core/plan-graph-model.js +126 -69
  133. package/vendor-core/plan-node-detail.js +31 -17
  134. package/vendor-core/plan-summary.js +19 -8
  135. package/vendor-core/recommendation-rollup.js +35 -39
  136. package/vendor-core/redact.js +72 -16
  137. package/vendor-core/rolling-log-reassembly.js +4 -6
  138. package/vendor-core/run-comparison.js +86 -72
  139. package/vendor-core/scaling-sim.js +5 -7
  140. package/vendor-core/session-snapshot.js +1 -1
  141. package/vendor-core/shs-fetch.js +4 -6
  142. package/vendor-core/shs-load.js +9 -13
  143. package/vendor-core/shs-request.js +1 -1
  144. package/vendor-core/stage-quantiles.js +14 -0
  145. package/vendor-core/types.js +78 -18
  146. package/vendor-core/wasted-core-hours.js +7 -12
@@ -1,11 +1,7 @@
1
- // Sweep-line "busy cores over time" computation. Pure and side-effect-free so
2
- // it can run inside parser-worker.js (over retained task launch/finish
3
- // timestamps) or on the main thread (scaling simulator). One core per task:
4
- // Spark's default task-to-core mapping.
5
- //
6
- // `hypotheticalCores` clamps the concurrent busy-core count to N (the
7
- // deterministic "utilization at N cores" curve). It does NOT re-schedule tasks;
8
- // makespan re-estimation is the scaling simulator's job, layered on this signal.
1
+ // Sweep-line "busy cores over time". Pure so it runs in parser-worker (over retained task
2
+ // timestamps) or on the main thread (scaling simulator). One core per task: Spark's default.
3
+ // `hypotheticalCores` clamps the concurrent busy-core count to N; it does NOT re-schedule
4
+ // tasks (makespan re-estimation is the scaling simulator's job, layered on this signal).
9
5
 
10
6
 
11
7
 
@@ -19,18 +15,16 @@
19
15
 
20
16
 
21
17
  function buildStepFunction(intervals , hypotheticalCores ) {
22
- // Emit +1 at launch, -1 at finish. Skip empty/negative intervals.
23
18
  const events = [];
24
19
  for (const { launch, finish } of intervals) {
25
20
  if (!(finish > launch)) continue;
26
21
  events.push({ t: launch, delta: 1 });
27
22
  events.push({ t: finish, delta: -1 });
28
23
  }
29
- // Sort by time; at equal time process -1 before +1 so [launch, finish) is
30
- // half-open (a task finishing exactly as another launches does not overlap).
24
+ // At equal time process -1 before +1 so [launch, finish) is half-open (a task finishing
25
+ // exactly as another launches does not overlap).
31
26
  events.sort((a, b) => (a.t - b.t) || (a.delta - b.delta));
32
27
 
33
- // Walk consecutive timestamps, emitting the busy level held on each interval.
34
28
  const segments = [];
35
29
  let busy = 0;
36
30
  for (let i = 0; i < events.length; i++) {
@@ -1,7 +1,6 @@
1
- // Stage-granular approximation of core-usage-by-locality over time. See the
2
- // plan's Task 6 DEVIATION note: per-task locality is not retained, so this
3
- // distributes each stage's executorRunTime across its wall-clock window,
4
- // split by localityStats proportions. Not exact; approximate by construction.
1
+ // Stage-granular approximation of core-usage-by-locality over time. Per-task locality is not
2
+ // retained, so this distributes each stage's executorRunTime across its wall-clock window,
3
+ // split by localityStats proportions. Approximate by construction.
5
4
  export const LOCALITY_TIERS = ['PROCESS_LOCAL', 'NODE_LOCAL', 'RACK_LOCAL', 'NO_PREF', 'ANY'];
6
5
 
7
6