sparkforensics-cli 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (360) hide show
  1. package/README.md +6 -0
  2. package/bin/sparkforensics-analyze.mjs +113 -48
  3. package/export-template/docs/404.html +25 -0
  4. package/export-template/docs/assets/app.CndaAS6v.js +1 -0
  5. package/export-template/docs/assets/aqe-loop.IwQSATHw.svg +1 -0
  6. package/export-template/docs/assets/aqe-loop.dark.DGbaxqJE.svg +1 -0
  7. package/export-template/docs/assets/broadcast-vs-shuffle.Db4WY1XK.svg +1 -0
  8. package/export-template/docs/assets/broadcast-vs-shuffle.dark.C7Bxs0mG.svg +1 -0
  9. package/export-template/docs/assets/cache-lifecycle.dark.B-hS7AgU.svg +1 -0
  10. package/export-template/docs/assets/cache-lifecycle.rEOVYQNU.svg +1 -0
  11. package/export-template/docs/assets/chunks/@localSearchIndexroot.DNY8bVcl.js +1 -0
  12. package/export-template/docs/assets/chunks/VPLocalSearchBox.yJbZbsEo.js +9 -0
  13. package/export-template/docs/assets/chunks/duplicate-plan-subtree.dark.Cdp70QhV.js +1 -0
  14. package/export-template/docs/assets/chunks/framework.DSg0KOwT.js +20 -0
  15. package/export-template/docs/assets/chunks/retry-escalation-ladder.dark.DHipdJgZ.js +1 -0
  16. package/export-template/docs/assets/chunks/theme.Df2VAG9w.js +2 -0
  17. package/export-template/docs/assets/cold-start-timeline.DxC_Sc7w.svg +1 -0
  18. package/export-template/docs/assets/cold-start-timeline.dark.CZ17YcAG.svg +1 -0
  19. package/export-template/docs/assets/columnar-layout.PghGeOEA.svg +1 -0
  20. package/export-template/docs/assets/columnar-layout.dark.BVNlz0ff.svg +1 -0
  21. package/export-template/docs/assets/container-memory.DIO0AnIm.svg +1 -0
  22. package/export-template/docs/assets/container-memory.dark.CP-5zuCl.svg +1 -0
  23. package/export-template/docs/assets/contributor-guide_architecture_board-widgets.md.B-OsL91z.js +1 -0
  24. package/export-template/docs/assets/contributor-guide_architecture_board-widgets.md.B-OsL91z.lean.js +1 -0
  25. package/export-template/docs/assets/contributor-guide_architecture_detector-contract.md.BOeH4d1J.js +1 -0
  26. package/export-template/docs/assets/contributor-guide_architecture_detector-contract.md.BOeH4d1J.lean.js +1 -0
  27. package/export-template/docs/assets/contributor-guide_architecture_drill-down.md.BtPdlM7r.js +1 -0
  28. package/export-template/docs/assets/contributor-guide_architecture_drill-down.md.BtPdlM7r.lean.js +1 -0
  29. package/export-template/docs/assets/contributor-guide_architecture_impact-estimation.md.DYCDPgkh.js +1 -0
  30. package/export-template/docs/assets/contributor-guide_architecture_impact-estimation.md.DYCDPgkh.lean.js +1 -0
  31. package/export-template/docs/assets/contributor-guide_architecture_index.md.3TO9ic6w.js +1 -0
  32. package/export-template/docs/assets/contributor-guide_architecture_index.md.3TO9ic6w.lean.js +1 -0
  33. package/export-template/docs/assets/contributor-guide_architecture_overview.md.CehiRmGn.js +1 -0
  34. package/export-template/docs/assets/contributor-guide_architecture_overview.md.CehiRmGn.lean.js +1 -0
  35. package/export-template/docs/assets/contributor-guide_architecture_state-and-history.md.m3S3UdMk.js +1 -0
  36. package/export-template/docs/assets/contributor-guide_architecture_state-and-history.md.m3S3UdMk.lean.js +1 -0
  37. package/export-template/docs/assets/contributor-guide_architecture_widget-rendering.md.DbqPf2OT.js +1 -0
  38. package/export-template/docs/assets/contributor-guide_architecture_widget-rendering.md.DbqPf2OT.lean.js +1 -0
  39. package/export-template/docs/assets/contributor-guide_architecture_worker-protocol.md.B93qJ_tT.js +6 -0
  40. package/export-template/docs/assets/contributor-guide_architecture_worker-protocol.md.B93qJ_tT.lean.js +1 -0
  41. package/export-template/docs/assets/contributor-guide_contributing.md.CvRsdr6J.js +1 -0
  42. package/export-template/docs/assets/contributor-guide_contributing.md.CvRsdr6J.lean.js +1 -0
  43. package/export-template/docs/assets/contributor-guide_development-setup.md.DvAN_9mK.js +12 -0
  44. package/export-template/docs/assets/contributor-guide_development-setup.md.DvAN_9mK.lean.js +1 -0
  45. package/export-template/docs/assets/contributor-guide_testing.md.6rIKqSyY.js +1 -0
  46. package/export-template/docs/assets/contributor-guide_testing.md.6rIKqSyY.lean.js +1 -0
  47. package/export-template/docs/assets/dag-stages.DSz_S937.svg +1 -0
  48. package/export-template/docs/assets/dag-stages.dark.F72UzxH4.svg +1 -0
  49. package/export-template/docs/assets/driver-executor.D5pQ7YN1.svg +1 -0
  50. package/export-template/docs/assets/driver-executor.dark.BmX9cPvh.svg +1 -0
  51. package/export-template/docs/assets/duplicate-plan-subtree.B4cvN6fj.svg +1 -0
  52. package/export-template/docs/assets/duplicate-plan-subtree.dark.Dw8wS0Ag.svg +1 -0
  53. package/export-template/docs/assets/index.md.CHJVslga.js +1 -0
  54. package/export-template/docs/assets/index.md.CHJVslga.lean.js +1 -0
  55. package/export-template/docs/assets/inter-italic-cyrillic-ext.r48I6akx.woff2 +0 -0
  56. package/export-template/docs/assets/inter-italic-cyrillic.By2_1cv3.woff2 +0 -0
  57. package/export-template/docs/assets/inter-italic-greek-ext.1u6EdAuj.woff2 +0 -0
  58. package/export-template/docs/assets/inter-italic-greek.DJ8dCoTZ.woff2 +0 -0
  59. package/export-template/docs/assets/inter-italic-latin-ext.CN1xVJS-.woff2 +0 -0
  60. package/export-template/docs/assets/inter-italic-latin.C2AdPX0b.woff2 +0 -0
  61. package/export-template/docs/assets/inter-italic-vietnamese.BSbpV94h.woff2 +0 -0
  62. package/export-template/docs/assets/inter-roman-cyrillic-ext.BBPuwvHQ.woff2 +0 -0
  63. package/export-template/docs/assets/inter-roman-cyrillic.C5lxZ8CY.woff2 +0 -0
  64. package/export-template/docs/assets/inter-roman-greek-ext.CqjqNYQ-.woff2 +0 -0
  65. package/export-template/docs/assets/inter-roman-greek.BBVDIX6e.woff2 +0 -0
  66. package/export-template/docs/assets/inter-roman-latin-ext.4ZJIpNVo.woff2 +0 -0
  67. package/export-template/docs/assets/inter-roman-latin.Di8DUHzh.woff2 +0 -0
  68. package/export-template/docs/assets/inter-roman-vietnamese.BjW4sHH5.woff2 +0 -0
  69. package/export-template/docs/assets/join-strategy.C_FvrCEo.svg +1 -0
  70. package/export-template/docs/assets/join-strategy.dark.ChMLnNII.svg +1 -0
  71. package/export-template/docs/assets/memory-borrowing.BqQRJg0u.svg +1 -0
  72. package/export-template/docs/assets/memory-borrowing.dark.Yhh20O9C.svg +1 -0
  73. package/export-template/docs/assets/memory-regions.XHvO7jHG.svg +1 -0
  74. package/export-template/docs/assets/memory-regions.dark.D4TP9_08.svg +1 -0
  75. package/export-template/docs/assets/repartition-vs-coalesce.BovLRrpj.svg +1 -0
  76. package/export-template/docs/assets/repartition-vs-coalesce.dark.BhAczKZQ.svg +1 -0
  77. package/export-template/docs/assets/retry-escalation-ladder.DyTKJJmZ.svg +1 -0
  78. package/export-template/docs/assets/retry-escalation-ladder.dark.BdsabtU3.svg +1 -0
  79. package/export-template/docs/assets/shuffle-map-reduce.KuOEZVmg.svg +1 -0
  80. package/export-template/docs/assets/shuffle-map-reduce.dark.BgQZnFSb.svg +1 -0
  81. package/export-template/docs/assets/spill-classification.BU2euYDO.svg +1 -0
  82. package/export-template/docs/assets/spill-classification.dark.D7i1M40d.svg +1 -0
  83. package/export-template/docs/assets/style.DXOMCXxn.css +1 -0
  84. package/export-template/docs/assets/tuning-reference_anti-patterns.md.Df1YMIHu.js +1 -0
  85. package/export-template/docs/assets/tuning-reference_anti-patterns.md.Df1YMIHu.lean.js +1 -0
  86. package/export-template/docs/assets/tuning-reference_aqe.md.BIsCtLzm.js +1 -0
  87. package/export-template/docs/assets/tuning-reference_aqe.md.BIsCtLzm.lean.js +1 -0
  88. package/export-template/docs/assets/tuning-reference_bottleneck-broadcast-sizing.md.CEstB3Ia.js +1 -0
  89. package/export-template/docs/assets/tuning-reference_bottleneck-broadcast-sizing.md.CEstB3Ia.lean.js +1 -0
  90. package/export-template/docs/assets/tuning-reference_bottleneck-cold-start.md.CEuy-72y.js +7 -0
  91. package/export-template/docs/assets/tuning-reference_bottleneck-cold-start.md.CEuy-72y.lean.js +1 -0
  92. package/export-template/docs/assets/tuning-reference_bottleneck-duplicate-plan-subtree.md.CIohQDfn.js +1 -0
  93. package/export-template/docs/assets/tuning-reference_bottleneck-duplicate-plan-subtree.md.CIohQDfn.lean.js +1 -0
  94. package/export-template/docs/assets/tuning-reference_bottleneck-failures.md.4z5BXGJ2.js +6 -0
  95. package/export-template/docs/assets/tuning-reference_bottleneck-failures.md.4z5BXGJ2.lean.js +1 -0
  96. package/export-template/docs/assets/tuning-reference_bottleneck-gc.md.DSxzZRK7.js +6 -0
  97. package/export-template/docs/assets/tuning-reference_bottleneck-gc.md.DSxzZRK7.lean.js +1 -0
  98. package/export-template/docs/assets/tuning-reference_bottleneck-job-failure-rate.md.BaJl__1W.js +8 -0
  99. package/export-template/docs/assets/tuning-reference_bottleneck-job-failure-rate.md.BaJl__1W.lean.js +1 -0
  100. package/export-template/docs/assets/tuning-reference_bottleneck-memory-utilization.md.DbP-SJZc.js +7 -0
  101. package/export-template/docs/assets/tuning-reference_bottleneck-memory-utilization.md.DbP-SJZc.lean.js +1 -0
  102. package/export-template/docs/assets/tuning-reference_bottleneck-retry-waste.md.D5JMjVOt.js +1 -0
  103. package/export-template/docs/assets/tuning-reference_bottleneck-retry-waste.md.D5JMjVOt.lean.js +1 -0
  104. package/export-template/docs/assets/tuning-reference_bottleneck-shuffle.md.CM-nTmIH.js +12 -0
  105. package/export-template/docs/assets/tuning-reference_bottleneck-shuffle.md.CM-nTmIH.lean.js +1 -0
  106. package/export-template/docs/assets/tuning-reference_bottleneck-skew.md.BdUwiDhn.js +14 -0
  107. package/export-template/docs/assets/tuning-reference_bottleneck-skew.md.BdUwiDhn.lean.js +1 -0
  108. package/export-template/docs/assets/tuning-reference_bottleneck-slow-host.md.BlIo6UDW.js +7 -0
  109. package/export-template/docs/assets/tuning-reference_bottleneck-slow-host.md.BlIo6UDW.lean.js +1 -0
  110. package/export-template/docs/assets/tuning-reference_bottleneck-small-files.md.B8kloyx8.js +5 -0
  111. package/export-template/docs/assets/tuning-reference_bottleneck-small-files.md.B8kloyx8.lean.js +1 -0
  112. package/export-template/docs/assets/tuning-reference_bottleneck-spill.md.PNH7mITt.js +6 -0
  113. package/export-template/docs/assets/tuning-reference_bottleneck-spill.md.PNH7mITt.lean.js +1 -0
  114. package/export-template/docs/assets/tuning-reference_bottleneck-straggler.md.DY36fHN5.js +7 -0
  115. package/export-template/docs/assets/tuning-reference_bottleneck-straggler.md.DY36fHN5.lean.js +1 -0
  116. package/export-template/docs/assets/tuning-reference_bottleneck-tiny-tasks.md.QTV7O8kU.js +8 -0
  117. package/export-template/docs/assets/tuning-reference_bottleneck-tiny-tasks.md.QTV7O8kU.lean.js +1 -0
  118. package/export-template/docs/assets/tuning-reference_bottleneck-utilization.md.DTiueZC3.js +5 -0
  119. package/export-template/docs/assets/tuning-reference_bottleneck-utilization.md.DTiueZC3.lean.js +1 -0
  120. package/export-template/docs/assets/tuning-reference_caching.md.B7aQ8asB.js +1 -0
  121. package/export-template/docs/assets/tuning-reference_caching.md.B7aQ8asB.lean.js +1 -0
  122. package/export-template/docs/assets/tuning-reference_cluster-config.md.ZVmDGsQ3.js +1 -0
  123. package/export-template/docs/assets/tuning-reference_cluster-config.md.ZVmDGsQ3.lean.js +1 -0
  124. package/export-template/docs/assets/tuning-reference_config.md.UvveiWG3.js +1 -0
  125. package/export-template/docs/assets/tuning-reference_config.md.UvveiWG3.lean.js +1 -0
  126. package/export-template/docs/assets/tuning-reference_data-formats.md.bjCAWH3N.js +1 -0
  127. package/export-template/docs/assets/tuning-reference_data-formats.md.bjCAWH3N.lean.js +1 -0
  128. package/export-template/docs/assets/tuning-reference_index.md.BQ_NooMV.js +1 -0
  129. package/export-template/docs/assets/tuning-reference_index.md.BQ_NooMV.lean.js +1 -0
  130. package/export-template/docs/assets/tuning-reference_intro.md.CobD-lGB.js +1 -0
  131. package/export-template/docs/assets/tuning-reference_intro.md.CobD-lGB.lean.js +1 -0
  132. package/export-template/docs/assets/tuning-reference_joins.md.BtKs_CuW.js +1 -0
  133. package/export-template/docs/assets/tuning-reference_joins.md.BtKs_CuW.lean.js +1 -0
  134. package/export-template/docs/assets/tuning-reference_memory-model.md.DhT-n4y3.js +1 -0
  135. package/export-template/docs/assets/tuning-reference_memory-model.md.DhT-n4y3.lean.js +1 -0
  136. package/export-template/docs/assets/tuning-reference_metrics.md.mLOh7Apj.js +1 -0
  137. package/export-template/docs/assets/tuning-reference_metrics.md.mLOh7Apj.lean.js +1 -0
  138. package/export-template/docs/assets/tuning-reference_partitioning.md.q0zKF_8X.js +1 -0
  139. package/export-template/docs/assets/tuning-reference_partitioning.md.q0zKF_8X.lean.js +1 -0
  140. package/export-template/docs/assets/tuning-reference_pyspark.md.DDCfvN9t.js +6 -0
  141. package/export-template/docs/assets/tuning-reference_pyspark.md.DDCfvN9t.lean.js +1 -0
  142. package/export-template/docs/assets/tuning-reference_shuffle.md.BZZ7R4Ix.js +1 -0
  143. package/export-template/docs/assets/tuning-reference_shuffle.md.BZZ7R4Ix.lean.js +1 -0
  144. package/export-template/docs/assets/tuning-reference_spark-architecture.md.Dwzm5avO.js +1 -0
  145. package/export-template/docs/assets/tuning-reference_spark-architecture.md.Dwzm5avO.lean.js +1 -0
  146. package/export-template/docs/assets/tuning-reference_table-formats.md.D6wj-2dX.js +1 -0
  147. package/export-template/docs/assets/tuning-reference_table-formats.md.D6wj-2dX.lean.js +1 -0
  148. package/export-template/docs/assets/udf-execution-models.BUFDICuG.svg +1 -0
  149. package/export-template/docs/assets/udf-execution-models.dark.YTNS6GDq.svg +1 -0
  150. package/export-template/docs/assets/user-guide_alternative-log-retrieval.md.sU3KGarf.js +1 -0
  151. package/export-template/docs/assets/user-guide_alternative-log-retrieval.md.sU3KGarf.lean.js +1 -0
  152. package/export-template/docs/assets/user-guide_getting-started.md.DtEM37MK.js +3 -0
  153. package/export-template/docs/assets/user-guide_getting-started.md.DtEM37MK.lean.js +1 -0
  154. package/export-template/docs/assets/user-guide_mcp-tools.md.C8MiIu7F.js +125 -0
  155. package/export-template/docs/assets/user-guide_mcp-tools.md.C8MiIu7F.lean.js +1 -0
  156. package/export-template/docs/assets/user-guide_run-comparison.md.S0TWWmLY.js +1 -0
  157. package/export-template/docs/assets/user-guide_run-comparison.md.S0TWWmLY.lean.js +1 -0
  158. package/export-template/docs/assets/user-guide_understanding-findings.md.D0R_Y-R2.js +1 -0
  159. package/export-template/docs/assets/user-guide_understanding-findings.md.D0R_Y-R2.lean.js +1 -0
  160. package/export-template/docs/contributor-guide/architecture/board-widgets.html +25 -0
  161. package/export-template/docs/contributor-guide/architecture/detector-contract.html +25 -0
  162. package/export-template/docs/contributor-guide/architecture/drill-down.html +25 -0
  163. package/export-template/docs/contributor-guide/architecture/impact-estimation.html +25 -0
  164. package/export-template/docs/contributor-guide/architecture/index.html +25 -0
  165. package/export-template/docs/contributor-guide/architecture/overview.html +25 -0
  166. package/export-template/docs/contributor-guide/architecture/state-and-history.html +25 -0
  167. package/export-template/docs/contributor-guide/architecture/widget-rendering.html +25 -0
  168. package/export-template/docs/contributor-guide/architecture/worker-protocol.html +30 -0
  169. package/export-template/docs/contributor-guide/contributing.html +25 -0
  170. package/export-template/docs/contributor-guide/development-setup.html +36 -0
  171. package/export-template/docs/contributor-guide/testing.html +25 -0
  172. package/export-template/docs/favicon.svg +4 -0
  173. package/export-template/docs/hashmap.json +1 -0
  174. package/export-template/docs/index.html +25 -0
  175. package/export-template/docs/package.json +1 -0
  176. package/export-template/docs/tuning-reference/anti-patterns.html +25 -0
  177. package/export-template/docs/tuning-reference/aqe.html +25 -0
  178. package/export-template/docs/tuning-reference/bottleneck-broadcast-sizing.html +25 -0
  179. package/export-template/docs/tuning-reference/bottleneck-cold-start.html +31 -0
  180. package/export-template/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html +25 -0
  181. package/export-template/docs/tuning-reference/bottleneck-failures.html +30 -0
  182. package/export-template/docs/tuning-reference/bottleneck-gc.html +30 -0
  183. package/export-template/docs/tuning-reference/bottleneck-job-failure-rate.html +32 -0
  184. package/export-template/docs/tuning-reference/bottleneck-memory-utilization.html +31 -0
  185. package/export-template/docs/tuning-reference/bottleneck-retry-waste.html +25 -0
  186. package/export-template/docs/tuning-reference/bottleneck-shuffle.html +36 -0
  187. package/export-template/docs/tuning-reference/bottleneck-skew.html +38 -0
  188. package/export-template/docs/tuning-reference/bottleneck-slow-host.html +31 -0
  189. package/export-template/docs/tuning-reference/bottleneck-small-files.html +29 -0
  190. package/export-template/docs/tuning-reference/bottleneck-spill.html +30 -0
  191. package/export-template/docs/tuning-reference/bottleneck-straggler.html +31 -0
  192. package/export-template/docs/tuning-reference/bottleneck-tiny-tasks.html +32 -0
  193. package/export-template/docs/tuning-reference/bottleneck-utilization.html +29 -0
  194. package/export-template/docs/tuning-reference/caching.html +25 -0
  195. package/export-template/docs/tuning-reference/cluster-config.html +25 -0
  196. package/export-template/docs/tuning-reference/config.html +25 -0
  197. package/export-template/docs/tuning-reference/data-formats.html +25 -0
  198. package/export-template/docs/tuning-reference/index.html +25 -0
  199. package/export-template/docs/tuning-reference/intro.html +25 -0
  200. package/export-template/docs/tuning-reference/joins.html +25 -0
  201. package/export-template/docs/tuning-reference/memory-model.html +25 -0
  202. package/export-template/docs/tuning-reference/metrics.html +25 -0
  203. package/export-template/docs/tuning-reference/partitioning.html +25 -0
  204. package/export-template/docs/tuning-reference/pyspark.html +30 -0
  205. package/export-template/docs/tuning-reference/shuffle.html +25 -0
  206. package/export-template/docs/tuning-reference/spark-architecture.html +25 -0
  207. package/export-template/docs/tuning-reference/table-formats.html +25 -0
  208. package/export-template/docs/user-guide/alternative-log-retrieval.html +25 -0
  209. package/export-template/docs/user-guide/getting-started.html +27 -0
  210. package/export-template/docs/user-guide/mcp-tools.html +149 -0
  211. package/export-template/docs/user-guide/run-comparison.html +25 -0
  212. package/export-template/docs/user-guide/understanding-findings.html +25 -0
  213. package/export-template/docs/vp-icons.css +0 -0
  214. package/export-template/favicon.svg +4 -0
  215. package/export-template/index.html +115 -0
  216. package/export-template/parser-worker-QqyEE4m9.js +64 -0
  217. package/package.json +16 -3
  218. package/vendor-core/analyzer.js +74 -74
  219. package/vendor-core/cli/budgets.js +13 -27
  220. package/vendor-core/cli/collect-run.js +43 -19
  221. package/vendor-core/core-count.js +25 -27
  222. package/vendor-core/core-locality-ratio.js +4 -11
  223. package/vendor-core/core-time-series.js +6 -12
  224. package/vendor-core/core-usage-locality.js +3 -4
  225. package/vendor-core/detectors.js +256 -375
  226. package/vendor-core/docs-config.js +69 -21
  227. package/vendor-core/docs-content/chapters/01-intro.md +32 -0
  228. package/vendor-core/docs-content/chapters/02-spark-architecture.md +76 -0
  229. package/vendor-core/docs-content/chapters/03-memory-model.md +73 -0
  230. package/vendor-core/docs-content/chapters/04-partitioning.md +65 -0
  231. package/vendor-core/docs-content/chapters/05-joins.md +62 -0
  232. package/vendor-core/docs-content/chapters/06-shuffle.md +59 -0
  233. package/vendor-core/docs-content/chapters/07-data-formats.md +81 -0
  234. package/vendor-core/docs-content/chapters/07b-table-formats.md +56 -0
  235. package/vendor-core/docs-content/chapters/08-caching.md +58 -0
  236. package/vendor-core/docs-content/chapters/09-pyspark.md +78 -0
  237. package/vendor-core/docs-content/chapters/10-aqe.md +167 -0
  238. package/vendor-core/docs-content/chapters/11-cluster-config.md +170 -0
  239. package/vendor-core/docs-content/chapters/12-anti-patterns.md +171 -0
  240. package/vendor-core/docs-content/chapters/14-metrics.md +87 -0
  241. package/vendor-core/docs-content/chapters/15-config.md +93 -0
  242. package/vendor-core/docs-content/chapters/nav-index.json +370 -0
  243. package/vendor-core/docs-content/detection/cache.md +6 -0
  244. package/vendor-core/docs-content/detection/cfg.md +15 -0
  245. package/vendor-core/docs-content/detection/chrn.md +7 -0
  246. package/vendor-core/docs-content/detection/cold.md +4 -0
  247. package/vendor-core/docs-content/detection/cstor.md +4 -0
  248. package/vendor-core/docs-content/detection/fail.md +5 -0
  249. package/vendor-core/docs-content/detection/gc.md +4 -0
  250. package/vendor-core/docs-content/detection/host.md +5 -0
  251. package/vendor-core/docs-content/detection/incmp.md +6 -0
  252. package/vendor-core/docs-content/detection/jobs.md +4 -0
  253. package/vendor-core/docs-content/detection/local.md +7 -0
  254. package/vendor-core/docs-content/detection/mem.md +10 -0
  255. package/vendor-core/docs-content/detection/part.md +5 -0
  256. package/vendor-core/docs-content/detection/plan.md +14 -0
  257. package/vendor-core/docs-content/detection/retry.md +4 -0
  258. package/vendor-core/docs-content/detection/sfail.md +5 -0
  259. package/vendor-core/docs-content/detection/shape.md +5 -0
  260. package/vendor-core/docs-content/detection/shfl.md +4 -0
  261. package/vendor-core/docs-content/detection/skew.md +6 -0
  262. package/vendor-core/docs-content/detection/slow.md +6 -0
  263. package/vendor-core/docs-content/detection/spec.md +7 -0
  264. package/vendor-core/docs-content/detection/spill.md +7 -0
  265. package/vendor-core/docs-content/detection/strag.md +5 -0
  266. package/vendor-core/docs-content/detection/tiny.md +4 -0
  267. package/vendor-core/docs-content/detection/util.md +4 -0
  268. package/vendor-core/docs-content/diagrams/aqe-loop.dark.svg +1 -0
  269. package/vendor-core/docs-content/diagrams/aqe-loop.svg +1 -0
  270. package/vendor-core/docs-content/diagrams/broadcast-vs-shuffle.dark.svg +1 -0
  271. package/vendor-core/docs-content/diagrams/broadcast-vs-shuffle.svg +1 -0
  272. package/vendor-core/docs-content/diagrams/cache-lifecycle.dark.svg +1 -0
  273. package/vendor-core/docs-content/diagrams/cache-lifecycle.svg +1 -0
  274. package/vendor-core/docs-content/diagrams/cold-start-timeline.dark.svg +1 -0
  275. package/vendor-core/docs-content/diagrams/cold-start-timeline.svg +1 -0
  276. package/vendor-core/docs-content/diagrams/columnar-layout.dark.svg +1 -0
  277. package/vendor-core/docs-content/diagrams/columnar-layout.svg +1 -0
  278. package/vendor-core/docs-content/diagrams/container-memory.dark.svg +1 -0
  279. package/vendor-core/docs-content/diagrams/container-memory.svg +1 -0
  280. package/vendor-core/docs-content/diagrams/dag-stages.dark.svg +1 -0
  281. package/vendor-core/docs-content/diagrams/dag-stages.svg +1 -0
  282. package/vendor-core/docs-content/diagrams/driver-executor.dark.svg +1 -0
  283. package/vendor-core/docs-content/diagrams/driver-executor.svg +1 -0
  284. package/vendor-core/docs-content/diagrams/duplicate-plan-subtree.dark.svg +1 -0
  285. package/vendor-core/docs-content/diagrams/duplicate-plan-subtree.svg +1 -0
  286. package/vendor-core/docs-content/diagrams/join-strategy.dark.svg +1 -0
  287. package/vendor-core/docs-content/diagrams/join-strategy.svg +1 -0
  288. package/vendor-core/docs-content/diagrams/memory-borrowing.dark.svg +1 -0
  289. package/vendor-core/docs-content/diagrams/memory-borrowing.svg +1 -0
  290. package/vendor-core/docs-content/diagrams/memory-regions.dark.svg +1 -0
  291. package/vendor-core/docs-content/diagrams/memory-regions.svg +1 -0
  292. package/vendor-core/docs-content/diagrams/repartition-vs-coalesce.dark.svg +1 -0
  293. package/vendor-core/docs-content/diagrams/repartition-vs-coalesce.svg +1 -0
  294. package/vendor-core/docs-content/diagrams/retry-escalation-ladder.dark.svg +1 -0
  295. package/vendor-core/docs-content/diagrams/retry-escalation-ladder.svg +1 -0
  296. package/vendor-core/docs-content/diagrams/shuffle-map-reduce.dark.svg +1 -0
  297. package/vendor-core/docs-content/diagrams/shuffle-map-reduce.svg +1 -0
  298. package/vendor-core/docs-content/diagrams/spill-classification.dark.svg +1 -0
  299. package/vendor-core/docs-content/diagrams/spill-classification.svg +1 -0
  300. package/vendor-core/docs-content/diagrams/udf-execution-models.dark.svg +1 -0
  301. package/vendor-core/docs-content/diagrams/udf-execution-models.svg +1 -0
  302. package/vendor-core/docs-content/tuning/broadcast-sizing.md +78 -0
  303. package/vendor-core/docs-content/tuning/cold-start.md +81 -0
  304. package/vendor-core/docs-content/tuning/duplicate-plan-subtree.md +45 -0
  305. package/vendor-core/docs-content/tuning/failures.md +124 -0
  306. package/vendor-core/docs-content/tuning/gc.md +110 -0
  307. package/vendor-core/docs-content/tuning/job-failure-rate.md +101 -0
  308. package/vendor-core/docs-content/tuning/memory-utilization.md +58 -0
  309. package/vendor-core/docs-content/tuning/retry-waste.md +90 -0
  310. package/vendor-core/docs-content/tuning/shuffle.md +154 -0
  311. package/vendor-core/docs-content/tuning/skew.md +123 -0
  312. package/vendor-core/docs-content/tuning/slow-host.md +117 -0
  313. package/vendor-core/docs-content/tuning/small-files.md +99 -0
  314. package/vendor-core/docs-content/tuning/spill.md +114 -0
  315. package/vendor-core/docs-content/tuning/straggler.md +103 -0
  316. package/vendor-core/docs-content/tuning/tiny-tasks.md +94 -0
  317. package/vendor-core/docs-content/tuning/utilization.md +90 -0
  318. package/vendor-core/docs-site-config.js +10 -17
  319. package/vendor-core/efficiency-model.js +7 -13
  320. package/vendor-core/etl-phases.js +3 -5
  321. package/vendor-core/event-handlers.js +232 -134
  322. package/vendor-core/event-schemas.js +48 -114
  323. package/vendor-core/evidence-availability.js +5 -10
  324. package/vendor-core/evidence-report.js +72 -122
  325. package/vendor-core/export-data.js +48 -0
  326. package/vendor-core/finding-action-label.js +4 -10
  327. package/vendor-core/finding-filter-predicate.js +3 -7
  328. package/vendor-core/finding-generic-recommendation.js +112 -0
  329. package/vendor-core/finding-names.js +51 -0
  330. package/vendor-core/format-utils.js +112 -38
  331. package/vendor-core/impact-band.js +18 -24
  332. package/vendor-core/impact-estimator.js +38 -74
  333. package/vendor-core/ingest.js +7 -13
  334. package/vendor-core/job-groups.js +3 -6
  335. package/vendor-core/list-runs.js +278 -0
  336. package/vendor-core/load-vendored.js +6 -12
  337. package/vendor-core/log-header-peek.js +81 -0
  338. package/vendor-core/lz4-block.js +4 -6
  339. package/vendor-core/mcp-server-factory.js +38 -8
  340. package/vendor-core/mcp-tools.js +105 -76
  341. package/vendor-core/model-assembler.js +8 -16
  342. package/vendor-core/occupancy.js +5 -9
  343. package/vendor-core/parser-worker.js +18 -27
  344. package/vendor-core/plan-dot.js +2 -5
  345. package/vendor-core/plan-duration-attribution.js +78 -29
  346. package/vendor-core/plan-graph-model.js +126 -69
  347. package/vendor-core/plan-node-detail.js +31 -17
  348. package/vendor-core/plan-summary.js +19 -8
  349. package/vendor-core/recommendation-rollup.js +35 -39
  350. package/vendor-core/redact.js +72 -16
  351. package/vendor-core/rolling-log-reassembly.js +4 -6
  352. package/vendor-core/run-comparison.js +65 -70
  353. package/vendor-core/scaling-sim.js +5 -7
  354. package/vendor-core/session-snapshot.js +1 -1
  355. package/vendor-core/shs-fetch.js +4 -6
  356. package/vendor-core/shs-load.js +9 -13
  357. package/vendor-core/shs-request.js +1 -1
  358. package/vendor-core/stage-quantiles.js +14 -0
  359. package/vendor-core/types.js +78 -18
  360. package/vendor-core/wasted-core-hours.js +7 -12
@@ -1,26 +1,24 @@
1
1
  // Flattens a resolved planTree into a { nodes, edges } graph shape for the
2
2
  // React Flow + Dagre plan graph view ("Plan graph view" in
3
- // docs-site/contributor-guide/architecture/drill-down.md). Every Exchange
4
- // node is split into paired write/read halves: the
5
- // write half stays with its children's producer segment and the read half
6
- // stays with its parent's consumer segment. A duration share stays on the
7
- // half in the original source node's attributed segment. Both halves share a
8
- // `sourceNodeId` back-reference to the pre-split node's id, so any aggregate
9
- // logic over nodes never double-counts one Exchange.
3
+ // docs-site/contributor-guide/architecture/drill-down.md). The Exchange
4
+ // write/read split is done upstream now, in resolvePlanTree
5
+ // (event-handlers.ts): this file just walks the already-split tree and maps
6
+ // each real PlanNode onto a PlanGraphNodeData. sourceNodeId still lets a
7
+ // write half point back to its read half's id, so aggregate logic over
8
+ // nodes can group a pair without special-casing exchangeRole everywhere.
10
9
  import { walkPlanTree } from './plan-tree-walk.js';
11
10
  import {
12
11
  classifyNode,
13
12
  buildDurationMap,
14
13
  parseOperatorDetail,
15
14
  getPrimaryMetric,
15
+ formatPlanMetricValue,
16
16
  } from './plan-node-detail.js';
17
17
  import { computeSegments, mapSegmentsToStagesForDisplay } from './plan-duration-attribution.js';
18
18
  import { stageIdsForSqlExec } from './detectors.js';
19
+
19
20
 
20
-
21
- function isExchangeNode(node ) {
22
- return classifyNode(node.name) === 'exchange';
23
- }
21
+
24
22
 
25
23
  function makeGraphNode({
26
24
  id,
@@ -30,6 +28,8 @@ function makeGraphNode({
30
28
  segmentIndex,
31
29
  splitRole,
32
30
  durationShare,
31
+ exclusiveDurationShare,
32
+ findings,
33
33
  }
34
34
 
35
35
 
@@ -38,6 +38,8 @@ function makeGraphNode({
38
38
 
39
39
 
40
40
 
41
+
42
+
41
43
  ) {
42
44
  return {
43
45
  id,
@@ -49,21 +51,29 @@ function makeGraphNode({
49
51
  segmentIndex,
50
52
  splitRole,
51
53
  durationShare: durationShare ?? null,
54
+ exclusiveDurationShare: exclusiveDurationShare ?? null,
55
+ findings,
56
+ metrics: (planNode.metrics ?? []).map((m) => ({ name: m.name, value: formatPlanMetricValue(m) })),
57
+ // Drop a detail that only repeats the operator name (no extra information
58
+ // for the card); parseOperatorDetail already treats that case as empty.
59
+ detailText: planNode.detail && planNode.detail !== planNode.name ? planNode.detail : '',
52
60
  };
53
61
  }
54
62
 
55
63
  export function buildPlanGraphModel(
56
64
  planTree ,
57
- opts ,
65
+ opts
66
+
67
+
68
+
69
+
70
+
71
+ ,
58
72
  ) {
59
- const { scope, stageId, appModel } = opts;
73
+ const { scope, stageId, appModel, findings = [], durationMode = 'exclusive' } = opts;
60
74
  if (!planTree) return { nodes: [], edges: [], segmentIndex: null, segmentCount: 0, scope: 'full', segmentStageIds: new Map () };
61
75
 
62
76
  const stage = appModel.stages.get(stageId);
63
- // Coalesce undefined -> null so `sqlExecutionId` has a stable `number | null`
64
- // type: every branch below narrows off the same variable instead of
65
- // re-deriving it from `stage`, which TS can't re-narrow after the `stage?.`
66
- // optional-chain expression above collapses to a plain boolean.
67
77
  const sqlExecutionId = stage?.sqlExecutionId ?? null;
68
78
  const sqlExec = sqlExecutionId != null ? appModel.sql.get(sqlExecutionId) : null;
69
79
 
@@ -71,20 +81,21 @@ export function buildPlanGraphModel(
71
81
  const segmentCount = segments.filter(Boolean).length;
72
82
 
73
83
  const durationMap =
74
- sqlExec && sqlExecutionId != null ? buildDurationMap(planTree, appModel, sqlExec, sqlExecutionId) : null;
75
-
76
- // Segment -> stage-id association, computed once up front regardless of
77
- // `scope`: 'segment' needs it below to resolve which segment `stageId`
78
- // belongs to, and 'full' needs the same map to label each segment's
79
- // compound-group header (Task 11) with its stage id; it must not be
80
- // computed only inside the 'segment' branch, or the full-plan view would
81
- // have no stage-id data for its group headers. mapSegmentsToStagesForDisplay
82
- // fills in segments that lost the strict zip's Math.min(segments, stages)
83
- // pairing slot from their nearest paired neighbor, so a group box only
84
- // falls back to an unexplained "Stage —" when there's truly no stage to
85
- // borrow from (see its own doc comment); the earlier `targetSegmentIndex`
86
- // lookup below stays correct either way, since a strictly-paired segment's
87
- // canonical entry is always inserted before any inherited duplicate.
84
+ sqlExec && sqlExecutionId != null ? buildDurationMap(planTree, appModel, sqlExec, sqlExecutionId, durationMode) : null;
85
+ // A second, always-exclusive map, independent of the requested `durationMode`.
86
+ // Exclusive shares are a true, non-overlapping partition of each paired
87
+ // stage's wall time (computeExclusiveSharesForPairs), so their sum is a
88
+ // stable normalization total. PlanGraphCanvas needs this alongside
89
+ // `durationMap` (the mode-selected values actually shown) because summing
90
+ // `durationMap` itself would double/triple-count under 'inclusive' mode,
91
+ // where an ancestor's share already folds in every descendant's share.
92
+ // durationMode === 'exclusive' already computed the same map above; only
93
+ // redo the work when the requested mode actually differs.
94
+ const exclusiveDurationMap =
95
+ durationMode === 'exclusive'
96
+ ? durationMap
97
+ : sqlExec && sqlExecutionId != null ? buildDurationMap(planTree, appModel, sqlExec, sqlExecutionId, 'exclusive') : null;
98
+
88
99
  const segmentStageIds = new Map ();
89
100
  const linkedStageIds =
90
101
  sqlExec && sqlExecutionId != null ? stageIdsForSqlExec(sqlExecutionId, appModel.stages) : [];
@@ -99,50 +110,99 @@ export function buildPlanGraphModel(
99
110
  }
100
111
  }
101
112
 
102
- // A full walk always happens first (cheap: one tree pass), splitting
103
- // every Exchange and tagging each half's own segmentIndex correctly.
104
- // Scope filtering is a post-pass over this output, since a split node's
105
- // two halves can land in two different segments (see module doc comment)
106
- // and a pre-split per-original-node filter can't express that.
107
- const idOf = new WeakMap ();
108
- let counter = 0;
113
+ // resolvePlanTree (event-handlers.ts) now prefixes every id with its owning
114
+ // execution, but this filter stays as defense in depth: a finding is scoped
115
+ // to the execution whose graph is being built before it's indexed by node
116
+ // id, so a stale/synthetic finding whose planNodeIds happen to collide with
117
+ // this tree's ids (e.g. hand-built test fixtures, or data from before the
118
+ // prefix existed) still can't badge onto the wrong tree. No linked SQL
119
+ // execution means no findings attach, matching the sqlExec-falsy behavior
120
+ // used elsewhere in this function (durationMap, linkedStageIds).
121
+ const findingsByNodeId = new Map ();
122
+ if (sqlExecutionId != null) {
123
+ for (const finding of findings) {
124
+ if (finding.executionId !== sqlExecutionId) continue;
125
+ for (const nodeId of finding.planNodeIds ?? []) {
126
+ const list = findingsByNodeId.get(nodeId);
127
+ if (list) list.push(finding);
128
+ else findingsByNodeId.set(nodeId, [finding]);
129
+ }
130
+ }
131
+ }
132
+
109
133
  const allNodes = [];
110
134
  const allEdges = [];
111
135
 
136
+ // Both halves of one split Exchange share a sourceNodeId (the read half's id:
137
+ // the read half points at itself, the write half at its read parent). Collect
138
+ // them under that key during the walk so the post-pass below can cross-link
139
+ // the pair and mirror the shuffle-boundary bytes onto both halves, even
140
+ // though only one half is visible from any single-segment view.
141
+ const splitPairs = new Map ();
142
+
112
143
  walkPlanTree(planTree, (node, parent) => {
113
- const id = `n${counter++}`;
144
+ // resolvePlanTree always sets id; safe downstream of it.
145
+ const id = node.id ;
114
146
  const category = classifyNode(node.name);
115
- // computeSegments (above) already walked this same planTree with the
116
- // same `{ dedupe: true }` traversal, so every node this walk visits is
117
- // guaranteed to already have a segOf entry; the `!` reflects that
118
- // invariant, not an actual nullable field.
119
147
  const seg = segOf.get(node) ;
120
- const parentId = parent ? idOf.get(parent) : null;
121
-
122
- if (isExchangeNode(node)) {
123
- const readId = `${id}-read`;
124
- const writeId = `${id}-write`;
125
- const writeSegment = node.children?.length ? segOf.get(node.children[0]) : seg;
126
- allNodes.push(makeGraphNode({
127
- id: readId, sourceNodeId: id, planNode: node, category, segmentIndex: seg, splitRole: 'read',
128
- durationShare: durationMap?.get(node),
129
- }));
130
- allNodes.push(makeGraphNode({
131
- id: writeId, sourceNodeId: id, planNode: node, category, segmentIndex: writeSegment, splitRole: 'write',
132
- }));
133
- allEdges.push({ id: `${readId}=>${writeId}`, source: readId, target: writeId });
134
- if (parentId) allEdges.push({ id: `${parentId}->${readId}`, source: parentId, target: readId });
135
- idOf.set(node, writeId);
136
- } else {
137
- allNodes.push(makeGraphNode({
138
- id, sourceNodeId: id, planNode: node, category, segmentIndex: seg, splitRole: null,
139
- durationShare: durationMap?.get(node),
140
- }));
141
- if (parentId) allEdges.push({ id: `${parentId}->${id}`, source: parentId, target: id });
142
- idOf.set(node, id);
148
+ const parentId = parent ? parent.id : null;
149
+ const sourceNodeId = node.exchangeRole === 'write' && parent ? parent.id : id;
150
+
151
+ const graphNode = makeGraphNode({
152
+ id,
153
+ sourceNodeId,
154
+ planNode: node,
155
+ category,
156
+ segmentIndex: seg,
157
+ splitRole: node.exchangeRole ?? null,
158
+ durationShare: durationMap?.get(node),
159
+ exclusiveDurationShare: exclusiveDurationMap?.get(node),
160
+ findings: findingsByNodeId.get(id) ?? [],
161
+ });
162
+ allNodes.push(graphNode);
163
+ if (node.exchangeRole) {
164
+ const entry = splitPairs.get(sourceNodeId) ?? {};
165
+ entry[node.exchangeRole] = graphNode;
166
+ splitPairs.set(sourceNodeId, entry);
167
+ }
168
+ if (parentId) {
169
+ const edge = { id: `${parentId}->${id}`, source: parentId, target: id };
170
+ // The read->write pairing edge is the shuffle boundary: weight it by the
171
+ // bytes written across it. The write half lives in the producer segment,
172
+ // and its stage's shuffleWriteBytes is exactly the data materialized to
173
+ // this shuffle, falling back to shuffleReadBytes when the write side
174
+ // wasn't captured. Zero/absent on both means no weight, so a broadcast
175
+ // (which moves little by design) or an empty exchange stays a plain edge.
176
+ if (node.exchangeRole === 'write') {
177
+ const producingStageId = segmentStageIds.get(seg);
178
+ const producingStage = producingStageId != null ? appModel.stages.get(producingStageId) : undefined;
179
+ const shuffleBytes = producingStage?.shuffleWriteBytes || producingStage?.shuffleReadBytes;
180
+ if (shuffleBytes) {
181
+ edge.shuffleBytes = shuffleBytes;
182
+ const entry = splitPairs.get(sourceNodeId) ?? {};
183
+ entry.shuffleBytes = shuffleBytes;
184
+ splitPairs.set(sourceNodeId, entry);
185
+ }
186
+ }
187
+ allEdges.push(edge);
143
188
  }
144
189
  }, { dedupe: true });
145
190
 
191
+ // Cross-link each Exchange pair and mirror the shuffle-boundary bytes onto
192
+ // both halves, so the detail card can reach the partner and show the volume
193
+ // whichever half is open. Runs over the full node set before any scope
194
+ // filtering, so pairedNodeId stays set even when the partner is scoped out.
195
+ for (const { read, write, shuffleBytes } of splitPairs.values()) {
196
+ if (read && write) {
197
+ read.pairedNodeId = write.id;
198
+ write.pairedNodeId = read.id;
199
+ }
200
+ if (shuffleBytes != null) {
201
+ if (read) read.exchangeShuffleBytes = shuffleBytes;
202
+ if (write) write.exchangeShuffleBytes = shuffleBytes;
203
+ }
204
+ }
205
+
146
206
  if (scope === 'full') {
147
207
  return { nodes: allNodes, edges: allEdges, segmentIndex: null, segmentCount, scope: 'full', segmentStageIds };
148
208
  }
@@ -153,9 +213,6 @@ export function buildPlanGraphModel(
153
213
  }
154
214
 
155
215
  if (targetSegmentIndex === null) {
156
- // Segment lookup failed (no sql/stage linkage, or this stage's segment
157
- // fell outside the Math.min(segments, stages) truncation), fall back
158
- // to full-plan scope rather than showing nothing.
159
216
  return { nodes: allNodes, edges: allEdges, segmentIndex: null, segmentCount, scope: 'full', segmentStageIds };
160
217
  }
161
218
 
@@ -1,13 +1,12 @@
1
1
  // Per-operator plan-node rendering helpers for the stage-detail modal's
2
- // hierarchical plan tree. Ported near-verbatim from the legacy canvas widget
3
- // (src/widgets/stage-detail.js, deleted at 1db5365): pure string-parsing /
4
- // tree-walking logic, no DOM. Per AGENTS.md, plan-text parsing is
5
- // best-effort: unparseable fragments are silently skipped, never thrown.
2
+ // hierarchical plan tree. Pure string-parsing / tree-walking, no DOM.
3
+ // Plan-text parsing is best-effort: unparseable fragments are silently
4
+ // skipped, never thrown.
6
5
 
7
6
  import { pathBasename, formatBytes, formatDuration } from './format-utils.js';
8
- import { attributeStageDurationToPlan } from './plan-duration-attribution.js';
7
+ import { attributeStageDurationToPlan, attributeStageDurationToPlanInclusive } from './plan-duration-attribution.js';
9
8
  import { stageIdsForSqlExec } from './detectors.js';
10
-
9
+
11
10
 
12
11
  const BOILERPLATE_PREFIXES = ['serializefromobject', 'deserializetoobject', 'mapelements',
13
12
  'mappartitions', 'inputadapter', 'columnartorow', 'rowtocolumnar', 'union'];
@@ -27,6 +26,14 @@ export function classifyNode(
27
26
  return 'transform';
28
27
  }
29
28
 
29
+ export function isExchangeNode(node ) {
30
+ return node.exchangeRole !== undefined || classifyNode(node.name) === 'exchange';
31
+ }
32
+
33
+ export function isBroadcastExchangeNode(name ) {
34
+ return name === 'BroadcastExchange';
35
+ }
36
+
30
37
  export function parseOperatorDetail(name , simpleString ) {
31
38
  if (!simpleString || simpleString === name) return '';
32
39
  const n = (name ?? '').toLowerCase();
@@ -112,11 +119,16 @@ export function parseOperatorDetail(name , simpleString )
112
119
  return fmtM ? fmtM[1] : '';
113
120
  }
114
121
 
115
- const stripped = stripAliases(simpleString);
122
+ // Some operators (e.g. InMemoryTableScan) have no dedicated branch above, so
123
+ // their simpleString falls through here unparsed and repeats the operator
124
+ // name verbatim as a prefix ("InMemoryTableScan [DATE#1, COUNTRY#2, ...]"),
125
+ // which would otherwise duplicate the bold node name shown right next to it.
126
+ const escapedName = name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
127
+ const stripped = stripAliases(simpleString).replace(new RegExp(`^${escapedName}\\s*`, 'i'), '').trim();
116
128
  return stripped.length > 80 ? stripped.slice(0, 80) + '…' : stripped;
117
129
  }
118
130
 
119
- export function formatMetricValue(metric ) {
131
+ export function formatPlanMetricValue(metric ) {
120
132
  const { value, metricType } = metric;
121
133
  if (typeof value === 'string') return value;
122
134
  if (metricType === 'size') return formatBytes(value);
@@ -131,17 +143,18 @@ export function getPrimaryMetric(
131
143
  if (!metrics || metrics.length === 0) return '';
132
144
  const priority = ['number of output rows', 'size of files read', 'spill size', 'duration'];
133
145
  const found = priority.map(n => metrics.find(m => m.name === n)).find(Boolean) ?? metrics[0];
134
- return formatMetricValue(found);
146
+ return formatPlanMetricValue(found);
135
147
  }
136
148
 
137
- // Approximate per-node wall-time (dataflint-style segment attribution): see
138
- // attributeStageDurationToPlan. Returns null when there's nothing to attribute
139
- // (no plan tree, or the SQL execution doesn't carry a stageIds list).
149
+ // Approximate per-node wall-time via segment attribution (see
150
+ // attributeStageDurationToPlan). Returns null when there's nothing to attribute
151
+ // (no plan tree, or the SQL execution carries no stageIds list).
140
152
  export function buildDurationMap(
141
153
  planTree ,
142
154
  appModel ,
143
155
  sqlExec ,
144
156
  executionId ,
157
+ durationMode = 'exclusive',
145
158
  ) {
146
159
  if (!planTree || !sqlExec) return null;
147
160
  const stageIds = stageIdsForSqlExec(executionId, appModel.stages);
@@ -151,9 +164,10 @@ export function buildDurationMap(
151
164
  const s = appModel.stages.get(id);
152
165
  if (s) stagesById.set(id, { submittedAt: s.submittedAt, completedAt: s.completedAt });
153
166
  }
154
- // attributeStageDurationToPlan still reads its 3rd arg's own `.stageIds`
155
- // field internally (Task 2 didn't change that signature); override it
156
- // with the derived array rather than the (always-empty, on real data)
157
- // one already on `sqlExec`, without needing to change that function.
158
- return attributeStageDurationToPlan(planTree, stagesById, { ...sqlExec, stageIds });
167
+ // attributeStageDurationToPlan reads its 3rd arg's own `.stageIds` field;
168
+ // override it with the derived array rather than the (always-empty on real
169
+ // data) one already on `sqlExec`.
170
+ return durationMode === 'inclusive'
171
+ ? attributeStageDurationToPlanInclusive(planTree, stagesById, { ...sqlExec, stageIds })
172
+ : attributeStageDurationToPlan(planTree, stagesById, { ...sqlExec, stageIds });
159
173
  }
@@ -1,11 +1,10 @@
1
1
  // Best-effort summary of a resolved plan tree (parser-worker `resolvePlanTree`).
2
- // Per AGENTS.md the plan surface is best-effort: unparseable fragments are
3
- // silently skipped, never surfaced as an error. This replaces the old regex
4
- // `plan-extractor.js`: the single plan representation is the tree.
2
+ // Unparseable fragments are silently skipped, never surfaced as an error.
5
3
 
6
4
 
7
5
  import { pathBasename } from './format-utils.js';
8
6
  import { walkPlanTree } from './plan-tree-walk.js';
7
+ import { isExchangeNode } from './plan-node-detail.js';
9
8
 
10
9
 
11
10
 
@@ -70,8 +69,8 @@ export function scanRelationId(name , detail ) {
70
69
  // 2. Internal Delta metadata (the _delta_log state scan and anonymous
71
70
  // commit/checkpoint reads) → null. Location: is often truncated to `Del...`,
72
71
  // so also key off DeltaLogFileIndex or the Delta-log action column signature.
73
- // ponytail: column-signature heuristic; a future Spark that renames those
74
- // action columns would leak a noise row; acceptable, verified clean on all 9 logs.
72
+ // (Column-signature heuristic: a future Spark renaming those action columns
73
+ // would leak a noise row; acceptable.)
75
74
  if (/_delta_log/.test(name) || /_delta_log/.test(detail) ||
76
75
  /DeltaLogFileIndex/i.test(name + detail) ||
77
76
  /checkpointMetadata#|sidecar#|commitInfo#/.test(name + detail)) {
@@ -181,7 +180,7 @@ export function summarizePlanTree(planTree ) {
181
180
  else if (visitScan(name, detail, result)) { /* handled */ }
182
181
  else if (/Join/i.test(name) && visitJoin(name, detail, result)) { /* handled */ }
183
182
  else if (/Aggregate/i.test(name) && visitAgg(name, detail, result)) { /* handled */ }
184
- else if (/^Exchange/i.test(name)) visitExchange(detail, result);
183
+ else if (n.exchangeRole === 'read') visitExchange(detail, result);
185
184
  else if (/^Filter/i.test(name)) {
186
185
  const condM = detail.match(/^Filter\s+([\s\S]+)/);
187
186
  if (condM && condM[1].trim().length > 1000) pushLongFilterWarning(result, condM[1].trim().length);
@@ -204,18 +203,25 @@ export function describePlanNode(
204
203
  const name = node.name ?? '';
205
204
  const detail = node.detail ?? '';
206
205
  const result = { scans: [], joins: [], aggs: [], exchanges: [], warnings: [] };
206
+ let recognizedType = null;
207
207
 
208
208
  if (/CartesianProduct/i.test(name) || /CartesianProduct/i.test(detail)) {
209
+ recognizedType = 'cartesian';
209
210
  pushCrossJoinWarning(result, 'CartesianProduct');
210
211
  } else if (visitScan(name, detail, result)) {
212
+ recognizedType = 'scan';
211
213
  // handled
212
214
  } else if (/Join/i.test(name) && visitJoin(name, detail, result)) {
215
+ recognizedType = 'join';
213
216
  // handled (a Cross join type may also have pushed a crossJoin warning)
214
217
  } else if (/Aggregate/i.test(name) && visitAgg(name, detail, result)) {
218
+ recognizedType = 'agg';
215
219
  // handled (a partial-aggregation pass pushes nothing into result.aggs)
216
- } else if (/^Exchange/i.test(name)) {
220
+ } else if (isExchangeNode(node)) {
221
+ recognizedType = 'exchange';
217
222
  visitExchange(detail, result);
218
223
  } else if (/^Filter/i.test(name)) {
224
+ recognizedType = 'filter';
219
225
  const condM = detail.match(/^Filter\s+([\s\S]+)/);
220
226
  if (condM && condM[1].trim().length > 1000) pushLongFilterWarning(result, condM[1].trim().length);
221
227
  }
@@ -224,10 +230,15 @@ export function describePlanNode(
224
230
  : result.joins[0] ? 'join'
225
231
  : result.aggs[0] ? 'agg'
226
232
  : result.exchanges[0] ? 'exchange'
233
+ : recognizedType === 'exchange' && node.exchangeRole !== 'write' ? 'exchange' // ReusedExchange recognized as exchange but no pattern match; write halves stay null
227
234
  : null;
228
235
  const fields = result.scans[0] ?? result.joins[0] ?? result.aggs[0] ?? result.exchanges[0] ?? null;
229
236
  const warning = result.warnings[0] ?? null;
230
237
 
231
- if (!kind && !warning) return null;
238
+ // Return null if no recognized type, or if it's agg/filter with no content/warning
239
+ if (!recognizedType) return null;
240
+ if ((recognizedType === 'agg' || recognizedType === 'filter') && !kind && !warning) return null;
241
+
242
+ // For exchange or other types, or if there's content/warning, return the object
232
243
  return { kind, warning, ...(fields ?? {}) };
233
244
  }
@@ -1,10 +1,5 @@
1
- // Explicit .ts extensions (unlike a bundler, plain Node's ESM resolver -
2
- // the runtime bin/sparkforensics-analyze.mjs and src/mcp-tools.ts both run
3
- // under, now that src/evidence-report.ts imports this module too - requires
4
- // the exact specifier, not an extensionless one a bundler would resolve for
5
- // you): this file's prior only consumer, src/view/widgets/FixTheseFirst.tsx,
6
- // went through Vite, which tolerates the extensionless form, so this never
7
- // surfaced until the CLI/MCP path started importing it.
1
+ // Explicit .ts extensions: plain Node's ESM resolver (the runtime CLI/MCP path
2
+ // runs under) requires the exact specifier, unlike a bundler.
8
3
  import { mergeIntervals } from './wall-clock.js';
9
4
  import { worstImpactBand, IMPACT_BAND_ORDER } from './format-utils.js';
10
5
 
@@ -14,13 +9,10 @@ import { worstImpactBand, IMPACT_BAND_ORDER } from './format-utils.js';
14
9
 
15
10
 
16
11
 
17
- /** The cross-finding grouping layer the spec calls for: given every stage
18
- * touched by a group of findings, collect their intervals and merge once
19
- * across the combined set, instead of summing each finding's own
20
- * already-clipped wallClock figure (which double-counts concurrent stages
21
- * across different findings of the same detector type). Reuses the generic
22
- * mergeIntervals primitive (src/wall-clock.ts) rather than widening the
23
- * single-finding-scoped estimateMultiStage/unionMs in src/occupancy.ts. */
12
+ /** Cross-finding grouping: given every stage touched by a group of findings,
13
+ * merge their intervals once across the combined set, instead of summing each
14
+ * finding's own already-clipped wallClock figure (which double-counts
15
+ * concurrent stages across different findings of the same detector type). */
24
16
  export function computeStageUnionMs(
25
17
  stageIds ,
26
18
  stages ,
@@ -124,36 +116,41 @@ export function buildRecommendationRollup(
124
116
  const kindDelta = KIND_ORDER[a.kind] - KIND_ORDER[b.kind];
125
117
  if (kindDelta !== 0) return kindDelta;
126
118
  if (a.kind === 'time' && b.kind === 'time') return b.recoverableMsHigh - a.recoverableMsHigh;
127
- // resource/count groups carry no shared magnitude to rank by (spec section
128
- // 5 forbids comparing incompatible units); break ties by worst impact band
129
- // instead, the same tie-break FixTheseFirst used pre-consolidation.
119
+ // resource/count groups carry no shared magnitude to rank by (comparing
120
+ // incompatible units is forbidden); break ties by worst impact band instead.
130
121
  const aImpactBand = IMPACT_BAND_ORDER[worstImpactBand(a.findings) ?? 'info'] ?? 9;
131
122
  const bImpactBand = IMPACT_BAND_ORDER[worstImpactBand(b.findings) ?? 'info'] ?? 9;
132
123
  return aImpactBand - bImpactBand;
133
124
  });
134
125
  }
135
126
 
136
- /** True when a finding is a candidate for the "fix these first" ranking at
137
- * all. Shared by `src/evidence-report.ts` (CLI/MCP path) and
138
- * `src/view/widgets/FixTheseFirst.tsx` (the dashboard widget) so these two
139
- * hardcoded exclusions can't drift between the two surfaces (they used to be
140
- * duplicated, near-verbatim, in both files). Doesn't check
141
- * `REGISTRY[finding.type] != null` (FixTheseFirst.tsx's own extra check):
142
- * `REGISTRY` lives in a `.tsx` file, not importable from this core module;
143
- * FixTheseFirst.tsx layers that extra check on top of this one instead. */
127
+ /** True when a finding is real evidence of an issue, as opposed to a mere
128
+ * evidence-unavailable caveat. Shared by `isEligible` below and by anything
129
+ * else that decides whether a REGISTRY type has "something to show" (an
130
+ * active widget card vs. a Clean-checks line): memoryUtilization's
131
+ * memoryBand/dataUnavailable variant reports a missing-evidence caveat
132
+ * (spark.eventLog.logStageExecutorMetrics=true not on for this run), not an
133
+ * optimization or a clean bill of health; the exact same fact already lives
134
+ * in the Evidence availability ledger's own `executorMetrics` entry
135
+ * (src/evidence-availability.ts), so it belongs there, not as its own card. */
136
+ export function isRealFinding(finding ) {
137
+ if (finding.type === 'memoryUtilization' && finding.variant === 'memoryBand' && finding.dataUnavailable) return false;
138
+ return true;
139
+ }
140
+
141
+ /** True when a finding is a candidate for the "fix these first" ranking.
142
+ * Shared by evidence-report.ts (CLI/MCP path) and FixTheseFirst.tsx (the
143
+ * dashboard widget) so these hardcoded exclusions can't drift between the two
144
+ * surfaces. Doesn't check `REGISTRY[finding.type] != null`: `REGISTRY` lives
145
+ * in a `.tsx` file, not importable from this core module, so FixTheseFirst.tsx
146
+ * layers that extra check on top of this one. */
144
147
  export function isEligible(finding ) {
145
148
  // incompleteRun is a pipeline-completeness signal, not an addressable fix:
146
- // hardcoded here by type, not read from Detector.fixEffort. That field
147
- // exists only to back its own contract test today (tests/fix-effort.test.js);
148
- // nothing in ranking/exclusion logic reads it.
149
+ // hardcoded here by type, not read from Detector.fixEffort. Unlike
150
+ // `isRealFinding`, this exclusion doesn't apply beyond the rollup: an
151
+ // incompleteRun finding still backs its own ordinary active widget card.
149
152
  if (finding.type === 'incompleteRun') return false;
150
- // memoryUtilization's memoryBand/dataUnavailable variant reports a missing
151
- // -evidence caveat (spark.eventLog.logStageExecutorMetrics=true not on for
152
- // this run), not an optimization: the exact same fact already lives in the
153
- // Evidence availability ledger's own `executorMetrics` entry
154
- // (src/evidence-availability.ts), so it belongs there, not here.
155
- if (finding.type === 'memoryUtilization' && finding.variant === 'memoryBand' && finding.dataUnavailable) return false;
156
- return true;
153
+ return isRealFinding(finding);
157
154
  }
158
155
 
159
156
  /** Ranking tier for a single finding: time-based findings (a real `wallClock`
@@ -186,10 +183,9 @@ export function rankFindings(findings ) {
186
183
  const tierDelta = tierOf(a) - tierOf(b);
187
184
  if (tierDelta !== 0) return tierDelta;
188
185
  if (tierOf(a) === TIER_TIME) {
189
- // Same unit (ms) on both sides: the one comparison that is meaningful.
190
- // .high, matching deriveImpactBand (src/impact-band.ts) and
191
- // triage-target.ts, so this ordering, the badge color, and the "click
192
- // to investigate" target all agree on the same figure.
186
+ // Same unit (ms) on both sides. Rank on .high, matching deriveImpactBand
187
+ // and triage-target.ts, so ordering, badge color, and the "click to
188
+ // investigate" target all agree on the same figure.
193
189
  return b.impactEstimate .wallClock .high - a.impactEstimate .wallClock .high;
194
190
  }
195
191
  return (IMPACT_BAND_ORDER[a.impactBand] ?? 9) - (IMPACT_BAND_ORDER[b.impactBand] ?? 9);