sparkforensics-server 0.2.4 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +5 -1
- package/package.json +1 -1
- package/public/assets/{AutoscalingChurn-BOiUUfuY.js → AutoscalingChurn-lxZHWIwq.js} +1 -1
- package/public/assets/CacheUtilization-DuAJ-Lve.js +1 -0
- package/public/assets/CachingOpportunity-DC1t7o2_.js +1 -0
- package/public/assets/{ColdStart-ZHxPrJmI.js → ColdStart-BOYXeiUZ.js} +1 -1
- package/public/assets/{ConfigAudit-BkVnq2XE.js → ConfigAudit-MCPrJomC.js} +1 -1
- package/public/assets/{CoreUsageArea-DobrSXk2.js → CoreUsageArea-ChNuqGA0.js} +1 -1
- package/public/assets/Dashboard-VuyUGvZx.js +1 -0
- package/public/assets/DuplicatePlanSubtree-JZut9xwq.js +1 -0
- package/public/assets/{DurationHistogram-DlQv4ZCB.js → DurationHistogram-_7C1w0l4.js} +1 -1
- package/public/assets/{EmptyState-DqnSxTjJ.js → EmptyState-D_ulfIIi.js} +1 -1
- package/public/assets/{ExecutorUtilization-CuQUxaCd.js → ExecutorUtilization-B08E76jT.js} +1 -1
- package/public/assets/GcPressure-BZMi2fTx.js +1 -0
- package/public/assets/{ImpactEstimate-Df_laG-n.js → ImpactEstimate-36WxuPdV.js} +1 -1
- package/public/assets/{IncompleteRun-CpIkS4XZ.js → IncompleteRun-t8JU7HRt.js} +1 -1
- package/public/assets/{JobFailures-DKbvYc4J.js → JobFailures-CguB1bec.js} +1 -1
- package/public/assets/{MemoryUtilization-C5kHmXIb.js → MemoryUtilization-DDk8lz9-.js} +1 -1
- package/public/assets/OverBroadcast-BHXLt6s7.js +1 -0
- package/public/assets/PartitionSizing-COj3uwV-.js +1 -0
- package/public/assets/{PlanExplorer-DjGTZifP.js → PlanExplorer-CnCdEY_i.js} +1 -1
- package/public/assets/PlanGraphRoute-BesZOW1n.js +1 -0
- package/public/assets/{RetryWaste-C0znHd_w.js → RetryWaste-CcU4ctAn.js} +1 -1
- package/public/assets/{RowPagination-2icPidMP.js → RowPagination-D2aIcfGv.js} +1 -1
- package/public/assets/{ShuffleIO-L_XSnJ_V.js → ShuffleIO-NPHkWbg1.js} +1 -1
- package/public/assets/{Skew-w4uV5H_g.js → Skew-CzztP-C4.js} +1 -1
- package/public/assets/SlowHost-Dh2KHypN.js +1 -0
- package/public/assets/SmallFiles-CgYSuZQw.js +1 -0
- package/public/assets/{SpeculationWaste-xCKwjz2K.js → SpeculationWaste-C8J5z1WS.js} +1 -1
- package/public/assets/Spill-DwTQPZ_L.js +1 -0
- package/public/assets/{StageFailed-De3vVDbc.js → StageFailed-aFeIEh3p.js} +1 -1
- package/public/assets/{StageFindingGroup-R4OW-g4U.js → StageFindingGroup-DQmAE3uk.js} +1 -1
- package/public/assets/{StageHeader-CGeWqC-0.js → StageHeader-D6GBbcZp.js} +1 -1
- package/public/assets/{StagePill-BwwKLcFl.js → StagePill-CSKyM8Kd.js} +1 -1
- package/public/assets/{StageShape-D4LmPVyV.js → StageShape-J6U4cUcM.js} +1 -1
- package/public/assets/{StageSlowness-8-z6qn4Q.js → StageSlowness-C5F2v_ZO.js} +1 -1
- package/public/assets/Straggler-DcP3dFGA.js +1 -0
- package/public/assets/TaskFailures-C-vIEheo.js +1 -0
- package/public/assets/{TinyTask-DYXks5sT.js → TinyTask-Be-pvTjJ.js} +1 -1
- package/public/assets/UnderBroadcast-DfjYGhoK.js +1 -0
- package/public/assets/{finding-anchor-CDbHtqKT.js → finding-anchor-vVODA_VC.js} +1 -1
- package/public/assets/{inbox-M_EuWNiK.js → inbox-LtbS_kBS.js} +1 -1
- package/public/assets/{index-pnImGbgQ.js → index-27MbqTFr.js} +5 -3
- package/public/assets/index-CWjjbtyo.css +2 -0
- package/public/assets/parser-worker-jAP0kWvS.js +122 -0
- package/public/assets/{select-C-MZGNpb.js → select-CC2Lyrmi.js} +1 -1
- package/public/assets/{toggle-group-C3nz7UQ_.js → toggle-group-lmgO21t9.js} +1 -1
- package/public/assets/useRecentFiles-qH5Ap4xm.js +12 -0
- package/public/assets/{useSortMode-DHlKZeWv.js → useSortMode-CvPY99UV.js} +1 -1
- package/public/docs/404.html +2 -2
- package/public/docs/assets/{app.DsFQcynM.js → app.C5E-wy6e.js} +1 -1
- package/public/docs/assets/chunks/@localSearchIndexroot.CJBoRv1b.js +1 -0
- package/public/docs/assets/chunks/{VPLocalSearchBox.oa8NbaiV.js → VPLocalSearchBox.DtyygpbL.js} +1 -1
- package/public/docs/assets/chunks/{theme.Ca4dR9dL.js → theme.DQS9-w_Z.js} +2 -2
- package/public/docs/assets/{contributor-guide_architecture_detector-contract.md.aQQfP7pj.js → contributor-guide_architecture_detector-contract.md.CsCytCho.js} +1 -1
- package/public/docs/assets/{contributor-guide_architecture_detector-contract.md.aQQfP7pj.lean.js → contributor-guide_architecture_detector-contract.md.CsCytCho.lean.js} +1 -1
- package/public/docs/assets/contributor-guide_architecture_overview.md.rhfxoGsI.js +1 -0
- package/public/docs/assets/contributor-guide_architecture_overview.md.rhfxoGsI.lean.js +1 -0
- package/public/docs/assets/{contributor-guide_architecture_state-and-history.md.C-xxn0q7.js → contributor-guide_architecture_state-and-history.md.B8Wc_ZTg.js} +1 -1
- package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.CBJkwH24.js +6 -0
- package/public/docs/assets/{contributor-guide_architecture_worker-protocol.md.P7d1vtEJ.lean.js → contributor-guide_architecture_worker-protocol.md.CBJkwH24.lean.js} +1 -1
- package/public/docs/assets/contributor-guide_contributing.md.B2E9ni79.js +1 -0
- package/public/docs/assets/contributor-guide_contributing.md.B2E9ni79.lean.js +1 -0
- package/public/docs/assets/contributor-guide_development-setup.md.s85j2l8b.js +15 -0
- package/public/docs/assets/contributor-guide_development-setup.md.s85j2l8b.lean.js +1 -0
- package/public/docs/assets/contributor-guide_testing.md.Cnn6gz1C.js +1 -0
- package/public/docs/assets/contributor-guide_testing.md.Cnn6gz1C.lean.js +1 -0
- package/public/docs/assets/{user-guide_alternative-log-retrieval.md.B4tPGIal.js → user-guide_alternative-log-retrieval.md.ZHPSbBkF.js} +1 -1
- package/public/docs/assets/{user-guide_alternative-log-retrieval.md.B4tPGIal.lean.js → user-guide_alternative-log-retrieval.md.ZHPSbBkF.lean.js} +1 -1
- package/public/docs/assets/user-guide_getting-started.md.Cpe0U-eO.js +3 -0
- package/public/docs/assets/{user-guide_getting-started.md.BJvwLEIM.lean.js → user-guide_getting-started.md.Cpe0U-eO.lean.js} +1 -1
- package/public/docs/assets/{user-guide_mcp-tools.md.DGvolGzt.js → user-guide_mcp-tools.md.DrphooJv.js} +1 -1
- package/public/docs/assets/user-guide_understanding-findings.md.YBBLOwWi.js +1 -0
- package/public/docs/contributor-guide/architecture/board-widgets.html +3 -3
- package/public/docs/contributor-guide/architecture/detector-contract.html +5 -5
- package/public/docs/contributor-guide/architecture/drill-down.html +3 -3
- package/public/docs/contributor-guide/architecture/impact-estimation.html +3 -3
- package/public/docs/contributor-guide/architecture/index.html +3 -3
- package/public/docs/contributor-guide/architecture/overview.html +5 -5
- package/public/docs/contributor-guide/architecture/state-and-history.html +5 -5
- package/public/docs/contributor-guide/architecture/widget-rendering.html +3 -3
- package/public/docs/contributor-guide/architecture/worker-protocol.html +6 -6
- package/public/docs/contributor-guide/contributing.html +5 -5
- package/public/docs/contributor-guide/development-setup.html +11 -8
- package/public/docs/contributor-guide/testing.html +5 -5
- package/public/docs/hashmap.json +1 -1
- package/public/docs/index.html +3 -3
- package/public/docs/tuning-reference/anti-patterns.html +3 -3
- package/public/docs/tuning-reference/aqe.html +3 -3
- package/public/docs/tuning-reference/bottleneck-broadcast-sizing.html +3 -3
- package/public/docs/tuning-reference/bottleneck-cold-start.html +3 -3
- package/public/docs/tuning-reference/bottleneck-duplicate-plan-subtree.html +3 -3
- package/public/docs/tuning-reference/bottleneck-failures.html +3 -3
- package/public/docs/tuning-reference/bottleneck-gc.html +3 -3
- package/public/docs/tuning-reference/bottleneck-job-failure-rate.html +3 -3
- package/public/docs/tuning-reference/bottleneck-memory-utilization.html +3 -3
- package/public/docs/tuning-reference/bottleneck-retry-waste.html +3 -3
- package/public/docs/tuning-reference/bottleneck-shuffle.html +3 -3
- package/public/docs/tuning-reference/bottleneck-skew.html +3 -3
- package/public/docs/tuning-reference/bottleneck-slow-host.html +3 -3
- package/public/docs/tuning-reference/bottleneck-small-files.html +3 -3
- package/public/docs/tuning-reference/bottleneck-spill.html +3 -3
- package/public/docs/tuning-reference/bottleneck-straggler.html +3 -3
- package/public/docs/tuning-reference/bottleneck-tiny-tasks.html +3 -3
- package/public/docs/tuning-reference/bottleneck-utilization.html +3 -3
- package/public/docs/tuning-reference/caching.html +3 -3
- package/public/docs/tuning-reference/cluster-config.html +3 -3
- package/public/docs/tuning-reference/config.html +3 -3
- package/public/docs/tuning-reference/data-formats.html +3 -3
- package/public/docs/tuning-reference/index.html +3 -3
- package/public/docs/tuning-reference/intro.html +3 -3
- package/public/docs/tuning-reference/joins.html +3 -3
- package/public/docs/tuning-reference/memory-model.html +3 -3
- package/public/docs/tuning-reference/metrics.html +3 -3
- package/public/docs/tuning-reference/partitioning.html +3 -3
- package/public/docs/tuning-reference/pyspark.html +3 -3
- package/public/docs/tuning-reference/shuffle.html +3 -3
- package/public/docs/tuning-reference/spark-architecture.html +3 -3
- package/public/docs/tuning-reference/table-formats.html +3 -3
- package/public/docs/user-guide/alternative-log-retrieval.html +5 -5
- package/public/docs/user-guide/getting-started.html +6 -6
- package/public/docs/user-guide/mcp-tools.html +5 -5
- package/public/docs/user-guide/run-comparison.html +3 -3
- package/public/docs/user-guide/understanding-findings.html +5 -5
- package/public/index.html +2 -2
- package/vendor-core/cli/native-zstd.js +2 -2
- package/vendor-core/detectors.js +16 -2
- package/vendor-core/docs-content/detection/fail.md +6 -2
- package/vendor-core/event-handlers.js +21 -0
- package/vendor-core/event-schemas.js +8 -0
- package/vendor-core/evidence-report.js +20 -1
- package/vendor-core/parser-worker.js +21 -4
- package/vendor-core/redact.js +24 -2
- package/vendor-core/run-comparison.js +8 -0
- package/vendor-core/shs-fetch.js +175 -65
- package/vendor-core/shs-load.js +1 -1
- package/vendor-core/stage-quantiles.js +12 -0
- package/vendor-core/task-failure.js +151 -0
- package/vendor-core/vendor/fflate.js +1 -1
- package/vendor-core/zip-archive.js +167 -0
- package/public/assets/CacheUtilization-DwX7lisd.js +0 -1
- package/public/assets/CachingOpportunity-DUE3GKpx.js +0 -1
- package/public/assets/Dashboard-y3_9iQJo.js +0 -1
- package/public/assets/DuplicatePlanSubtree-BSQZFeNq.js +0 -1
- package/public/assets/GcPressure-CLT5hkMJ.js +0 -1
- package/public/assets/OverBroadcast-C5pF66aq.js +0 -1
- package/public/assets/PartitionSizing-BvEJQ5M1.js +0 -1
- package/public/assets/PlanGraphRoute-DoIGA7_H.js +0 -1
- package/public/assets/SlowHost-D5tWJ1UR.js +0 -1
- package/public/assets/SmallFiles-CTtNiSOh.js +0 -1
- package/public/assets/Spill-BT8my24K.js +0 -1
- package/public/assets/Straggler-BIgxsowI.js +0 -1
- package/public/assets/TaskFailures-B9lLt6Rw.js +0 -1
- package/public/assets/UnderBroadcast-BWqWMpeG.js +0 -1
- package/public/assets/index-CjVklAdW.css +0 -2
- package/public/assets/parser-worker-C3BAO6z9.js +0 -117
- package/public/assets/useRecentFiles-0XuhS91q.js +0 -11
- package/public/docs/assets/chunks/@localSearchIndexroot.BEfYag5Q.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_overview.md.BXQdgG5u.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_overview.md.BXQdgG5u.lean.js +0 -1
- package/public/docs/assets/contributor-guide_architecture_worker-protocol.md.P7d1vtEJ.js +0 -6
- package/public/docs/assets/contributor-guide_contributing.md.CvRsdr6J.js +0 -1
- package/public/docs/assets/contributor-guide_contributing.md.CvRsdr6J.lean.js +0 -1
- package/public/docs/assets/contributor-guide_development-setup.md.DvMV1xhQ.js +0 -12
- package/public/docs/assets/contributor-guide_development-setup.md.DvMV1xhQ.lean.js +0 -1
- package/public/docs/assets/contributor-guide_testing.md.BU6vyA-9.js +0 -1
- package/public/docs/assets/contributor-guide_testing.md.BU6vyA-9.lean.js +0 -1
- package/public/docs/assets/user-guide_getting-started.md.BJvwLEIM.js +0 -3
- package/public/docs/assets/user-guide_understanding-findings.md.B29f0ucS.js +0 -1
- /package/public/docs/assets/{contributor-guide_architecture_state-and-history.md.C-xxn0q7.lean.js → contributor-guide_architecture_state-and-history.md.B8Wc_ZTg.lean.js} +0 -0
- /package/public/docs/assets/{user-guide_mcp-tools.md.DGvolGzt.lean.js → user-guide_mcp-tools.md.DrphooJv.lean.js} +0 -0
- /package/public/docs/assets/{user-guide_understanding-findings.md.B29f0ucS.lean.js → user-guide_understanding-findings.md.YBBLOwWi.lean.js} +0 -0
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
// What actually went wrong behind a failed task attempt, read from its TaskEnd "Task End Reason".
|
|
2
|
+
// Spark's `Reason` is only the end-reason tag (ExceptionFailure, ExecutorLostFailure, ...); the
|
|
3
|
+
// real error lives in tag-specific fields (JsonProtocol.taskEndReasonToJson):
|
|
4
|
+
// ExceptionFailure Class Name, Description, Full Stack Trace
|
|
5
|
+
// ExecutorLostFailure Loss Reason (e.g. "Container killed by YARN for exceeding memory limits")
|
|
6
|
+
// FetchFailed Message (often a whole exception string, stack included)
|
|
7
|
+
// TaskKilled Kill Reason
|
|
8
|
+
// Every text field is bounded here, at ingest, so neither parser memory nor a report grows with
|
|
9
|
+
// the size of the traces Spark wrote.
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
const MAX_TEXT_CHARS = 300;
|
|
24
|
+
const MAX_EXCERPT_FRAMES = 8;
|
|
25
|
+
const MAX_EXCERPT_LINE_CHARS = 300;
|
|
26
|
+
const MAX_EXCERPT_CHARS = 2000;
|
|
27
|
+
// Distinct failures kept per stage while parsing, and shown per finding.
|
|
28
|
+
export const MAX_FAILURE_DETAILS_PER_STAGE = 50;
|
|
29
|
+
export const MAX_FAILURE_GROUPS = 5;
|
|
30
|
+
|
|
31
|
+
export const REDACTED_TEXT = '[redacted]';
|
|
32
|
+
|
|
33
|
+
function truncate(s , max ) {
|
|
34
|
+
return s.length > max ? `${s.slice(0, max - 3)}...` : s;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function text(v ) {
|
|
38
|
+
return typeof v === 'string' && v.trim().length > 0 ? v : null;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// First non-blank line, whitespace collapsed: a message can carry a whole stack trace after it.
|
|
42
|
+
function firstLine(s ) {
|
|
43
|
+
if (s == null) return null;
|
|
44
|
+
const line = s.split('\n').map((l) => l.trim()).find((l) => l.length > 0);
|
|
45
|
+
return line ? truncate(line.replace(/\s+/g, ' '), MAX_TEXT_CHARS) : null;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
const PYTHON_TRACEBACK_HEAD = /(?:^|: )(?:Traceback \(most recent call last\):|An exception was thrown from the Python worker)/;
|
|
49
|
+
const JAVA_FRAME = /^\s*at \S/;
|
|
50
|
+
|
|
51
|
+
// A PySpark error's text is a whole Python traceback whose own error line (`ValueError: bad row`)
|
|
52
|
+
// comes last, before any JVM frames: index of that line among non-blank `lines`, or -1.
|
|
53
|
+
function pythonErrorLineIndex(lines ) {
|
|
54
|
+
if (lines.length < 2 || !PYTHON_TRACEBACK_HEAD.test(lines[0].trim())) return -1;
|
|
55
|
+
let last = -1;
|
|
56
|
+
for (let i = 1; i < lines.length && !JAVA_FRAME.test(lines[i]); i++) last = i;
|
|
57
|
+
return last;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
// The message's headline: its first line, or a Python traceback's error line.
|
|
61
|
+
function messageLine(s ) {
|
|
62
|
+
if (s == null) return null;
|
|
63
|
+
const lines = s.split('\n').filter((l) => l.trim().length > 0);
|
|
64
|
+
const pyIdx = pythonErrorLineIndex(lines);
|
|
65
|
+
return pyIdx >= 0 ? firstLine(lines[pyIdx]) : firstLine(s);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/** Header line, the first frames, and (when cut off) a Python traceback's error line and the last
|
|
69
|
+
* `Caused by:` line, which usually name the root cause. Bounded by frame count, line length and
|
|
70
|
+
* total length. */
|
|
71
|
+
export function buildStackExcerpt(trace ) {
|
|
72
|
+
if (trace == null) return null;
|
|
73
|
+
const lines = trace.split('\n').map((l) => l.replace(/\s+$/, '')).filter((l) => l.trim().length > 0);
|
|
74
|
+
if (lines.length === 0) return null;
|
|
75
|
+
const headCount = 1 + MAX_EXCERPT_FRAMES;
|
|
76
|
+
const kept = lines.slice(0, headCount);
|
|
77
|
+
if (lines.length > headCount) {
|
|
78
|
+
const pyIdx = pythonErrorLineIndex(lines);
|
|
79
|
+
let causeIdx = -1;
|
|
80
|
+
for (let i = lines.length - 1; i >= headCount; i--) {
|
|
81
|
+
if (lines[i].startsWith('Caused by:')) { causeIdx = i; break; }
|
|
82
|
+
}
|
|
83
|
+
if (pyIdx >= headCount) kept.push('\t...', lines[pyIdx]);
|
|
84
|
+
if (pyIdx < lines.length - 1) kept.push('\t...');
|
|
85
|
+
if (causeIdx >= 0) kept.push(...lines.slice(causeIdx, causeIdx + 2));
|
|
86
|
+
}
|
|
87
|
+
return truncate(kept.map((l) => truncate(l, MAX_EXCERPT_LINE_CHARS)).join('\n'), MAX_EXCERPT_CHARS);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** Bounded failure detail for a failed attempt's end reason, or null when there is no end reason. */
|
|
91
|
+
export function extractTaskFailureDetail(endReason ) {
|
|
92
|
+
if (!endReason) return null;
|
|
93
|
+
const reason = text(endReason['Reason']);
|
|
94
|
+
const rawMessage = text(endReason['Description']) ?? text(endReason['Message']) ?? text(endReason['Kill Reason']);
|
|
95
|
+
// FetchFailed has no Full Stack Trace field, but its Message is a full exception string.
|
|
96
|
+
const trace = text(endReason['Full Stack Trace'])
|
|
97
|
+
?? (rawMessage != null && rawMessage.trim().includes('\n') ? rawMessage : null);
|
|
98
|
+
return {
|
|
99
|
+
reason,
|
|
100
|
+
className: firstLine(text(endReason['Class Name'])),
|
|
101
|
+
message: messageLine(rawMessage),
|
|
102
|
+
lossReason: firstLine(text(endReason['Loss Reason'])),
|
|
103
|
+
stackExcerpt: buildStackExcerpt(trace),
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** Grouping key: one group per distinct error. The excerpt is left out, so two traces of the same
|
|
108
|
+
* error share a group and the first one seen is kept. */
|
|
109
|
+
export function taskFailureKey(d ) {
|
|
110
|
+
return JSON.stringify([d.reason, d.className, d.message, d.lossReason]);
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/** Short name for the error: the exception class, or the end-reason tag with its loss reason. */
|
|
114
|
+
export function describeTaskFailure(d ) {
|
|
115
|
+
if (d.className) return d.className;
|
|
116
|
+
if (d.lossReason) return d.reason ? `${d.reason}: ${d.lossReason}` : d.lossReason;
|
|
117
|
+
return d.reason;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
const FRAME_LINE = /^\s*(?:at \S.*|\.\.\.(?: \d+ more)?)$/;
|
|
121
|
+
const CLASS_HEADER = /^((?:Caused by: |Suppressed: )?[\w$]+(?:\.[\w$]+)+)(?::.*)?$/;
|
|
122
|
+
|
|
123
|
+
// Keeps stack frames and exception class names only: a message (the text after "Class: ", and any
|
|
124
|
+
// continuation line such as a Python traceback's `File "/path"` lines) is dropped.
|
|
125
|
+
function stripExcerptMessages(excerpt ) {
|
|
126
|
+
const kept = [];
|
|
127
|
+
for (const line of excerpt.split('\n')) {
|
|
128
|
+
if (FRAME_LINE.test(line)) { kept.push(line); continue; }
|
|
129
|
+
const header = CLASS_HEADER.exec(line.trim());
|
|
130
|
+
if (header) kept.push(header[1]);
|
|
131
|
+
}
|
|
132
|
+
return kept.join('\n');
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/** Redacted copy of a failure group: the message and the message text inside the stack excerpt
|
|
136
|
+
* can carry file paths and data values. Class names, frames and the loss reason are kept (hosts in
|
|
137
|
+
* a loss reason are pseudonymized by the caller's host scan like any other free text). */
|
|
138
|
+
export function redactTaskFailureGroup (g ) {
|
|
139
|
+
return {
|
|
140
|
+
...g,
|
|
141
|
+
message: g.message == null ? null : REDACTED_TEXT,
|
|
142
|
+
stackExcerpt: g.stackExcerpt == null ? null : stripExcerptMessages(g.stackExcerpt),
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/** One-line headline for a failure group, for reports and the Failed Tasks widget. */
|
|
147
|
+
export function formatTaskFailureHeadline(d ) {
|
|
148
|
+
const name = d.className ?? d.reason ?? 'Unknown failure';
|
|
149
|
+
const detail = [d.message, d.lossReason].filter((s) => s != null).join(' · ');
|
|
150
|
+
return detail ? `${name}: ${detail}` : name;
|
|
151
|
+
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
// Vendored from fflate@0.8.3 (esm/browser.js), MIT license.
|
|
2
2
|
// https://github.com/101arrowz/fflate — sha256 b7ca4450b19559a1d50eb381adcee94b82449674be4cd17789d9beba7e6122a1
|
|
3
|
-
// Only
|
|
3
|
+
// Only Gunzip/UnzipInflate/gunzipSync/strFromU8 are used by this project's source (see src/shs-fetch.ts, src/zip-archive.ts); tests also use the zip writers and strToU8.
|
|
4
4
|
// DEFLATE is a complex format; to read this code, you should probably check the RFC first:
|
|
5
5
|
// https://tools.ietf.org/html/rfc1951
|
|
6
6
|
// You may also wish to take a look at the guide I made about this program:
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
// Random-access zip reader for Spark History Server log archives. Reads the
|
|
2
|
+
// central directory from the archive's tail, then inflates one entry at a time
|
|
3
|
+
// in bounded slices through fflate's streaming UnzipInflate: the source is read
|
|
4
|
+
// a slice at a time and no decompressed entry is ever held whole in memory.
|
|
5
|
+
//
|
|
6
|
+
// Why the central directory and not fflate's forward-scanning `Unzip`: the
|
|
7
|
+
// History Server writes its zip with Java's ZipOutputStream, which leaves
|
|
8
|
+
// every local header's sizes blank and puts them in a data descriptor after
|
|
9
|
+
// the entry. `Unzip` then has to find each entry's end by scanning the
|
|
10
|
+
// compressed bytes for the descriptor signature, and it hands rolling-log
|
|
11
|
+
// parts over in archive order, not the order they must be parsed in. The
|
|
12
|
+
// central directory has the real sizes and lets the caller pick the order.
|
|
13
|
+
import { UnzipInflate, strFromU8 } from './vendor/fflate.js';
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
const LOCAL_HEADER_SIG = 0x04034b50;
|
|
29
|
+
const CENTRAL_HEADER_SIG = 0x02014b50;
|
|
30
|
+
const EOCD_SIG = 0x06054b50;
|
|
31
|
+
const ZIP64_EOCD_LOCATOR_SIG = 0x07064b50;
|
|
32
|
+
const ZIP64_EOCD_SIG = 0x06064b50;
|
|
33
|
+
const ZIP64_EXTRA_ID = 0x0001;
|
|
34
|
+
const EOCD_MIN_SIZE = 22;
|
|
35
|
+
// The end-of-central-directory record ends with a comment of at most 65535 bytes.
|
|
36
|
+
const EOCD_MAX_SEARCH = EOCD_MIN_SIZE + 0xffff;
|
|
37
|
+
const UINT32_MAX = 0xffffffff;
|
|
38
|
+
|
|
39
|
+
// fflate's UnzipInflate is untyped vendor JS; this local shape types the call site.
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
const u16 = (d , i ) => d[i] | (d[i + 1] << 8);
|
|
47
|
+
const u32 = (d , i ) => (d[i] | (d[i + 1] << 8) | (d[i + 2] << 16) | (d[i + 3] << 24)) >>> 0;
|
|
48
|
+
const u64 = (d , i ) => u32(d, i) + u32(d, i + 4) * 2 ** 32;
|
|
49
|
+
|
|
50
|
+
async function readRange(source , start , end ) {
|
|
51
|
+
if (start < 0 || end > source.size || start > end) throw new Error('zip structure points outside the file');
|
|
52
|
+
return new Uint8Array(await source.slice(start, end).arrayBuffer());
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** True when `header` starts with a zip local-file header or an empty archive's EOCD record. */
|
|
56
|
+
export function isZip(header ) {
|
|
57
|
+
if (header.length < 4) return false;
|
|
58
|
+
const sig = u32(header, 0);
|
|
59
|
+
return sig === LOCAL_HEADER_SIG || sig === EOCD_SIG;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
// Locates the central directory via the EOCD record (or its zip64 variant).
|
|
63
|
+
async function readDirectoryLocation(source ) {
|
|
64
|
+
const tailStart = Math.max(0, source.size - EOCD_MAX_SEARCH);
|
|
65
|
+
const tail = await readRange(source, tailStart, source.size);
|
|
66
|
+
let eocd = tail.length - EOCD_MIN_SIZE;
|
|
67
|
+
while (eocd >= 0 && u32(tail, eocd) !== EOCD_SIG) eocd--;
|
|
68
|
+
if (eocd < 0) throw new Error('no end-of-central-directory record');
|
|
69
|
+
|
|
70
|
+
const location = { count: u16(tail, eocd + 10), size: u32(tail, eocd + 12), offset: u32(tail, eocd + 16) };
|
|
71
|
+
const locator = tailStart + eocd - 20;
|
|
72
|
+
if (locator >= 0) {
|
|
73
|
+
const loc = await readRange(source, locator, locator + 20);
|
|
74
|
+
if (u32(loc, 0) === ZIP64_EOCD_LOCATOR_SIG) {
|
|
75
|
+
const zip64Offset = u64(loc, 8);
|
|
76
|
+
const z = await readRange(source, zip64Offset, zip64Offset + 56);
|
|
77
|
+
if (u32(z, 0) !== ZIP64_EOCD_SIG) throw new Error('bad zip64 end-of-central-directory record');
|
|
78
|
+
return { count: u64(z, 32), size: u64(z, 40), offset: u64(z, 48) };
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
return location;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
// Reads the zip64 extended-information extra field, which carries (in this
|
|
85
|
+
// order) only the sizes/offset whose 32-bit central-directory field is 0xFFFFFFFF.
|
|
86
|
+
function applyZip64Extra(dir , extraStart , extraEnd , fields ) {
|
|
87
|
+
for (let p = extraStart; p + 4 <= extraEnd;) {
|
|
88
|
+
const id = u16(dir, p), len = u16(dir, p + 2);
|
|
89
|
+
if (id === ZIP64_EXTRA_ID) {
|
|
90
|
+
let q = p + 4;
|
|
91
|
+
for (const key of ['uncompressed', 'compressed', 'offset'] ) {
|
|
92
|
+
if (fields[key] === UINT32_MAX && q + 8 <= p + 4 + len) { fields[key] = u64(dir, q); q += 8; }
|
|
93
|
+
}
|
|
94
|
+
return;
|
|
95
|
+
}
|
|
96
|
+
p += 4 + len;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** Lists every entry in the archive's central directory, in directory order. */
|
|
101
|
+
export async function listZipEntries(source ) {
|
|
102
|
+
const { count, offset, size } = await readDirectoryLocation(source);
|
|
103
|
+
const dir = await readRange(source, offset, offset + size);
|
|
104
|
+
const entries = [];
|
|
105
|
+
let p = 0;
|
|
106
|
+
for (let i = 0; i < count; i++) {
|
|
107
|
+
if (p + 46 > dir.length || u32(dir, p) !== CENTRAL_HEADER_SIG) throw new Error('bad central-directory header');
|
|
108
|
+
const nameLength = u16(dir, p + 28), extraLength = u16(dir, p + 30), commentLength = u16(dir, p + 32);
|
|
109
|
+
const utf8 = (u16(dir, p + 8) & 0x800) !== 0;
|
|
110
|
+
const nameStart = p + 46, extraStart = nameStart + nameLength;
|
|
111
|
+
const fields = { uncompressed: u32(dir, p + 24), compressed: u32(dir, p + 20), offset: u32(dir, p + 42) };
|
|
112
|
+
applyZip64Extra(dir, extraStart, extraStart + extraLength, fields);
|
|
113
|
+
entries.push({
|
|
114
|
+
name: strFromU8(dir.subarray(nameStart, extraStart), !utf8),
|
|
115
|
+
compression: u16(dir, p + 10),
|
|
116
|
+
compressedSize: fields.compressed,
|
|
117
|
+
localHeaderOffset: fields.offset,
|
|
118
|
+
});
|
|
119
|
+
p = extraStart + extraLength + commentLength;
|
|
120
|
+
}
|
|
121
|
+
return entries;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* Streams one entry's decompressed bytes to `onChunk`, reading `chunkSize`
|
|
126
|
+
* compressed bytes at a time and awaiting `onChunk` before reading more, so an
|
|
127
|
+
* async consumer (an off-thread zstd decoder) applies backpressure. `onRead`
|
|
128
|
+
* reports compressed bytes consumed, for progress.
|
|
129
|
+
*/
|
|
130
|
+
export async function streamZipEntry(
|
|
131
|
+
source ,
|
|
132
|
+
entry ,
|
|
133
|
+
onChunk ,
|
|
134
|
+
{ chunkSize, onRead } ,
|
|
135
|
+
) {
|
|
136
|
+
const header = await readRange(source, entry.localHeaderOffset, entry.localHeaderOffset + 30);
|
|
137
|
+
if (u32(header, 0) !== LOCAL_HEADER_SIG) throw new Error(`bad local header for "${entry.name}"`);
|
|
138
|
+
const dataStart = entry.localHeaderOffset + 30 + u16(header, 26) + u16(header, 28);
|
|
139
|
+
const dataEnd = dataStart + entry.compressedSize;
|
|
140
|
+
if (dataEnd > source.size) throw new Error(`"${entry.name}" is truncated`);
|
|
141
|
+
if (entry.compression !== 0 && entry.compression !== 8) {
|
|
142
|
+
throw new Error(`"${entry.name}" uses unsupported zip compression method ${entry.compression}`);
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
const pending = [];
|
|
146
|
+
let inflateError = null;
|
|
147
|
+
const inflater = entry.compression === 8 ? new (UnzipInflate )() : null;
|
|
148
|
+
if (inflater) {
|
|
149
|
+
inflater.ondata = (err, data) => {
|
|
150
|
+
if (err) inflateError = err;
|
|
151
|
+
else if (data.length) pending.push(data);
|
|
152
|
+
};
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
for (let offset = dataStart; offset < dataEnd;) {
|
|
156
|
+
const end = Math.min(dataEnd, offset + chunkSize);
|
|
157
|
+
const slice = await readRange(source, offset, end);
|
|
158
|
+
offset = end;
|
|
159
|
+
if (inflater) inflater.push(slice, offset >= dataEnd);
|
|
160
|
+
else pending.push(slice);
|
|
161
|
+
if (inflateError) throw inflateError;
|
|
162
|
+
onRead?.(slice.length);
|
|
163
|
+
// Inflate output chunks are views that fflate may reuse on the next push,
|
|
164
|
+
// so each is fully consumed here before the loop reads further.
|
|
165
|
+
while (pending.length) await onChunk(pending.shift() );
|
|
166
|
+
}
|
|
167
|
+
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
import{r as e}from"./rolldown-runtime-hePW80VL.js";import{M as t}from"./charts-vendor-0VMaLVLN.js";import{p as n}from"./plan-graph-vendor-B0sEms9Z.js";import{A as r,D as i,O as a,_ as o,b as s,bt as c,gt as l,h as u,ht as d,i as f,k as p,l as m,m as h,ot as g,pt as _,ut as v,v as y,y as b}from"./index-pnImGbgQ.js";import{t as x}from"./WidgetLeadSummary-BdUUi7L5.js";import{t as S}from"./ImpactEstimate-Df_laG-n.js";import{t as C}from"./RowPagination-2icPidMP.js";import{n as w}from"./finding-anchor-CDbHtqKT.js";import{t as T}from"./usePagedRows-jNUWgVTD.js";var E=e(t(),1),D=n(),O=70;function k(e){if(!e)return`—`;let t=[];return e.useMemory&&t.push(`Memory`),e.useDisk&&t.push(`Disk`),e.deserialized?t.push(`Deserialized`):(e.useMemory||e.useDisk)&&t.push(`Serialized`),(e.replication??1)>1&&t.push(`x${e.replication}`),t.join(` · `)||`—`}function A(e){return e.name||`RDD ${e.id}`}function j(e){let t=l(A(e));return t.length<=O?t:`${t.slice(0,O)}…`}function M(e){let t=_(`pct`,d(e));return e.variant===`diskSpillover`?`${t} spilled to disk: ${v(e.diskSize??0)} disk / ${v(e.memorySize??0)} memory`:`${t} cached (${e.numCachedPartitions??0}/${e.numPartitions??0} partitions)`}function N({finding:e}){let t=w([e]),n=e.rddName??`RDD ${e.rddId}`;return(0,D.jsxs)(`div`,{ref:t.ref,tabIndex:t.tabIndex,"data-flashed":t.dataFlashed,className:`space-y-1 transition-colors ${t.flashClassName}`,children:[(0,D.jsxs)(`p`,{className:`flex flex-wrap items-start gap-2`,children:[(0,D.jsx)(i,{impactBand:e.impactBand,className:`mt-1.5`}),(0,D.jsx)(`span`,{children:n})]}),(0,D.jsx)(p,{children:(0,D.jsx)(`p`,{className:`text-xs text-muted-foreground`,children:M(e)})}),(0,D.jsx)(S,{finding:e}),(0,D.jsx)(`p`,{className:`text-sm`,children:e.recommendation})]})}var P=(0,E.memo)(function({appModel:e,catalog:t}){let[n,l]=(0,E.useState)(0),[d,p]=(0,E.useState)(0),_=e.app?.rddInfo,S=(0,E.useMemo)(()=>_?[..._.values()].filter(e=>(e.storageLevel?.useMemory||e.storageLevel?.useDisk)&&(e.numCachedPartitions??0)>0).sort((e,t)=>(t.memorySize??0)+(t.diskSize??0)-((e.memorySize??0)+(e.diskSize??0))):[],[_]),w=(0,E.useMemo)(()=>t.filter(e=>e.type===`cacheUtilization`).sort((e,t)=>g[e.impactBand]-g[t.impactBand]),[t]),O=T(S,n,l),M=m(),P=M?w.findIndex(e=>e===M.finding):null,F=T(w,d,p,P);if(S.length===0)return null;let I=e=>w.filter(t=>t.rddId===e),L=S.reduce((e,t)=>e+(t.memorySize??0)+(t.diskSize??0),0);return(0,D.jsx)(`div`,{"data-widget":`cache-utilization`,children:(0,D.jsxs)(f,{title:`Cache Storage`,impactBand:c(w),badges:(0,D.jsxs)(D.Fragment,{children:[w.length>0?(0,D.jsx)(a,{type:`cacheUtilization`,impactBand:c(w)??`info`}):null,(0,D.jsxs)(`span`,{className:`text-xs text-muted-foreground`,children:[S.length,` cached RDD`,S.length===1?``:`s`,`, see`,` `,(0,D.jsx)(r,{anchor:`#memory-model`,children:`how Spark accounts cached memory`})]})]}),defaultCollapsed:!0,summary:(0,D.jsx)(x,{value:v(L),context:`cached across memory + disk`}),children:[(0,D.jsxs)(h,{children:[(0,D.jsx)(b,{children:(0,D.jsxs)(s,{children:[(0,D.jsx)(y,{children:`RDD`}),(0,D.jsx)(y,{children:`Storage level`}),(0,D.jsx)(y,{className:`text-right`,children:`Cached partitions`}),(0,D.jsx)(y,{className:`text-right`,children:`Memory`}),(0,D.jsx)(y,{className:`text-right`,children:`Disk`})]})}),(0,D.jsx)(u,{children:O.visible.map(e=>{let t=A(e),n=j(e),r=I(e.id);return(0,D.jsxs)(s,{children:[(0,D.jsxs)(o,{className:`max-w-80 truncate`,title:t===n?void 0:t,children:[n,r.length>0&&(0,D.jsx)(i,{impactBand:c(r)??`info`,className:`ml-1.5`})]}),(0,D.jsx)(o,{children:k(e.storageLevel)}),(0,D.jsxs)(o,{className:`text-right`,children:[e.numCachedPartitions,` / `,e.numPartitions]}),(0,D.jsx)(o,{className:`text-right`,children:v(e.memorySize??0)}),(0,D.jsx)(o,{className:`text-right`,children:v(e.diskSize??0)})]},e.id)})})]}),(0,D.jsx)(C,{page:O.effectivePage,totalPages:O.totalPages,onPrev:()=>l(e=>e-1),onNext:()=>l(e=>e+1)}),w.length>0?(0,D.jsxs)(`div`,{className:`mt-3 space-y-2 text-sm`,children:[F.visible.map(e=>(0,D.jsx)(N,{finding:e},e.id)),(0,D.jsx)(C,{page:F.effectivePage,totalPages:F.totalPages,onPrev:()=>p(e=>e-1),onNext:()=>p(e=>e+1)})]}):null]})})});export{P as CacheUtilization};
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
import{r as e}from"./rolldown-runtime-hePW80VL.js";import{M as t}from"./charts-vendor-0VMaLVLN.js";import{p as n}from"./plan-graph-vendor-B0sEms9Z.js";import{S as r}from"./useRecentFiles-0XuhS91q.js";import{At as i,O as a,_ as o,b as s,h as c,i as l,j as u,k as d,l as f,m as p,ut as m,v as h,y as g}from"./index-pnImGbgQ.js";import{t as _}from"./WidgetLeadSummary-BdUUi7L5.js";import{t as v}from"./ImpactEstimate-Df_laG-n.js";import{t as y}from"./RowPagination-2icPidMP.js";import{n as b}from"./finding-anchor-CDbHtqKT.js";import{t as x}from"./usePagedRows-jNUWgVTD.js";var S={name:`git-merge`,size:24,node:[[`circle`,{cx:`18`,cy:`18`,r:`3`,key:`1xkwt0`}],[`circle`,{cx:`6`,cy:`6`,r:`3`,key:`1lh9wr`}],[`path`,{d:`M6 21V9a9 9 0 0 0 9 9`,key:`7kw0sc`}]]};S.node;var C=i(S),w=e(t(),1),T=n();function E({relations:e,operator:t}){let n=t===`union`?`union`:`join`;return(0,T.jsx)(`span`,{className:`inline-flex flex-wrap items-baseline`,children:e.map((e,t)=>(0,T.jsxs)(`span`,{children:[t>0?(0,T.jsxs)(T.Fragment,{children:[` `,(0,T.jsx)(`span`,{className:`mx-1 font-semibold uppercase text-xs tracking-wide text-primary`,children:n}),` `]}):null,e.relation]},e.relation))})}function D({finding:e}){let t=b([e]),n=e.variant===`composite`;return(0,T.jsxs)(s,{ref:t.ref,tabIndex:t.tabIndex,"data-flashed":t.dataFlashed,className:`transition-colors ${t.flashClassName}`,children:[(0,T.jsx)(o,{children:(0,T.jsxs)(`span`,{className:`inline-flex items-center gap-2`,children:[n?(0,T.jsx)(C,{className:`size-3.5 text-muted-foreground`,"aria-hidden":`true`}):null,n&&e.relations&&e.relations.length>0?(0,T.jsx)(E,{relations:e.relations,operator:e.operator}):(0,T.jsx)(`span`,{children:e.relation}),n?(0,T.jsx)(u,{variant:`outline`,className:`text-xs uppercase border-dashed`,children:e.operator?.toUpperCase()}):(0,T.jsx)(u,{variant:`secondary`,className:`text-xs uppercase`,children:e.format})]})}),(0,T.jsxs)(o,{children:[e.value,` queries`]}),(0,T.jsx)(o,{className:`text-right`,children:m(e.totalReadBytes??0)}),(0,T.jsxs)(o,{children:[(0,T.jsx)(v,{finding:e}),(0,T.jsx)(`p`,{className:`mt-1`,children:e.recommendation}),(0,T.jsx)(d,{children:(0,T.jsx)(r,{confidence:e.confidence,validationRequired:e.validationRequired})})]})]})}var O=(0,w.memo)(function({catalog:e,defaultCollapsed:t=!0}){let[n,r]=(0,w.useState)(0),i=(0,w.useMemo)(()=>e.filter(e=>e.type===`cachingOpportunity`).sort((e,t)=>(t.totalReadBytes??0)-(e.totalReadBytes??0)||(typeof t.value==`number`?t.value:0)-(typeof e.value==`number`?e.value:0)),[e]),o=f(),u=o?i.findIndex(e=>e===o.finding):null,{totalPages:m,effectivePage:v,visible:b}=x(i,n,r,u);if(i.length===0)return null;let S=i[0];return(0,T.jsx)(l,{title:`Caching Opportunities`,impactBand:`info`,badges:(0,T.jsx)(a,{type:`cachingOpportunity`,impactBand:`info`}),defaultCollapsed:t,summary:(0,T.jsx)(_,{value:`${S.value}×`,context:`read by ${S.value} queries: ${S.relation}`}),children:(0,T.jsxs)(`div`,{className:`space-y-3`,children:[(0,T.jsxs)(p,{children:[(0,T.jsx)(g,{children:(0,T.jsxs)(s,{children:[(0,T.jsx)(h,{children:`Relation`}),(0,T.jsx)(h,{children:`Read by`}),(0,T.jsx)(h,{className:`text-right`,children:`Data read`}),(0,T.jsx)(h,{children:`Recommendation`})]})}),(0,T.jsx)(c,{children:b.map((e,t)=>(0,T.jsx)(D,{finding:e},`${e.format}:${e.relation}:${(e.executionIds??[]).join(`,`)}:${t}`))})]}),(0,T.jsx)(d,{children:(0,T.jsx)(`p`,{className:`text-xs text-muted-foreground`,children:`Reuse is inferred from plan structure, not confirmed by execution. Verify before caching.`})}),(0,T.jsx)(y,{page:v,totalPages:m,onPrev:()=>r(e=>e-1),onNext:()=>r(e=>e+1)})]})})});export{O as CachingOpportunity};
|