sparkforensics-cli 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/sparkforensics-analyze.mjs +288 -0
- package/package.json +29 -0
- package/vendor-core/analyzer.js +167 -0
- package/vendor-core/assert-never.js +3 -0
- package/vendor-core/cli/budgets.js +203 -0
- package/vendor-core/cli/collect-run.js +107 -0
- package/vendor-core/core-count.js +68 -0
- package/vendor-core/core-locality-ratio.js +54 -0
- package/vendor-core/core-time-series.js +92 -0
- package/vendor-core/core-usage-locality.js +70 -0
- package/vendor-core/detectors.js +1989 -0
- package/vendor-core/docs-config.js +72 -0
- package/vendor-core/docs-site-config.js +23 -0
- package/vendor-core/efficiency-model.js +62 -0
- package/vendor-core/etl-phases.js +28 -0
- package/vendor-core/event-handlers.js +906 -0
- package/vendor-core/event-schemas.js +405 -0
- package/vendor-core/evidence-availability.js +121 -0
- package/vendor-core/evidence-report.js +459 -0
- package/vendor-core/finding-action-label.js +97 -0
- package/vendor-core/finding-filter-predicate.js +38 -0
- package/vendor-core/format-utils.js +167 -0
- package/vendor-core/impact-band.js +50 -0
- package/vendor-core/impact-estimator.js +428 -0
- package/vendor-core/ingest.js +139 -0
- package/vendor-core/job-groups.js +30 -0
- package/vendor-core/load-vendored.js +24 -0
- package/vendor-core/lz4-block.js +135 -0
- package/vendor-core/mcp-error.js +3 -0
- package/vendor-core/mcp-server-factory.js +115 -0
- package/vendor-core/mcp-tools.js +331 -0
- package/vendor-core/model-assembler.js +76 -0
- package/vendor-core/occupancy.js +202 -0
- package/vendor-core/parser-worker.js +249 -0
- package/vendor-core/plan-dot.js +25 -0
- package/vendor-core/plan-duration-attribution.js +185 -0
- package/vendor-core/plan-graph-model.js +171 -0
- package/vendor-core/plan-node-detail.js +159 -0
- package/vendor-core/plan-summary.js +233 -0
- package/vendor-core/plan-tree-walk.js +29 -0
- package/vendor-core/proxy.js +157 -0
- package/vendor-core/recommendation-rollup.js +197 -0
- package/vendor-core/redact.js +175 -0
- package/vendor-core/rolling-log-reassembly.js +52 -0
- package/vendor-core/run-aggregates.js +44 -0
- package/vendor-core/run-comparison.js +458 -0
- package/vendor-core/scaling-sim.js +73 -0
- package/vendor-core/session-snapshot.js +79 -0
- package/vendor-core/shs-fetch.js +196 -0
- package/vendor-core/shs-load.js +121 -0
- package/vendor-core/shs-request.js +101 -0
- package/vendor-core/shs-schemas.js +13 -0
- package/vendor-core/snappy-block.js +140 -0
- package/vendor-core/stage-quantiles.js +199 -0
- package/vendor-core/threshold-summary.js +35 -0
- package/vendor-core/types.js +286 -0
- package/vendor-core/vendor/fflate.js +2695 -0
- package/vendor-core/vendor/fzstd.js +768 -0
- package/vendor-core/wall-clock.js +36 -0
- package/vendor-core/wasted-core-hours.js +68 -0
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
import { unzipSync, Gunzip } from './vendor/fflate.js';
|
|
2
|
+
import { createLz4BlockDecoder } from './lz4-block.js';
|
|
3
|
+
import { Decompress as ZstdDecompress } from './vendor/fzstd.js';
|
|
4
|
+
import { createSnappyBlockDecoder } from './snappy-block.js';
|
|
5
|
+
import { buildProxyRequestUrl, isShsErrorCode } from './shs-request.js';
|
|
6
|
+
import { dispatchLine, buildChunkDecoder, emitParseCompletion, } from './event-handlers.js';
|
|
7
|
+
import { ShsProxyErrorBodySchema } from './shs-schemas.js';
|
|
8
|
+
import { naturalCompare, reassembleRollingEntries } from './rolling-log-reassembly.js';
|
|
9
|
+
|
|
10
|
+
export { naturalCompare, reassembleRollingEntries };
|
|
11
|
+
|
|
12
|
+
// Sniff a compression codec from leading magic bytes: gzip (1f 8b), Zstandard
|
|
13
|
+
// (28 b5 2f fd), Spark's custom "LZ4Block" framing, or Spark's Snappy framing
|
|
14
|
+
// (org.xerial.snappy's "\x82SNAPPY\0" header). Returns 'gz' | 'zstd' | 'lz4' |
|
|
15
|
+
// 'snappy' | null. More robust than a filename suffix: a dropped SHS log may
|
|
16
|
+
// have no extension.
|
|
17
|
+
export function sniffCodec(bytes ) {
|
|
18
|
+
if (bytes.length >= 2 && bytes[0] === 0x1f && bytes[1] === 0x8b) return 'gz';
|
|
19
|
+
if (bytes.length >= 4 && bytes[0] === 0x28 && bytes[1] === 0xb5 && bytes[2] === 0x2f && bytes[3] === 0xfd) return 'zstd';
|
|
20
|
+
const LZ4_MAGIC = [76, 90, 52, 66, 108, 111, 99, 107]; // "LZ4Block"
|
|
21
|
+
if (bytes.length >= 8 && LZ4_MAGIC.every((b, i) => bytes[i] === b)) return 'lz4';
|
|
22
|
+
const SNAPPY_MAGIC = [0x82, 0x53, 0x4e, 0x41, 0x50, 0x50, 0x59, 0x00]; // 0x82 'S' 'N' 'A' 'P' 'P' 'Y' 0x00
|
|
23
|
+
if (bytes.length >= 8 && SNAPPY_MAGIC.every((b, i) => bytes[i] === b)) return 'snappy';
|
|
24
|
+
return null;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
// Streams one already-in-memory zip entry through its codec's block-by-block
|
|
28
|
+
// decoder, invoking `onChunk` per decompressed piece; never materializes the
|
|
29
|
+
// whole decompressed entry as one buffer. A giant single-segment zstd frame
|
|
30
|
+
// (or gzip member) declares its full content size in the header; one-shotting
|
|
31
|
+
// it (fzstd's `decompress()`, fflate's `gunzipSync()`) allocates that entire
|
|
32
|
+
// size as a single ArrayBuffer up front, which can exceed what the browser
|
|
33
|
+
// will allocate for a multi-GB event log ("Array buffer allocation failed").
|
|
34
|
+
// fflate's Gunzip and fzstd's Decompress are untyped vendor JS (plain
|
|
35
|
+
// prototype classes, not `class` declarations), so TS can't infer a
|
|
36
|
+
// construct signature for them; this local shape is just enough to type
|
|
37
|
+
// the two call sites below without touching the vendored files.
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
function decodeEntry(name , raw , onChunk ) {
|
|
42
|
+
const codec = sniffCodec(raw);
|
|
43
|
+
if (codec === 'lz4' || name.endsWith('.lz4')) {
|
|
44
|
+
const lz4 = createLz4BlockDecoder(onChunk);
|
|
45
|
+
lz4.push(raw);
|
|
46
|
+
lz4.end();
|
|
47
|
+
} else if (codec === 'gz' || name.endsWith('.gz')) {
|
|
48
|
+
new (Gunzip )(onChunk).push(raw, true);
|
|
49
|
+
} else if (codec === 'zstd' || name.endsWith('.zstd') || name.endsWith('.zst')) {
|
|
50
|
+
new (ZstdDecompress )(onChunk).push(raw, true);
|
|
51
|
+
} else if (codec === 'snappy' || name.endsWith('.snappy')) {
|
|
52
|
+
const snappy = createSnappyBlockDecoder(onChunk);
|
|
53
|
+
snappy.push(raw);
|
|
54
|
+
snappy.end();
|
|
55
|
+
} else {
|
|
56
|
+
onChunk(raw);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
function emitShsError(emit , code ) {
|
|
63
|
+
emit({ type: 'error', source: 'shs', code });
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export async function runParseFromUrl(
|
|
67
|
+
request ,
|
|
68
|
+
state ,
|
|
69
|
+
{ fetchImpl = fetch, emit = (msg ) => self.postMessage(msg) } = {}
|
|
70
|
+
) {
|
|
71
|
+
// buildProxyRequestUrl is still-untyped JS (./shs-request.js, a plain
|
|
72
|
+
// untyped .js sibling); `request`'s real shape isn't pinned down here.
|
|
73
|
+
const url = buildProxyRequestUrl(request );
|
|
74
|
+
|
|
75
|
+
let res;
|
|
76
|
+
try {
|
|
77
|
+
res = await fetchImpl(url);
|
|
78
|
+
} catch {
|
|
79
|
+
emitShsError(emit, 'local-server-unavailable');
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
if (res.status === 404) {
|
|
83
|
+
emitShsError(emit, 'local-server-unavailable');
|
|
84
|
+
return;
|
|
85
|
+
}
|
|
86
|
+
if (!res.ok) {
|
|
87
|
+
let code = 'access-or-upstream-failure';
|
|
88
|
+
try {
|
|
89
|
+
const body = ShsProxyErrorBodySchema.parse(await res.json());
|
|
90
|
+
if (isShsErrorCode(body.code)) code = body.code;
|
|
91
|
+
} catch { /* Malformed proxy errors, or a body that fails schema validation, retain the generic safe code: same silent-skip treatment as a log-line validation failure. */ }
|
|
92
|
+
emitShsError(emit, code);
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
if (!res.body) {
|
|
97
|
+
emitShsError(emit, 'invalid-event-log');
|
|
98
|
+
return;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
let zipBytes;
|
|
102
|
+
let total;
|
|
103
|
+
try {
|
|
104
|
+
total = Number(res.headers.get('content-length')) || null;
|
|
105
|
+
const reader = res.body.getReader();
|
|
106
|
+
const chunks = [];
|
|
107
|
+
let received = 0;
|
|
108
|
+
for (;;) {
|
|
109
|
+
const { done, value } = await reader.read();
|
|
110
|
+
if (done) break;
|
|
111
|
+
chunks.push(value);
|
|
112
|
+
received += value.length;
|
|
113
|
+
if (!total || received < total) {
|
|
114
|
+
emit({ type: 'progress', pct: total ? (received / total) * 0.5 : null, linesProcessed: 0 });
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
zipBytes = new Uint8Array(received);
|
|
118
|
+
let writeOffset = 0;
|
|
119
|
+
for (const chunk of chunks) { zipBytes.set(chunk, writeOffset); writeOffset += chunk.length; }
|
|
120
|
+
} catch {
|
|
121
|
+
emitShsError(emit, 'local-server-unavailable');
|
|
122
|
+
return;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
const contentType = res.headers.get('content-type') || '';
|
|
126
|
+
if (contentType.startsWith('text/html')) {
|
|
127
|
+
emitShsError(emit, 'local-server-unavailable');
|
|
128
|
+
return;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
// The in-loop guard above intentionally withholds the tick for the chunk that
|
|
132
|
+
// completes the download (received === total), to avoid double-emitting it here.
|
|
133
|
+
// Emit it now that we know the download is complete: this is the one place a
|
|
134
|
+
// determinate download is guaranteed to reach pct 0.5. Indeterminate downloads
|
|
135
|
+
// (no Content-Length) already got their final tick from inside the loop, since
|
|
136
|
+
// the loop's guard never withholds when total is null.
|
|
137
|
+
if (total) {
|
|
138
|
+
emit({ type: 'progress', pct: 0.5, linesProcessed: 0 });
|
|
139
|
+
}
|
|
140
|
+
decodeShsArchive(zipBytes, state, emit);
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
export function decodeShsArchive(zipBytes , state , emit ) {
|
|
144
|
+
let entries ;
|
|
145
|
+
try {
|
|
146
|
+
entries = unzipSync(zipBytes);
|
|
147
|
+
} catch {
|
|
148
|
+
emitShsError(emit, 'invalid-event-log');
|
|
149
|
+
return;
|
|
150
|
+
}
|
|
151
|
+
const allNames = Object.keys(entries);
|
|
152
|
+
const isRolling = allNames.some(n => /^events_\d+_/.test(n));
|
|
153
|
+
let names;
|
|
154
|
+
if (isRolling) {
|
|
155
|
+
try {
|
|
156
|
+
names = reassembleRollingEntries(allNames);
|
|
157
|
+
} catch {
|
|
158
|
+
emitShsError(emit, 'invalid-event-log');
|
|
159
|
+
return;
|
|
160
|
+
}
|
|
161
|
+
} else {
|
|
162
|
+
names = allNames.filter(n => n.toLowerCase() !== 'appstatus').sort(naturalCompare);
|
|
163
|
+
}
|
|
164
|
+
if (names.length === 0) {
|
|
165
|
+
emitShsError(emit, 'invalid-event-log');
|
|
166
|
+
return;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
const decoder = buildChunkDecoder();
|
|
170
|
+
let linesProcessed = 0;
|
|
171
|
+
for (const name of names) {
|
|
172
|
+
try {
|
|
173
|
+
decodeEntry(name, entries[name], (bytes) => {
|
|
174
|
+
for (const line of decoder.decode(bytes)) {
|
|
175
|
+
dispatchLine(line, state, emit);
|
|
176
|
+
linesProcessed++;
|
|
177
|
+
if (linesProcessed % 2000 === 0) {
|
|
178
|
+
emit({ type: 'progress', pct: null, linesProcessed });
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
});
|
|
182
|
+
} catch {
|
|
183
|
+
emitShsError(emit, 'invalid-event-log');
|
|
184
|
+
return;
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
for (const line of decoder.flush()) {
|
|
188
|
+
dispatchLine(line, state, emit);
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
if (!state.app) {
|
|
192
|
+
emitShsError(emit, 'invalid-event-log');
|
|
193
|
+
return;
|
|
194
|
+
}
|
|
195
|
+
emitParseCompletion(state, emit, linesProcessed);
|
|
196
|
+
}
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import { validateShsRequest } from './shs-request.js';
|
|
2
|
+
import { fetchShsEventLog } from './proxy.js';
|
|
3
|
+
import { decodeShsArchive } from './parser-worker.js';
|
|
4
|
+
import { collectViaDispatch } from './cli/collect-run.js';
|
|
5
|
+
import { deriveEvidenceAvailability } from './evidence-availability.js';
|
|
6
|
+
import { mcpError } from './mcp-error.js';
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
function envInt(name , fallback ) {
|
|
10
|
+
const v = Number(process.env[name]);
|
|
11
|
+
return Number.isFinite(v) && v > 0 ? v : fallback;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export const DEFAULT_MAX_ARCHIVE_BYTES = envInt('SPARKFORENSICS_MAX_ARCHIVE_BYTES', 1024 * 1024 * 1024);
|
|
15
|
+
// Same knob the proxy uses: fetchShsEventLog covers the header phase; this
|
|
16
|
+
// covers the body, per chunk, so progressing downloads of any size are fine.
|
|
17
|
+
export const DEFAULT_IDLE_TIMEOUT_MS = envInt('SPARKFORENSICS_SHS_TIMEOUT_MS', 30_000);
|
|
18
|
+
|
|
19
|
+
function collectShsAppModel(zipBytes ) {
|
|
20
|
+
return collectViaDispatch(
|
|
21
|
+
(state, emit) => decodeShsArchive(zipBytes, state, emit),
|
|
22
|
+
(msg) => {
|
|
23
|
+
const m = msg ;
|
|
24
|
+
return mcpError(m?.code ?? 'invalid-event-log', m?.message ?? 'Failed to decode SHS archive.');
|
|
25
|
+
},
|
|
26
|
+
);
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
// Races one reader.read() against a fresh idle timer. The losing read() stays
|
|
30
|
+
// pending after a timeout; reader.cancel() in the caller settles it.
|
|
31
|
+
function readWithIdleTimeout(
|
|
32
|
+
reader ,
|
|
33
|
+
idleTimeoutMs ,
|
|
34
|
+
) {
|
|
35
|
+
let timer ;
|
|
36
|
+
const stalled = new Promise((_resolve, reject) => {
|
|
37
|
+
timer = setTimeout(() => reject(mcpError(
|
|
38
|
+
'upstream-unreachable',
|
|
39
|
+
`SHS archive body stalled for ${idleTimeoutMs} ms (override with SPARKFORENSICS_SHS_TIMEOUT_MS).`,
|
|
40
|
+
)), idleTimeoutMs);
|
|
41
|
+
});
|
|
42
|
+
return Promise.race([reader.read(), stalled]).finally(() => clearTimeout(timer));
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
// Reads the archive body with a byte cap: an unbounded arrayBuffer() on a
|
|
46
|
+
// hostile or misconfigured SHS response would OOM the long-running process.
|
|
47
|
+
// The per-chunk idle timeout keeps a stalled body from hanging the call
|
|
48
|
+
// forever with headers already received.
|
|
49
|
+
async function readArchiveBytes(upstream , maxBytes , idleTimeoutMs ) {
|
|
50
|
+
const tooLarge = () => mcpError(
|
|
51
|
+
'archive-too-large',
|
|
52
|
+
`SHS archive exceeds the ${maxBytes}-byte cap (override with SPARKFORENSICS_MAX_ARCHIVE_BYTES).`,
|
|
53
|
+
);
|
|
54
|
+
const declared = Number(upstream.headers?.get?.('content-length'));
|
|
55
|
+
if (Number.isFinite(declared) && declared > maxBytes) throw tooLarge();
|
|
56
|
+
|
|
57
|
+
// fetchShsEventLog (./proxy.js, a plain untyped .js sibling)
|
|
58
|
+
// already checks `!upstream.body` and returns `{ ok: false }` before ever
|
|
59
|
+
// returning `{ ok: true, upstream }`, so `body` is guaranteed present here
|
|
60
|
+
// even though DOM's Response.body type is nullable.
|
|
61
|
+
const reader = upstream.body .getReader();
|
|
62
|
+
// Grown geometrically instead of preallocated at `maxBytes`: a typical
|
|
63
|
+
// archive is nowhere near the cap, so starting small (or at the declared
|
|
64
|
+
// content-length, when trustworthy) and doubling on demand keeps peak
|
|
65
|
+
// memory close to the actual download size. Chunks are copied straight into
|
|
66
|
+
// this buffer as they arrive rather than buffered in an array and copied
|
|
67
|
+
// once at the end, so the previous ~2x-of-total peak (chunk array + a
|
|
68
|
+
// freshly allocated same-size output buffer, both live during the final
|
|
69
|
+
// copy) no longer happens.
|
|
70
|
+
let buf = new Uint8Array(Number.isFinite(declared) && declared > 0 ? Math.min(declared, maxBytes) : 65536);
|
|
71
|
+
let total = 0;
|
|
72
|
+
for (;;) {
|
|
73
|
+
let done , value ;
|
|
74
|
+
try {
|
|
75
|
+
({ done, value } = await readWithIdleTimeout(reader, idleTimeoutMs));
|
|
76
|
+
} catch (err) {
|
|
77
|
+
await reader.cancel().catch(() => {});
|
|
78
|
+
throw err;
|
|
79
|
+
}
|
|
80
|
+
if (done) break;
|
|
81
|
+
if (value) {
|
|
82
|
+
const newTotal = total + value.byteLength;
|
|
83
|
+
if (newTotal > maxBytes) {
|
|
84
|
+
await reader.cancel().catch(() => {});
|
|
85
|
+
throw tooLarge();
|
|
86
|
+
}
|
|
87
|
+
if (newTotal > buf.length) {
|
|
88
|
+
const grown = new Uint8Array(Math.min(maxBytes, Math.max(newTotal, buf.length * 2)));
|
|
89
|
+
grown.set(buf);
|
|
90
|
+
buf = grown;
|
|
91
|
+
}
|
|
92
|
+
buf.set(value, total);
|
|
93
|
+
total = newTotal;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
return buf.subarray(0, total);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
// ./proxy.js is a plain, untyped .js file whose exported fetchShsEventLog
|
|
100
|
+
// TS can only infer a loose shape for. Its real (verified by reading
|
|
101
|
+
// proxy.js) contract is this
|
|
102
|
+
// discriminated union (either branch, never a mix), so it's asserted here
|
|
103
|
+
// once at the boundary rather than widening every downstream read.
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
export async function resolveFromShs(
|
|
107
|
+
shsBaseUrl ,
|
|
108
|
+
appId ,
|
|
109
|
+
attemptId ,
|
|
110
|
+
opts = {},
|
|
111
|
+
) {
|
|
112
|
+
const { fetchImpl = fetch, maxArchiveBytes = DEFAULT_MAX_ARCHIVE_BYTES, idleTimeoutMs = DEFAULT_IDLE_TIMEOUT_MS } = opts;
|
|
113
|
+
const validated = validateShsRequest({ baseUrl: shsBaseUrl, appId, attemptId: attemptId ?? '' });
|
|
114
|
+
if (!validated.request) throw mcpError('access-or-upstream-failure', 'Invalid SHS request parameters.');
|
|
115
|
+
const fetched = await fetchShsEventLog(validated.request, { fetchImpl }) ;
|
|
116
|
+
if (!fetched.ok) throw mcpError(fetched.code, `SHS fetch failed: ${fetched.code}`);
|
|
117
|
+
const zipBytes = await readArchiveBytes(fetched.upstream, maxArchiveBytes, idleTimeoutMs);
|
|
118
|
+
const { appModel, skippedLines } = await collectShsAppModel(zipBytes);
|
|
119
|
+
appModel.evidenceAvailability = deriveEvidenceAvailability(appModel, { skippedLines });
|
|
120
|
+
return appModel;
|
|
121
|
+
}
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
export const SHS_ERROR_CODES = new Set([
|
|
2
|
+
'local-server-unavailable',
|
|
3
|
+
'upstream-unreachable',
|
|
4
|
+
'application-not-found',
|
|
5
|
+
'access-or-upstream-failure',
|
|
6
|
+
'invalid-event-log',
|
|
7
|
+
]);
|
|
8
|
+
|
|
9
|
+
const APP_ID_PATTERNS = [
|
|
10
|
+
/^application_\d+_\d+$/,
|
|
11
|
+
/^local-\d+$/,
|
|
12
|
+
/^app-[A-Za-z0-9][A-Za-z0-9._-]*$/,
|
|
13
|
+
/^spark-[A-Za-z0-9][A-Za-z0-9._~-]*$/,
|
|
14
|
+
/^driver-\d+$/,
|
|
15
|
+
];
|
|
16
|
+
const ATTEMPT_ID_RE = /^(?!\.{1,2}$)[A-Za-z0-9][A-Za-z0-9._~-]*$/;
|
|
17
|
+
|
|
18
|
+
function trimString(value) {
|
|
19
|
+
return typeof value === 'string' ? value.trim() : '';
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function normalizeBaseUrl(value) {
|
|
23
|
+
const baseUrl = trimString(value);
|
|
24
|
+
if (!baseUrl || baseUrl.includes('?') || baseUrl.includes('#')) return null;
|
|
25
|
+
|
|
26
|
+
let parsed;
|
|
27
|
+
try {
|
|
28
|
+
parsed = new URL(baseUrl);
|
|
29
|
+
} catch {
|
|
30
|
+
return null;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
if (
|
|
34
|
+
(parsed.protocol !== 'http:' && parsed.protocol !== 'https:')
|
|
35
|
+
|| parsed.username
|
|
36
|
+
|| parsed.password
|
|
37
|
+
|| parsed.search
|
|
38
|
+
|| parsed.hash
|
|
39
|
+
) return null;
|
|
40
|
+
|
|
41
|
+
parsed.pathname = `${parsed.pathname.replace(/\/+$/, '')}/`;
|
|
42
|
+
return parsed.toString();
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function isValidAppId(appId) {
|
|
46
|
+
return APP_ID_PATTERNS.some((pattern) => pattern.test(appId));
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function isValidAttemptId(attemptId) {
|
|
50
|
+
return ATTEMPT_ID_RE.test(attemptId);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export function validateShsRequest({ baseUrl = '', appId = '', attemptId = '' } = {}) {
|
|
54
|
+
const normalizedBaseUrl = normalizeBaseUrl(baseUrl);
|
|
55
|
+
const normalizedAppId = trimString(appId);
|
|
56
|
+
const normalizedAttemptId = trimString(attemptId);
|
|
57
|
+
const errors = {
|
|
58
|
+
baseUrl: normalizedBaseUrl ? null : 'Enter an absolute HTTP(S) base URL without credentials, query, or fragment.',
|
|
59
|
+
appId: isValidAppId(normalizedAppId) ? null : 'Enter a supported Spark application ID.',
|
|
60
|
+
attemptId: !normalizedAttemptId || isValidAttemptId(normalizedAttemptId)
|
|
61
|
+
? null
|
|
62
|
+
: 'Enter a URL-path-safe attempt ID.',
|
|
63
|
+
};
|
|
64
|
+
|
|
65
|
+
if (errors.baseUrl || errors.appId || errors.attemptId) return { request: null, errors };
|
|
66
|
+
|
|
67
|
+
return {
|
|
68
|
+
request: {
|
|
69
|
+
baseUrl: normalizedBaseUrl,
|
|
70
|
+
appId: normalizedAppId,
|
|
71
|
+
attemptId: normalizedAttemptId || null,
|
|
72
|
+
},
|
|
73
|
+
errors,
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
export function isShsRequestValid(result) {
|
|
78
|
+
// validateShsRequest guarantees `request` is null iff any field errored,
|
|
79
|
+
// so a non-null request already implies every error is null.
|
|
80
|
+
return result?.request != null;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
export function buildProxyRequestUrl({ baseUrl, appId, attemptId }) {
|
|
84
|
+
const params = [
|
|
85
|
+
['baseUrl', baseUrl],
|
|
86
|
+
['appId', appId],
|
|
87
|
+
];
|
|
88
|
+
if (attemptId !== null) params.push(['attemptId', attemptId]);
|
|
89
|
+
return `/shs-proxy?${params.map(([key, value]) => `${key}=${encodeURIComponent(value)}`).join('&')}`;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
export function buildUpstreamUrl({ baseUrl, appId, attemptId }) {
|
|
93
|
+
const segments = ['api', 'v1', 'applications', appId];
|
|
94
|
+
if (attemptId) segments.push(attemptId);
|
|
95
|
+
segments.push('logs');
|
|
96
|
+
return new URL(segments.map(encodeURIComponent).join('/'), baseUrl).toString();
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
export function isShsErrorCode(value) {
|
|
100
|
+
return typeof value === 'string' && SHS_ERROR_CODES.has(value);
|
|
101
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
|
|
3
|
+
// Proxy-level error envelope: the JSON body the local server's SHS proxy
|
|
4
|
+
// (./shs-request.js) sends back on a non-OK upstream response, e.g.
|
|
5
|
+
// `{ code: 'shs-unreachable' }`. This is NOT a SparkListener* event shape, so
|
|
6
|
+
// it lives here rather than in event-schemas.ts. `.passthrough()` since the
|
|
7
|
+
// proxy may attach extra debugging fields the consumer doesn't care about;
|
|
8
|
+
// only `code` is read.
|
|
9
|
+
export const ShsProxyErrorBodySchema = z.object({
|
|
10
|
+
code: z.string(),
|
|
11
|
+
}).passthrough();
|
|
12
|
+
|
|
13
|
+
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
// Decoder for the framing used by Spark's SnappyCompressionCodec, which wraps
|
|
2
|
+
// org.xerial.snappy.SnappyOutputStream: an 8-byte magic + 4-byte big-endian
|
|
3
|
+
// version + 4-byte big-endian compatible-version header, followed by a
|
|
4
|
+
// sequence of [4-byte big-endian compressed length][raw Snappy block] pairs.
|
|
5
|
+
// Each raw block is the standard Snappy format (varint uncompressed length +
|
|
6
|
+
// a sequence of literal/copy tagged elements): see
|
|
7
|
+
// https://github.com/google/snappy/blob/main/format_description.txt. This is
|
|
8
|
+
// a different framing from Spark's own LZ4Block format (src/lz4-block.js).
|
|
9
|
+
|
|
10
|
+
const MAGIC = [0x82, 0x53, 0x4e, 0x41, 0x50, 0x50, 0x59, 0x00]; // 0x82 'S' 'N' 'A' 'P' 'P' 'Y' 0x00
|
|
11
|
+
const HEADER_SIZE = 16; // 8-byte magic + 4-byte version + 4-byte compatible version
|
|
12
|
+
|
|
13
|
+
function readVarint(bytes , pos ) {
|
|
14
|
+
let result = 0, shift = 0, p = pos;
|
|
15
|
+
for (;;) {
|
|
16
|
+
const b = bytes[p++];
|
|
17
|
+
result |= (b & 0x7f) << shift;
|
|
18
|
+
if ((b & 0x80) === 0) break;
|
|
19
|
+
shift += 7;
|
|
20
|
+
}
|
|
21
|
+
return { value: result >>> 0, next: p };
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function readUint32BE(bytes , offset ) {
|
|
25
|
+
return ((bytes[offset] << 24) | (bytes[offset + 1] << 16) | (bytes[offset + 2] << 8) | bytes[offset + 3]) >>> 0;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function checkMagic(bytes , pos ) {
|
|
29
|
+
for (let i = 0; i < 8; i++) {
|
|
30
|
+
if (bytes[pos + i] !== MAGIC[i]) throw new Error(`Not a Spark SnappyCodec stream: bad magic at offset ${pos}.`);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
// Decompress one raw Snappy block (varint length + literal/copy elements).
|
|
35
|
+
function decompressSnappyBlock(bytes ) {
|
|
36
|
+
const { value: outSize, next: start } = readVarint(bytes, 0);
|
|
37
|
+
const out = new Uint8Array(outSize);
|
|
38
|
+
let ip = start, op = 0;
|
|
39
|
+
const n = bytes.length;
|
|
40
|
+
while (ip < n) {
|
|
41
|
+
const tag = bytes[ip];
|
|
42
|
+
const lowType = tag & 0x3;
|
|
43
|
+
if (lowType === 0) { // literal
|
|
44
|
+
const lenTag = tag >> 2;
|
|
45
|
+
let literalLen, headerLen;
|
|
46
|
+
if (lenTag < 60) {
|
|
47
|
+
literalLen = lenTag + 1;
|
|
48
|
+
headerLen = 1;
|
|
49
|
+
} else {
|
|
50
|
+
const extraBytes = lenTag - 59;
|
|
51
|
+
let lenMinus1 = 0;
|
|
52
|
+
for (let i = 0; i < extraBytes; i++) lenMinus1 |= bytes[ip + 1 + i] << (8 * i);
|
|
53
|
+
literalLen = (lenMinus1 >>> 0) + 1;
|
|
54
|
+
headerLen = 1 + extraBytes;
|
|
55
|
+
}
|
|
56
|
+
out.set(bytes.subarray(ip + headerLen, ip + headerLen + literalLen), op);
|
|
57
|
+
ip += headerLen + literalLen;
|
|
58
|
+
op += literalLen;
|
|
59
|
+
} else if (lowType === 1) { // copy, 1-byte offset: length [4..11], offset [0..2047]
|
|
60
|
+
const length = ((tag >> 2) & 0x7) + 4;
|
|
61
|
+
const offset = ((tag & 0xe0) << 3) | bytes[ip + 1];
|
|
62
|
+
let matchPos = op - offset;
|
|
63
|
+
for (let i = 0; i < length; i++) out[op++] = out[matchPos++];
|
|
64
|
+
ip += 2;
|
|
65
|
+
} else if (lowType === 2) { // copy, 2-byte offset: length [1..64], offset [0..65535]
|
|
66
|
+
const length = (tag >> 2) + 1;
|
|
67
|
+
const offset = bytes[ip + 1] | (bytes[ip + 2] << 8);
|
|
68
|
+
let matchPos = op - offset;
|
|
69
|
+
for (let i = 0; i < length; i++) out[op++] = out[matchPos++];
|
|
70
|
+
ip += 3;
|
|
71
|
+
} else { // copy, 4-byte offset
|
|
72
|
+
const length = (tag >> 2) + 1;
|
|
73
|
+
const offset = (bytes[ip + 1] | (bytes[ip + 2] << 8) | (bytes[ip + 3] << 16) | (bytes[ip + 4] << 24)) >>> 0;
|
|
74
|
+
let matchPos = op - offset;
|
|
75
|
+
for (let i = 0; i < length; i++) out[op++] = out[matchPos++];
|
|
76
|
+
ip += 5;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
return out;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export function decodeSnappyBlock(bytes ) {
|
|
83
|
+
checkMagic(bytes, 0);
|
|
84
|
+
const chunks = [];
|
|
85
|
+
let pos = HEADER_SIZE;
|
|
86
|
+
const n = bytes.length;
|
|
87
|
+
while (pos < n) {
|
|
88
|
+
if (n - pos < 4) throw new Error(`Truncated Snappy block length at offset ${pos}.`);
|
|
89
|
+
const blockLen = readUint32BE(bytes, pos);
|
|
90
|
+
const bodyStart = pos + 4;
|
|
91
|
+
if (n - bodyStart < blockLen) throw new Error(`Truncated Snappy block body at offset ${pos}.`);
|
|
92
|
+
const body = bytes.subarray(bodyStart, bodyStart + blockLen);
|
|
93
|
+
chunks.push(decompressSnappyBlock(body));
|
|
94
|
+
pos = bodyStart + blockLen;
|
|
95
|
+
}
|
|
96
|
+
const total = chunks.reduce((sum, c) => sum + c.length, 0);
|
|
97
|
+
const out = new Uint8Array(total);
|
|
98
|
+
let offset = 0;
|
|
99
|
+
for (const c of chunks) { out.set(c, offset); offset += c.length; }
|
|
100
|
+
return out;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// Streaming counterpart: push arbitrary byte slices, each fully-received
|
|
104
|
+
// block is decompressed and handed to `onChunk` as it completes. Mirrors
|
|
105
|
+
// createLz4BlockDecoder's shape/contract (see src/lz4-block.js).
|
|
106
|
+
export function createSnappyBlockDecoder(
|
|
107
|
+
onChunk ,
|
|
108
|
+
) {
|
|
109
|
+
let buf = new Uint8Array(0);
|
|
110
|
+
let headerConsumed = false;
|
|
111
|
+
return {
|
|
112
|
+
push(chunk ) {
|
|
113
|
+
if (buf.length === 0) buf = chunk;
|
|
114
|
+
else if (chunk.length) {
|
|
115
|
+
const merged = new Uint8Array(buf.length + chunk.length);
|
|
116
|
+
merged.set(buf); merged.set(chunk, buf.length);
|
|
117
|
+
buf = merged;
|
|
118
|
+
}
|
|
119
|
+
let pos = 0;
|
|
120
|
+
if (!headerConsumed) {
|
|
121
|
+
if (buf.length < HEADER_SIZE) return;
|
|
122
|
+
checkMagic(buf, 0);
|
|
123
|
+
pos = HEADER_SIZE;
|
|
124
|
+
headerConsumed = true;
|
|
125
|
+
}
|
|
126
|
+
while (buf.length - pos >= 4) {
|
|
127
|
+
const blockLen = readUint32BE(buf, pos);
|
|
128
|
+
const bodyStart = pos + 4;
|
|
129
|
+
if (buf.length - bodyStart < blockLen) break; // block not fully arrived yet
|
|
130
|
+
const body = buf.subarray(bodyStart, bodyStart + blockLen);
|
|
131
|
+
onChunk(decompressSnappyBlock(body));
|
|
132
|
+
pos = bodyStart + blockLen;
|
|
133
|
+
}
|
|
134
|
+
buf = pos > 0 ? buf.slice(pos) : buf;
|
|
135
|
+
},
|
|
136
|
+
end() {
|
|
137
|
+
if (buf.length !== 0) throw new Error(`Trailing ${buf.length} undecoded bytes in Snappy stream.`);
|
|
138
|
+
},
|
|
139
|
+
};
|
|
140
|
+
}
|