sparkforensics-cli 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/bin/sparkforensics-analyze.mjs +288 -0
  2. package/package.json +29 -0
  3. package/vendor-core/analyzer.js +167 -0
  4. package/vendor-core/assert-never.js +3 -0
  5. package/vendor-core/cli/budgets.js +203 -0
  6. package/vendor-core/cli/collect-run.js +107 -0
  7. package/vendor-core/core-count.js +68 -0
  8. package/vendor-core/core-locality-ratio.js +54 -0
  9. package/vendor-core/core-time-series.js +92 -0
  10. package/vendor-core/core-usage-locality.js +70 -0
  11. package/vendor-core/detectors.js +1989 -0
  12. package/vendor-core/docs-config.js +72 -0
  13. package/vendor-core/docs-site-config.js +23 -0
  14. package/vendor-core/efficiency-model.js +62 -0
  15. package/vendor-core/etl-phases.js +28 -0
  16. package/vendor-core/event-handlers.js +906 -0
  17. package/vendor-core/event-schemas.js +405 -0
  18. package/vendor-core/evidence-availability.js +121 -0
  19. package/vendor-core/evidence-report.js +459 -0
  20. package/vendor-core/finding-action-label.js +97 -0
  21. package/vendor-core/finding-filter-predicate.js +38 -0
  22. package/vendor-core/format-utils.js +167 -0
  23. package/vendor-core/impact-band.js +50 -0
  24. package/vendor-core/impact-estimator.js +428 -0
  25. package/vendor-core/ingest.js +139 -0
  26. package/vendor-core/job-groups.js +30 -0
  27. package/vendor-core/load-vendored.js +24 -0
  28. package/vendor-core/lz4-block.js +135 -0
  29. package/vendor-core/mcp-error.js +3 -0
  30. package/vendor-core/mcp-server-factory.js +115 -0
  31. package/vendor-core/mcp-tools.js +331 -0
  32. package/vendor-core/model-assembler.js +76 -0
  33. package/vendor-core/occupancy.js +202 -0
  34. package/vendor-core/parser-worker.js +249 -0
  35. package/vendor-core/plan-dot.js +25 -0
  36. package/vendor-core/plan-duration-attribution.js +185 -0
  37. package/vendor-core/plan-graph-model.js +171 -0
  38. package/vendor-core/plan-node-detail.js +159 -0
  39. package/vendor-core/plan-summary.js +233 -0
  40. package/vendor-core/plan-tree-walk.js +29 -0
  41. package/vendor-core/proxy.js +157 -0
  42. package/vendor-core/recommendation-rollup.js +197 -0
  43. package/vendor-core/redact.js +175 -0
  44. package/vendor-core/rolling-log-reassembly.js +52 -0
  45. package/vendor-core/run-aggregates.js +44 -0
  46. package/vendor-core/run-comparison.js +458 -0
  47. package/vendor-core/scaling-sim.js +73 -0
  48. package/vendor-core/session-snapshot.js +79 -0
  49. package/vendor-core/shs-fetch.js +196 -0
  50. package/vendor-core/shs-load.js +121 -0
  51. package/vendor-core/shs-request.js +101 -0
  52. package/vendor-core/shs-schemas.js +13 -0
  53. package/vendor-core/snappy-block.js +140 -0
  54. package/vendor-core/stage-quantiles.js +199 -0
  55. package/vendor-core/threshold-summary.js +35 -0
  56. package/vendor-core/types.js +286 -0
  57. package/vendor-core/vendor/fflate.js +2695 -0
  58. package/vendor-core/vendor/fzstd.js +768 -0
  59. package/vendor-core/wall-clock.js +36 -0
  60. package/vendor-core/wasted-core-hours.js +68 -0
@@ -0,0 +1,196 @@
1
+ import { unzipSync, Gunzip } from './vendor/fflate.js';
2
+ import { createLz4BlockDecoder } from './lz4-block.js';
3
+ import { Decompress as ZstdDecompress } from './vendor/fzstd.js';
4
+ import { createSnappyBlockDecoder } from './snappy-block.js';
5
+ import { buildProxyRequestUrl, isShsErrorCode } from './shs-request.js';
6
+ import { dispatchLine, buildChunkDecoder, emitParseCompletion, } from './event-handlers.js';
7
+ import { ShsProxyErrorBodySchema } from './shs-schemas.js';
8
+ import { naturalCompare, reassembleRollingEntries } from './rolling-log-reassembly.js';
9
+
10
+ export { naturalCompare, reassembleRollingEntries };
11
+
12
+ // Sniff a compression codec from leading magic bytes: gzip (1f 8b), Zstandard
13
+ // (28 b5 2f fd), Spark's custom "LZ4Block" framing, or Spark's Snappy framing
14
+ // (org.xerial.snappy's "\x82SNAPPY\0" header). Returns 'gz' | 'zstd' | 'lz4' |
15
+ // 'snappy' | null. More robust than a filename suffix: a dropped SHS log may
16
+ // have no extension.
17
+ export function sniffCodec(bytes ) {
18
+ if (bytes.length >= 2 && bytes[0] === 0x1f && bytes[1] === 0x8b) return 'gz';
19
+ if (bytes.length >= 4 && bytes[0] === 0x28 && bytes[1] === 0xb5 && bytes[2] === 0x2f && bytes[3] === 0xfd) return 'zstd';
20
+ const LZ4_MAGIC = [76, 90, 52, 66, 108, 111, 99, 107]; // "LZ4Block"
21
+ if (bytes.length >= 8 && LZ4_MAGIC.every((b, i) => bytes[i] === b)) return 'lz4';
22
+ const SNAPPY_MAGIC = [0x82, 0x53, 0x4e, 0x41, 0x50, 0x50, 0x59, 0x00]; // 0x82 'S' 'N' 'A' 'P' 'P' 'Y' 0x00
23
+ if (bytes.length >= 8 && SNAPPY_MAGIC.every((b, i) => bytes[i] === b)) return 'snappy';
24
+ return null;
25
+ }
26
+
27
+ // Streams one already-in-memory zip entry through its codec's block-by-block
28
+ // decoder, invoking `onChunk` per decompressed piece; never materializes the
29
+ // whole decompressed entry as one buffer. A giant single-segment zstd frame
30
+ // (or gzip member) declares its full content size in the header; one-shotting
31
+ // it (fzstd's `decompress()`, fflate's `gunzipSync()`) allocates that entire
32
+ // size as a single ArrayBuffer up front, which can exceed what the browser
33
+ // will allocate for a multi-GB event log ("Array buffer allocation failed").
34
+ // fflate's Gunzip and fzstd's Decompress are untyped vendor JS (plain
35
+ // prototype classes, not `class` declarations), so TS can't infer a
36
+ // construct signature for them; this local shape is just enough to type
37
+ // the two call sites below without touching the vendored files.
38
+
39
+
40
+
41
+ function decodeEntry(name , raw , onChunk ) {
42
+ const codec = sniffCodec(raw);
43
+ if (codec === 'lz4' || name.endsWith('.lz4')) {
44
+ const lz4 = createLz4BlockDecoder(onChunk);
45
+ lz4.push(raw);
46
+ lz4.end();
47
+ } else if (codec === 'gz' || name.endsWith('.gz')) {
48
+ new (Gunzip )(onChunk).push(raw, true);
49
+ } else if (codec === 'zstd' || name.endsWith('.zstd') || name.endsWith('.zst')) {
50
+ new (ZstdDecompress )(onChunk).push(raw, true);
51
+ } else if (codec === 'snappy' || name.endsWith('.snappy')) {
52
+ const snappy = createSnappyBlockDecoder(onChunk);
53
+ snappy.push(raw);
54
+ snappy.end();
55
+ } else {
56
+ onChunk(raw);
57
+ }
58
+ }
59
+
60
+
61
+
62
+ function emitShsError(emit , code ) {
63
+ emit({ type: 'error', source: 'shs', code });
64
+ }
65
+
66
+ export async function runParseFromUrl(
67
+ request ,
68
+ state ,
69
+ { fetchImpl = fetch, emit = (msg ) => self.postMessage(msg) } = {}
70
+ ) {
71
+ // buildProxyRequestUrl is still-untyped JS (./shs-request.js, a plain
72
+ // untyped .js sibling); `request`'s real shape isn't pinned down here.
73
+ const url = buildProxyRequestUrl(request );
74
+
75
+ let res;
76
+ try {
77
+ res = await fetchImpl(url);
78
+ } catch {
79
+ emitShsError(emit, 'local-server-unavailable');
80
+ return;
81
+ }
82
+ if (res.status === 404) {
83
+ emitShsError(emit, 'local-server-unavailable');
84
+ return;
85
+ }
86
+ if (!res.ok) {
87
+ let code = 'access-or-upstream-failure';
88
+ try {
89
+ const body = ShsProxyErrorBodySchema.parse(await res.json());
90
+ if (isShsErrorCode(body.code)) code = body.code;
91
+ } catch { /* Malformed proxy errors, or a body that fails schema validation, retain the generic safe code: same silent-skip treatment as a log-line validation failure. */ }
92
+ emitShsError(emit, code);
93
+ return;
94
+ }
95
+
96
+ if (!res.body) {
97
+ emitShsError(emit, 'invalid-event-log');
98
+ return;
99
+ }
100
+
101
+ let zipBytes;
102
+ let total;
103
+ try {
104
+ total = Number(res.headers.get('content-length')) || null;
105
+ const reader = res.body.getReader();
106
+ const chunks = [];
107
+ let received = 0;
108
+ for (;;) {
109
+ const { done, value } = await reader.read();
110
+ if (done) break;
111
+ chunks.push(value);
112
+ received += value.length;
113
+ if (!total || received < total) {
114
+ emit({ type: 'progress', pct: total ? (received / total) * 0.5 : null, linesProcessed: 0 });
115
+ }
116
+ }
117
+ zipBytes = new Uint8Array(received);
118
+ let writeOffset = 0;
119
+ for (const chunk of chunks) { zipBytes.set(chunk, writeOffset); writeOffset += chunk.length; }
120
+ } catch {
121
+ emitShsError(emit, 'local-server-unavailable');
122
+ return;
123
+ }
124
+
125
+ const contentType = res.headers.get('content-type') || '';
126
+ if (contentType.startsWith('text/html')) {
127
+ emitShsError(emit, 'local-server-unavailable');
128
+ return;
129
+ }
130
+
131
+ // The in-loop guard above intentionally withholds the tick for the chunk that
132
+ // completes the download (received === total), to avoid double-emitting it here.
133
+ // Emit it now that we know the download is complete: this is the one place a
134
+ // determinate download is guaranteed to reach pct 0.5. Indeterminate downloads
135
+ // (no Content-Length) already got their final tick from inside the loop, since
136
+ // the loop's guard never withholds when total is null.
137
+ if (total) {
138
+ emit({ type: 'progress', pct: 0.5, linesProcessed: 0 });
139
+ }
140
+ decodeShsArchive(zipBytes, state, emit);
141
+ }
142
+
143
+ export function decodeShsArchive(zipBytes , state , emit ) {
144
+ let entries ;
145
+ try {
146
+ entries = unzipSync(zipBytes);
147
+ } catch {
148
+ emitShsError(emit, 'invalid-event-log');
149
+ return;
150
+ }
151
+ const allNames = Object.keys(entries);
152
+ const isRolling = allNames.some(n => /^events_\d+_/.test(n));
153
+ let names;
154
+ if (isRolling) {
155
+ try {
156
+ names = reassembleRollingEntries(allNames);
157
+ } catch {
158
+ emitShsError(emit, 'invalid-event-log');
159
+ return;
160
+ }
161
+ } else {
162
+ names = allNames.filter(n => n.toLowerCase() !== 'appstatus').sort(naturalCompare);
163
+ }
164
+ if (names.length === 0) {
165
+ emitShsError(emit, 'invalid-event-log');
166
+ return;
167
+ }
168
+
169
+ const decoder = buildChunkDecoder();
170
+ let linesProcessed = 0;
171
+ for (const name of names) {
172
+ try {
173
+ decodeEntry(name, entries[name], (bytes) => {
174
+ for (const line of decoder.decode(bytes)) {
175
+ dispatchLine(line, state, emit);
176
+ linesProcessed++;
177
+ if (linesProcessed % 2000 === 0) {
178
+ emit({ type: 'progress', pct: null, linesProcessed });
179
+ }
180
+ }
181
+ });
182
+ } catch {
183
+ emitShsError(emit, 'invalid-event-log');
184
+ return;
185
+ }
186
+ }
187
+ for (const line of decoder.flush()) {
188
+ dispatchLine(line, state, emit);
189
+ }
190
+
191
+ if (!state.app) {
192
+ emitShsError(emit, 'invalid-event-log');
193
+ return;
194
+ }
195
+ emitParseCompletion(state, emit, linesProcessed);
196
+ }
@@ -0,0 +1,121 @@
1
+ import { validateShsRequest } from './shs-request.js';
2
+ import { fetchShsEventLog } from './proxy.js';
3
+ import { decodeShsArchive } from './parser-worker.js';
4
+ import { collectViaDispatch } from './cli/collect-run.js';
5
+ import { deriveEvidenceAvailability } from './evidence-availability.js';
6
+ import { mcpError } from './mcp-error.js';
7
+
8
+
9
+ function envInt(name , fallback ) {
10
+ const v = Number(process.env[name]);
11
+ return Number.isFinite(v) && v > 0 ? v : fallback;
12
+ }
13
+
14
+ export const DEFAULT_MAX_ARCHIVE_BYTES = envInt('SPARKFORENSICS_MAX_ARCHIVE_BYTES', 1024 * 1024 * 1024);
15
+ // Same knob the proxy uses: fetchShsEventLog covers the header phase; this
16
+ // covers the body, per chunk, so progressing downloads of any size are fine.
17
+ export const DEFAULT_IDLE_TIMEOUT_MS = envInt('SPARKFORENSICS_SHS_TIMEOUT_MS', 30_000);
18
+
19
+ function collectShsAppModel(zipBytes ) {
20
+ return collectViaDispatch(
21
+ (state, emit) => decodeShsArchive(zipBytes, state, emit),
22
+ (msg) => {
23
+ const m = msg ;
24
+ return mcpError(m?.code ?? 'invalid-event-log', m?.message ?? 'Failed to decode SHS archive.');
25
+ },
26
+ );
27
+ }
28
+
29
+ // Races one reader.read() against a fresh idle timer. The losing read() stays
30
+ // pending after a timeout; reader.cancel() in the caller settles it.
31
+ function readWithIdleTimeout(
32
+ reader ,
33
+ idleTimeoutMs ,
34
+ ) {
35
+ let timer ;
36
+ const stalled = new Promise((_resolve, reject) => {
37
+ timer = setTimeout(() => reject(mcpError(
38
+ 'upstream-unreachable',
39
+ `SHS archive body stalled for ${idleTimeoutMs} ms (override with SPARKFORENSICS_SHS_TIMEOUT_MS).`,
40
+ )), idleTimeoutMs);
41
+ });
42
+ return Promise.race([reader.read(), stalled]).finally(() => clearTimeout(timer));
43
+ }
44
+
45
+ // Reads the archive body with a byte cap: an unbounded arrayBuffer() on a
46
+ // hostile or misconfigured SHS response would OOM the long-running process.
47
+ // The per-chunk idle timeout keeps a stalled body from hanging the call
48
+ // forever with headers already received.
49
+ async function readArchiveBytes(upstream , maxBytes , idleTimeoutMs ) {
50
+ const tooLarge = () => mcpError(
51
+ 'archive-too-large',
52
+ `SHS archive exceeds the ${maxBytes}-byte cap (override with SPARKFORENSICS_MAX_ARCHIVE_BYTES).`,
53
+ );
54
+ const declared = Number(upstream.headers?.get?.('content-length'));
55
+ if (Number.isFinite(declared) && declared > maxBytes) throw tooLarge();
56
+
57
+ // fetchShsEventLog (./proxy.js, a plain untyped .js sibling)
58
+ // already checks `!upstream.body` and returns `{ ok: false }` before ever
59
+ // returning `{ ok: true, upstream }`, so `body` is guaranteed present here
60
+ // even though DOM's Response.body type is nullable.
61
+ const reader = upstream.body .getReader();
62
+ // Grown geometrically instead of preallocated at `maxBytes`: a typical
63
+ // archive is nowhere near the cap, so starting small (or at the declared
64
+ // content-length, when trustworthy) and doubling on demand keeps peak
65
+ // memory close to the actual download size. Chunks are copied straight into
66
+ // this buffer as they arrive rather than buffered in an array and copied
67
+ // once at the end, so the previous ~2x-of-total peak (chunk array + a
68
+ // freshly allocated same-size output buffer, both live during the final
69
+ // copy) no longer happens.
70
+ let buf = new Uint8Array(Number.isFinite(declared) && declared > 0 ? Math.min(declared, maxBytes) : 65536);
71
+ let total = 0;
72
+ for (;;) {
73
+ let done , value ;
74
+ try {
75
+ ({ done, value } = await readWithIdleTimeout(reader, idleTimeoutMs));
76
+ } catch (err) {
77
+ await reader.cancel().catch(() => {});
78
+ throw err;
79
+ }
80
+ if (done) break;
81
+ if (value) {
82
+ const newTotal = total + value.byteLength;
83
+ if (newTotal > maxBytes) {
84
+ await reader.cancel().catch(() => {});
85
+ throw tooLarge();
86
+ }
87
+ if (newTotal > buf.length) {
88
+ const grown = new Uint8Array(Math.min(maxBytes, Math.max(newTotal, buf.length * 2)));
89
+ grown.set(buf);
90
+ buf = grown;
91
+ }
92
+ buf.set(value, total);
93
+ total = newTotal;
94
+ }
95
+ }
96
+ return buf.subarray(0, total);
97
+ }
98
+
99
+ // ./proxy.js is a plain, untyped .js file whose exported fetchShsEventLog
100
+ // TS can only infer a loose shape for. Its real (verified by reading
101
+ // proxy.js) contract is this
102
+ // discriminated union (either branch, never a mix), so it's asserted here
103
+ // once at the boundary rather than widening every downstream read.
104
+
105
+
106
+ export async function resolveFromShs(
107
+ shsBaseUrl ,
108
+ appId ,
109
+ attemptId ,
110
+ opts = {},
111
+ ) {
112
+ const { fetchImpl = fetch, maxArchiveBytes = DEFAULT_MAX_ARCHIVE_BYTES, idleTimeoutMs = DEFAULT_IDLE_TIMEOUT_MS } = opts;
113
+ const validated = validateShsRequest({ baseUrl: shsBaseUrl, appId, attemptId: attemptId ?? '' });
114
+ if (!validated.request) throw mcpError('access-or-upstream-failure', 'Invalid SHS request parameters.');
115
+ const fetched = await fetchShsEventLog(validated.request, { fetchImpl }) ;
116
+ if (!fetched.ok) throw mcpError(fetched.code, `SHS fetch failed: ${fetched.code}`);
117
+ const zipBytes = await readArchiveBytes(fetched.upstream, maxArchiveBytes, idleTimeoutMs);
118
+ const { appModel, skippedLines } = await collectShsAppModel(zipBytes);
119
+ appModel.evidenceAvailability = deriveEvidenceAvailability(appModel, { skippedLines });
120
+ return appModel;
121
+ }
@@ -0,0 +1,101 @@
1
+ export const SHS_ERROR_CODES = new Set([
2
+ 'local-server-unavailable',
3
+ 'upstream-unreachable',
4
+ 'application-not-found',
5
+ 'access-or-upstream-failure',
6
+ 'invalid-event-log',
7
+ ]);
8
+
9
+ const APP_ID_PATTERNS = [
10
+ /^application_\d+_\d+$/,
11
+ /^local-\d+$/,
12
+ /^app-[A-Za-z0-9][A-Za-z0-9._-]*$/,
13
+ /^spark-[A-Za-z0-9][A-Za-z0-9._~-]*$/,
14
+ /^driver-\d+$/,
15
+ ];
16
+ const ATTEMPT_ID_RE = /^(?!\.{1,2}$)[A-Za-z0-9][A-Za-z0-9._~-]*$/;
17
+
18
+ function trimString(value) {
19
+ return typeof value === 'string' ? value.trim() : '';
20
+ }
21
+
22
+ function normalizeBaseUrl(value) {
23
+ const baseUrl = trimString(value);
24
+ if (!baseUrl || baseUrl.includes('?') || baseUrl.includes('#')) return null;
25
+
26
+ let parsed;
27
+ try {
28
+ parsed = new URL(baseUrl);
29
+ } catch {
30
+ return null;
31
+ }
32
+
33
+ if (
34
+ (parsed.protocol !== 'http:' && parsed.protocol !== 'https:')
35
+ || parsed.username
36
+ || parsed.password
37
+ || parsed.search
38
+ || parsed.hash
39
+ ) return null;
40
+
41
+ parsed.pathname = `${parsed.pathname.replace(/\/+$/, '')}/`;
42
+ return parsed.toString();
43
+ }
44
+
45
+ function isValidAppId(appId) {
46
+ return APP_ID_PATTERNS.some((pattern) => pattern.test(appId));
47
+ }
48
+
49
+ function isValidAttemptId(attemptId) {
50
+ return ATTEMPT_ID_RE.test(attemptId);
51
+ }
52
+
53
+ export function validateShsRequest({ baseUrl = '', appId = '', attemptId = '' } = {}) {
54
+ const normalizedBaseUrl = normalizeBaseUrl(baseUrl);
55
+ const normalizedAppId = trimString(appId);
56
+ const normalizedAttemptId = trimString(attemptId);
57
+ const errors = {
58
+ baseUrl: normalizedBaseUrl ? null : 'Enter an absolute HTTP(S) base URL without credentials, query, or fragment.',
59
+ appId: isValidAppId(normalizedAppId) ? null : 'Enter a supported Spark application ID.',
60
+ attemptId: !normalizedAttemptId || isValidAttemptId(normalizedAttemptId)
61
+ ? null
62
+ : 'Enter a URL-path-safe attempt ID.',
63
+ };
64
+
65
+ if (errors.baseUrl || errors.appId || errors.attemptId) return { request: null, errors };
66
+
67
+ return {
68
+ request: {
69
+ baseUrl: normalizedBaseUrl,
70
+ appId: normalizedAppId,
71
+ attemptId: normalizedAttemptId || null,
72
+ },
73
+ errors,
74
+ };
75
+ }
76
+
77
+ export function isShsRequestValid(result) {
78
+ // validateShsRequest guarantees `request` is null iff any field errored,
79
+ // so a non-null request already implies every error is null.
80
+ return result?.request != null;
81
+ }
82
+
83
+ export function buildProxyRequestUrl({ baseUrl, appId, attemptId }) {
84
+ const params = [
85
+ ['baseUrl', baseUrl],
86
+ ['appId', appId],
87
+ ];
88
+ if (attemptId !== null) params.push(['attemptId', attemptId]);
89
+ return `/shs-proxy?${params.map(([key, value]) => `${key}=${encodeURIComponent(value)}`).join('&')}`;
90
+ }
91
+
92
+ export function buildUpstreamUrl({ baseUrl, appId, attemptId }) {
93
+ const segments = ['api', 'v1', 'applications', appId];
94
+ if (attemptId) segments.push(attemptId);
95
+ segments.push('logs');
96
+ return new URL(segments.map(encodeURIComponent).join('/'), baseUrl).toString();
97
+ }
98
+
99
+ export function isShsErrorCode(value) {
100
+ return typeof value === 'string' && SHS_ERROR_CODES.has(value);
101
+ }
@@ -0,0 +1,13 @@
1
+ import { z } from 'zod';
2
+
3
+ // Proxy-level error envelope: the JSON body the local server's SHS proxy
4
+ // (./shs-request.js) sends back on a non-OK upstream response, e.g.
5
+ // `{ code: 'shs-unreachable' }`. This is NOT a SparkListener* event shape, so
6
+ // it lives here rather than in event-schemas.ts. `.passthrough()` since the
7
+ // proxy may attach extra debugging fields the consumer doesn't care about;
8
+ // only `code` is read.
9
+ export const ShsProxyErrorBodySchema = z.object({
10
+ code: z.string(),
11
+ }).passthrough();
12
+
13
+
@@ -0,0 +1,140 @@
1
+ // Decoder for the framing used by Spark's SnappyCompressionCodec, which wraps
2
+ // org.xerial.snappy.SnappyOutputStream: an 8-byte magic + 4-byte big-endian
3
+ // version + 4-byte big-endian compatible-version header, followed by a
4
+ // sequence of [4-byte big-endian compressed length][raw Snappy block] pairs.
5
+ // Each raw block is the standard Snappy format (varint uncompressed length +
6
+ // a sequence of literal/copy tagged elements): see
7
+ // https://github.com/google/snappy/blob/main/format_description.txt. This is
8
+ // a different framing from Spark's own LZ4Block format (src/lz4-block.js).
9
+
10
+ const MAGIC = [0x82, 0x53, 0x4e, 0x41, 0x50, 0x50, 0x59, 0x00]; // 0x82 'S' 'N' 'A' 'P' 'P' 'Y' 0x00
11
+ const HEADER_SIZE = 16; // 8-byte magic + 4-byte version + 4-byte compatible version
12
+
13
+ function readVarint(bytes , pos ) {
14
+ let result = 0, shift = 0, p = pos;
15
+ for (;;) {
16
+ const b = bytes[p++];
17
+ result |= (b & 0x7f) << shift;
18
+ if ((b & 0x80) === 0) break;
19
+ shift += 7;
20
+ }
21
+ return { value: result >>> 0, next: p };
22
+ }
23
+
24
+ function readUint32BE(bytes , offset ) {
25
+ return ((bytes[offset] << 24) | (bytes[offset + 1] << 16) | (bytes[offset + 2] << 8) | bytes[offset + 3]) >>> 0;
26
+ }
27
+
28
+ function checkMagic(bytes , pos ) {
29
+ for (let i = 0; i < 8; i++) {
30
+ if (bytes[pos + i] !== MAGIC[i]) throw new Error(`Not a Spark SnappyCodec stream: bad magic at offset ${pos}.`);
31
+ }
32
+ }
33
+
34
+ // Decompress one raw Snappy block (varint length + literal/copy elements).
35
+ function decompressSnappyBlock(bytes ) {
36
+ const { value: outSize, next: start } = readVarint(bytes, 0);
37
+ const out = new Uint8Array(outSize);
38
+ let ip = start, op = 0;
39
+ const n = bytes.length;
40
+ while (ip < n) {
41
+ const tag = bytes[ip];
42
+ const lowType = tag & 0x3;
43
+ if (lowType === 0) { // literal
44
+ const lenTag = tag >> 2;
45
+ let literalLen, headerLen;
46
+ if (lenTag < 60) {
47
+ literalLen = lenTag + 1;
48
+ headerLen = 1;
49
+ } else {
50
+ const extraBytes = lenTag - 59;
51
+ let lenMinus1 = 0;
52
+ for (let i = 0; i < extraBytes; i++) lenMinus1 |= bytes[ip + 1 + i] << (8 * i);
53
+ literalLen = (lenMinus1 >>> 0) + 1;
54
+ headerLen = 1 + extraBytes;
55
+ }
56
+ out.set(bytes.subarray(ip + headerLen, ip + headerLen + literalLen), op);
57
+ ip += headerLen + literalLen;
58
+ op += literalLen;
59
+ } else if (lowType === 1) { // copy, 1-byte offset: length [4..11], offset [0..2047]
60
+ const length = ((tag >> 2) & 0x7) + 4;
61
+ const offset = ((tag & 0xe0) << 3) | bytes[ip + 1];
62
+ let matchPos = op - offset;
63
+ for (let i = 0; i < length; i++) out[op++] = out[matchPos++];
64
+ ip += 2;
65
+ } else if (lowType === 2) { // copy, 2-byte offset: length [1..64], offset [0..65535]
66
+ const length = (tag >> 2) + 1;
67
+ const offset = bytes[ip + 1] | (bytes[ip + 2] << 8);
68
+ let matchPos = op - offset;
69
+ for (let i = 0; i < length; i++) out[op++] = out[matchPos++];
70
+ ip += 3;
71
+ } else { // copy, 4-byte offset
72
+ const length = (tag >> 2) + 1;
73
+ const offset = (bytes[ip + 1] | (bytes[ip + 2] << 8) | (bytes[ip + 3] << 16) | (bytes[ip + 4] << 24)) >>> 0;
74
+ let matchPos = op - offset;
75
+ for (let i = 0; i < length; i++) out[op++] = out[matchPos++];
76
+ ip += 5;
77
+ }
78
+ }
79
+ return out;
80
+ }
81
+
82
+ export function decodeSnappyBlock(bytes ) {
83
+ checkMagic(bytes, 0);
84
+ const chunks = [];
85
+ let pos = HEADER_SIZE;
86
+ const n = bytes.length;
87
+ while (pos < n) {
88
+ if (n - pos < 4) throw new Error(`Truncated Snappy block length at offset ${pos}.`);
89
+ const blockLen = readUint32BE(bytes, pos);
90
+ const bodyStart = pos + 4;
91
+ if (n - bodyStart < blockLen) throw new Error(`Truncated Snappy block body at offset ${pos}.`);
92
+ const body = bytes.subarray(bodyStart, bodyStart + blockLen);
93
+ chunks.push(decompressSnappyBlock(body));
94
+ pos = bodyStart + blockLen;
95
+ }
96
+ const total = chunks.reduce((sum, c) => sum + c.length, 0);
97
+ const out = new Uint8Array(total);
98
+ let offset = 0;
99
+ for (const c of chunks) { out.set(c, offset); offset += c.length; }
100
+ return out;
101
+ }
102
+
103
+ // Streaming counterpart: push arbitrary byte slices, each fully-received
104
+ // block is decompressed and handed to `onChunk` as it completes. Mirrors
105
+ // createLz4BlockDecoder's shape/contract (see src/lz4-block.js).
106
+ export function createSnappyBlockDecoder(
107
+ onChunk ,
108
+ ) {
109
+ let buf = new Uint8Array(0);
110
+ let headerConsumed = false;
111
+ return {
112
+ push(chunk ) {
113
+ if (buf.length === 0) buf = chunk;
114
+ else if (chunk.length) {
115
+ const merged = new Uint8Array(buf.length + chunk.length);
116
+ merged.set(buf); merged.set(chunk, buf.length);
117
+ buf = merged;
118
+ }
119
+ let pos = 0;
120
+ if (!headerConsumed) {
121
+ if (buf.length < HEADER_SIZE) return;
122
+ checkMagic(buf, 0);
123
+ pos = HEADER_SIZE;
124
+ headerConsumed = true;
125
+ }
126
+ while (buf.length - pos >= 4) {
127
+ const blockLen = readUint32BE(buf, pos);
128
+ const bodyStart = pos + 4;
129
+ if (buf.length - bodyStart < blockLen) break; // block not fully arrived yet
130
+ const body = buf.subarray(bodyStart, bodyStart + blockLen);
131
+ onChunk(decompressSnappyBlock(body));
132
+ pos = bodyStart + blockLen;
133
+ }
134
+ buf = pos > 0 ? buf.slice(pos) : buf;
135
+ },
136
+ end() {
137
+ if (buf.length !== 0) throw new Error(`Trailing ${buf.length} undecoded bytes in Snappy stream.`);
138
+ },
139
+ };
140
+ }