sparkforensics-cli 0.1.0 → 0.2.0

package/vendor-core/list-runs.js

@@ -0,0 +1,278 @@
+import { existsSync, readdirSync, statSync } from 'node:fs';
+import { resolve as resolvePath, join } from 'node:path';
+import { nodeLazyPeekableFromPath, isRollingLogDirectory } from './cli/collect-run.js';
+import { reassembleRollingEntries } from './parser-worker.js';
+import { peekLogHeader } from './log-header-peek.js';
+import { mcpError } from './mcp-error.js';
+import { normalizeBaseUrl } from './shs-request.js';
+import { DEFAULT_IDLE_TIMEOUT_MS, DEFAULT_MAX_ARCHIVE_BYTES } from './shs-load.js';
+
+                               
+                
+               
+                        
+                     
+                      
+                                                                                                           
+ 
+
+                                 
+                       
+                     
+ 
+
+                           
+                       
+                   
+                   
+                      
+                   
+ 
+
+// Local-mode-only convention: a plain file named `<name>_<n>.<ext>` is one attempt of a
+// multi-attempt run (e.g. `external-local-1430917381535_1.ndjson`), the `_<n>` disambiguating two
+// otherwise-identical appIds. Deliberately anchored on a file extension so a rolling-log
+// directory's own name (no extension) never matches this convention.
+const LOCAL_ATTEMPT_SUFFIX_RE = /^.+_(\d+)\.[^./\\]+$/;
+
+const DEFAULT_MAX_RESULTS = 100;
+
+// SHS's own timestamp format ends in a literal "GMT" suffix instead of "Z"/an offset, which
+// Date.parse cannot parse (returns NaN), normalize before storing, so minDate/maxDate filtering
+// (which relies on Date.parse) actually works, and so RunListEntry.startTime has one parseable
+// ISO format across both local and SHS mode.
+function normalizeShsTimestamp(raw        )                     {
+  if (!Number.isNaN(Date.parse(raw))) return new Date(raw).toISOString();
+  if (raw.endsWith('GMT')) {
+    const iso = `${raw.slice(0, -3)}Z`;
+    if (!Number.isNaN(Date.parse(iso))) return new Date(iso).toISOString();
+  }
+  return undefined; // genuinely unparseable: omit rather than propagate garbage
+}
+
+// Shared by local and SHS mode: both mapping steps produce a flat RunListEntry[] before this runs.
+export function applyFiltersAndCap(entries                , filters                 )                 {
+  // Date.parse of an unparseable string returns NaN, and every comparison against NaN is false,
+  // left unchecked, a typo'd minDate/maxDate would silently filter out every run rather than
+  // reporting the mistake, indistinguishable from a correctly-filtered "no runs matched" result.
+  const min = filters.minDate ? Date.parse(filters.minDate) : undefined;
+  if (min != null && Number.isNaN(min)) {
+    throw mcpError('invalid-date-filter', `minDate is not a parseable date: ${filters.minDate}`);
+  }
+  const max = filters.maxDate ? Date.parse(filters.maxDate) : undefined;
+  if (max != null && Number.isNaN(max)) {
+    throw mcpError('invalid-date-filter', `maxDate is not a parseable date: ${filters.maxDate}`);
+  }
+
+  let filtered = entries;
+  if (filters.namePattern) {
+    const needle = filters.namePattern.toLowerCase();
+    filtered = filtered.filter((e) => e.name.toLowerCase().includes(needle));
+  }
+  if (min != null) {
+    filtered = filtered.filter((e) => e.startTime != null && Date.parse(e.startTime) >= min);
+  }
+  if (max != null) {
+    filtered = filtered.filter((e) => e.startTime != null && Date.parse(e.startTime) <= max);
+  }
+  const maxResults = filters.maxResults ?? DEFAULT_MAX_RESULTS;
+  const capped = filtered.length <= maxResults
+    ? { runs: filtered, truncated: false }
+    : { runs: filtered.slice(0, maxResults), truncated: true };
+  return filters.redact ? { ...capped, runs: redactRunListEntries(capped.runs) } : capped;
+}
+
+// Local counterpart to redact.ts's redactAppIdentity, sized for a listing of many distinct apps
+// rather than the single app that tool works over: redactAppIdentity has no way to keep two
+// entries sharing an appId in sync, so this builds its own stable per-distinct-appId pseudonym map
+// (numeric-aware sort, same scheme as redact.ts's buildMap) across the whole result set, two
+// attempts of the same app redact to the same identity, and applies it to every identity-bearing
+// field: appId, name (a listing's human-readable name is as identifying as the id itself), and the
+// source's own path/appId.
+function redactRunListEntries(entries                )                 {
+  const appIds = new Set        ();
+  for (const e of entries) appIds.add(e.appId);
+  const sorted = [...appIds].sort((a, b) => a.localeCompare(b, 'en', { numeric: true }));
+  const pseudonyms = new Map(sorted.map((id, i) => [id, `app-${i + 1}`]));
+  return entries.map((e) => {
+    const pseudonym = pseudonyms.get(e.appId) ?? e.appId;
+    const source = 'path' in e.source
+      ? { ...e.source, path: pseudonym }
+      : { ...e.source, appId: pseudonym };
+    return { ...e, appId: pseudonym, name: pseudonym, source };
+  });
+}
+
+export async function listRunsLocal(input                                   )                          {
+  const dir = resolvePath(input.dir);
+  if (!existsSync(dir) || !statSync(dir).isDirectory()) {
+    throw mcpError('directory-not-found', `No such directory: ${dir}`);
+  }
+
+  let names          ;
+  try {
+    names = readdirSync(dir);
+  } catch {
+    throw mcpError('directory-not-found', `Could not read directory: ${dir}`);
+  }
+
+  // `path` is the entry's own source-worthy location (a file, or a rolling-log dir); `peekPath` is
+  // the single file peekLogHeader actually reads (the rolling dir's earliest segment). `attemptId`
+  // is only ever set for the plain-file case (see LOCAL_ATTEMPT_SUFFIX_RE), a rolling-log
+  // directory's name has no file extension, so it never matches the convention.
+  const candidates                                                                = [];
+  for (const name of names) {
+    const full = join(dir, name);
+    let st;
+    try {
+      st = statSync(full);
+    } catch {
+      continue; // vanished between readdir and stat: skip
+    }
+    if (st.isFile()) {
+      const attemptMatch = name.match(LOCAL_ATTEMPT_SUFFIX_RE);
+      candidates.push({ path: full, peekPath: full, ...(attemptMatch ? { attemptId: attemptMatch[1] } : {}) });
+    } else if (st.isDirectory()) {
+      let isRolling = false;
+      try {
+        isRolling = isRollingLogDirectory(full);
+      } catch {
+        continue; // permission denied or other error reading the directory: skip this candidate
+      }
+      if (!isRolling) continue; // not a rolling log directory: skip
+      let subNames          ;
+      try {
+        subNames = readdirSync(full);
+      } catch {
+        continue;
+      }
+      let ordered          ;
+      try {
+        ordered = reassembleRollingEntries(subNames);
+      } catch {
+        continue; // malformed rolling dir (index gap, etc.): skip like any unpeekable candidate
+      }
+      if (ordered.length === 0) continue;
+      candidates.push({ path: full, peekPath: join(full, ordered[0]) });
+    }
+    // Anything else (a plain non-rolling subdirectory, a socket, ...): not a candidate, skip.
+  }
+
+  const entries                 = [];
+  for (const c of candidates) {
+    let peeked;
+    try {
+      peeked = await peekLogHeader(nodeLazyPeekableFromPath(c.peekPath));
+    } catch {
+      continue; // corrupt/unreadable candidate: one bad file shouldn't fail the whole listing
+    }
+    if (!peeked || !peeked.appId) continue; // no usable appId: not enough to hand back as a candidate
+    entries.push({
+      appId: peeked.appId,
+      name: peeked.name ?? '',
+      ...(peeked.sparkVersion ? { sparkVersion: peeked.sparkVersion } : {}),
+      ...(peeked.startTimeMs != null ? { startTime: new Date(peeked.startTimeMs).toISOString() } : {}),
+      source: { path: c.path, ...(c.attemptId ? { attemptId: c.attemptId } : {}) },
+    });
+  }
+
+  // readdir order is arbitrary, and applyFiltersAndCap keeps the first maxResults, without a sort
+  // a directory of thousands of logs would hand back an arbitrary 100 rather than the newest 100.
+  // Newest-first also matches SHS mode, which inherits that order from SHS's own attempt ordering.
+  entries.sort((a, b) => {
+    if (a.startTime == null && b.startTime == null) return 0;
+    if (a.startTime == null) return 1; // undated entries sort last, after every dated one
+    if (b.startTime == null) return -1;
+    return Date.parse(b.startTime) - Date.parse(a.startTime);
+  });
+
+  return applyFiltersAndCap(entries, input);
+}
+
+export async function listRunsShs(
+  input                                          ,
+  opts                               ,
+)                          {
+  const fetchImpl = opts?.fetchImpl ?? fetch;
+  const normalized = normalizeBaseUrl(input.shsBaseUrl);
+  if (!normalized) {
+    throw mcpError('invalid-shs-base-url', 'Enter an absolute HTTP(S) base URL without credentials, query, or fragment.');
+  }
+
+  const url = new URL('api/v1/applications', normalized);
+  if (input.minDate) url.searchParams.set('minDate', input.minDate);
+  if (input.maxDate) url.searchParams.set('maxDate', input.maxDate);
+
+  let res          ;
+  try {
+    // An unresponsive SHS would otherwise hang the tool call forever. A timed-out signal rejects
+    // the fetch with an AbortError, which this same catch turns into access-or-upstream-failure.
+    res = await fetchImpl(url.toString(), { signal: AbortSignal.timeout(DEFAULT_IDLE_TIMEOUT_MS) });
+  } catch (e) {
+    throw mcpError('access-or-upstream-failure', `Could not reach ${normalized}: ${e instanceof Error ? e.message : String(e)}`);
+  }
+  if (!res.ok) {
+    throw mcpError('access-or-upstream-failure', `SHS applications list request failed with status ${res.status}.`);
+  }
+  // Best-effort size guard: this endpoint returns a modest JSON listing, so a declared length over
+  // the archive cap means something is wrong. Absent (e.g. chunked) header: skip the check rather
+  // than pull in a streaming reader for a listing.
+  const declaredLength = Number.parseInt(res.headers.get('content-length') ?? '', 10);
+  if (Number.isFinite(declaredLength) && declaredLength > DEFAULT_MAX_ARCHIVE_BYTES) {
+    throw mcpError('access-or-upstream-failure', 'SHS applications list response exceeds the byte cap.');
+  }
+
+  let payload         ;
+  try {
+    payload = await res.json();
+  } catch {
+    throw mcpError('access-or-upstream-failure', 'SHS applications list response was not valid JSON.');
+  }
+  if (!Array.isArray(payload)) {
+    throw mcpError('access-or-upstream-failure', 'SHS applications list response was not an array.');
+  }
+
+  const entries                 = [];
+  for (const app of payload                                  ) {
+    if (!app || typeof app !== 'object') continue;
+    const id = typeof app.id === 'string' ? app.id : null;
+    if (!id) continue;
+    const name = typeof app.name === 'string' ? app.name : '';
+    const attempts = Array.isArray(app.attempts) ? app.attempts                                   : [];
+    const attempt = attempts[0]; // SHS's own ordering: most recent attempt first
+    if (!attempt) continue;
+    const sparkVersion = typeof attempt.appSparkVersion === 'string' ? attempt.appSparkVersion : undefined;
+    const startTime = typeof attempt.startTime === 'string' ? normalizeShsTimestamp(attempt.startTime) : undefined;
+    const durationMs = typeof attempt.duration === 'number' ? attempt.duration : undefined;
+    const attemptId = typeof attempt.attemptId === 'string' ? attempt.attemptId : undefined;
+    entries.push({
+      appId: id,
+      name,
+      ...(sparkVersion ? { sparkVersion } : {}),
+      ...(startTime ? { startTime } : {}),
+      ...(durationMs != null ? { durationMs } : {}),
+      source: { shsBaseUrl: normalized, appId: id, ...(attemptId ? { attemptId } : {}) },
+    });
+  }
+
+  return applyFiltersAndCap(entries, input);
+}
+
+export async function listRuns(
+  input                                                         ,
+  opts                               ,
+)                          {
+  if (input.dir) {
+    return listRunsLocal({
+      dir: input.dir, namePattern: input.namePattern, minDate: input.minDate, maxDate: input.maxDate, maxResults: input.maxResults, redact: input.redact,
+    });
+  }
+  if (input.shsBaseUrl) {
+    return listRunsShs(
+      { shsBaseUrl: input.shsBaseUrl, namePattern: input.namePattern, minDate: input.minDate, maxDate: input.maxDate, maxResults: input.maxResults, redact: input.redact },
+      opts,
+    );
+  }
+  throw mcpError('access-or-upstream-failure', 'Provide either dir or shsBaseUrl.');
+}
+

package/vendor-core/load-vendored.js

@@ -1,19 +1,13 @@
-// Plain JS (like proxy.js): scripts/vendor-core.mjs copies it byte-for-byte
-// into vendor-core/, so this exact file exists at both
-// vendor-core/load-vendored.js (published install) and
-// core/src/load-vendored.js (monorepo dev). Each published package's bin
-// entry point bootstraps by locating *this* file first (a tiny fixed-name
-// existsSync/join check, unavoidably duplicated per entry point since
-// nothing can resolve it for them), then uses the exports below for every
-// other core module, so the actual resolution logic lives in exactly one
-// place instead of being re-implemented per entry point.
+// Plain JS: scripts/vendor-core.mjs copies it byte-for-byte into vendor-core/, so this file exists
+// at both vendor-core/load-vendored.js (published) and core/src/load-vendored.js (dev). Each
+// package's bin bootstraps by locating this file first, then uses the exports below for every other
+// core module, so the resolution logic lives in one place instead of per entry point.
 import { existsSync } from 'node:fs';
 import { join } from 'node:path';
 import { pathToFileURL } from 'node:url';
 
-// moduleName is a path relative to core/src without extension, e.g.
-// 'cli/collect-run' or 'shs-load'. Pass srcExt: 'js' for modules that are
-// already plain JS in core/src (e.g. 'proxy') rather than TypeScript.
+// moduleName is a path relative to core/src without extension, e.g. 'cli/collect-run'. Pass
+// srcExt: 'js' for modules already plain JS in core/src (e.g. 'proxy') rather than TypeScript.
 export function resolveVendored(pkgDir, moduleName, { srcExt = 'ts' } = {}) {
   const vendored = join(pkgDir, 'vendor-core', `${moduleName}.js`);
   return existsSync(vendored) ? vendored : join(pkgDir, '..', 'core', 'src', `${moduleName}.${srcExt}`);

package/vendor-core/log-header-peek.js

@@ -0,0 +1,81 @@
+import { streamFile } from './parser-worker.js';
+import { buildChunkDecoder } from './event-handlers.js';
+
+                     
+               
+               
+                                                                             
+  
+
+                                
+                       
+                      
+                              
+                             
+ 
+
+// list_runs peeks every candidate file in a shared directory: bound the work per file so one
+// huge or malformed log can't blow out a directory scan. 50 lines is comfortably past where
+// LogStart/ApplicationStart appear in a real log (they're always among the first few events).
+const MAX_PEEK_LINES = 50;
+// Small on purpose: unlike the full parse path (CHUNK_SIZE = 512KB, tuned for progress-bar
+// granularity on multi-GB files), a peek wants the *first* read to already be enough: reading in
+// 64KB steps means the 50-line cap trips after very little decompression work, not after one
+// giant chunk decodes thousands of lines at once.
+const PEEK_CHUNK_BYTES = 64 * 1024;
+
+// Thrown from inside the streamFile->decoder->onChunk callback chain to unwind out of the
+// (possibly still-looping) decompression stream the instant enough header lines are seen: the
+// rest of the file is never requested from `file.slice()` after this.
+class PeekComplete extends Error {}
+
+export async function peekLogHeader(file              )                                {
+  if (file.size === 0) return null;
+
+  const decoder = buildChunkDecoder();
+  let linesRead = 0;
+  let sawLogStart = false;
+  let sawAppStart = false;
+  const result                = { appId: null, name: null, sparkVersion: null, startTimeMs: null };
+
+  const processLine = (line        ) => {
+    linesRead++;
+    let event                                 = null;
+    try {
+      event = JSON.parse(line);
+    } catch {
+      // Corrupt/partial line: same silent-skip as dispatchLine's state.skippedLines++ path.
+    }
+    if (event) {
+      if (event.Event === 'SparkListenerLogStart') {
+        if (typeof event['Spark Version'] === 'string') result.sparkVersion = event['Spark Version'];
+        sawLogStart = true;
+      } else if (event.Event === 'SparkListenerApplicationStart') {
+        if (typeof event['App ID'] === 'string') result.appId = event['App ID'];
+        if (typeof event['App Name'] === 'string') result.name = event['App Name'];
+        if (typeof event['Timestamp'] === 'number') result.startTimeMs = event['Timestamp'];
+        // ApplicationStart's own Spark Version is only a fallback: a preceding LogStart always
+        // wins, mirroring event-handlers.ts's startApplication precedence
+        // (state.pendingSparkVersion ?? event['Spark Version']).
+        if (!sawLogStart && typeof event['Spark Version'] === 'string') result.sparkVersion = event['Spark Version'];
+        sawAppStart = true;
+      }
+    }
+    if ((sawLogStart && sawAppStart) || linesRead >= MAX_PEEK_LINES) throw new PeekComplete();
+  };
+
+  const feed = (bytes            ) => {
+    for (const line of decoder.decode(bytes)) processLine(line);
+  };
+
+  try {
+    await streamFile(file, feed, PEEK_CHUNK_BYTES);
+    // A trailing header line with no terminating newline sits in the decoder's pending buffer
+    // until flushed, same as parser-worker.ts's runParse/runParseFiles do after their streamFile call.
+    for (const line of decoder.flush()) processLine(line);
+  } catch (e) {
+    if (!(e instanceof PeekComplete)) return null; // unsupported/corrupt codec: same silent-skip as any unpeekable candidate
+  }
+
+  return sawAppStart ? result : null;
+}

package/vendor-core/lz4-block.js

@@ -35,12 +35,10 @@ function decompressLz4Sequence(input            , outSize        )             {
   return out;
 }
 
-// Streaming counterpart to decodeLz4Block: push arbitrary byte slices, and each
-// fully-received LZ4Block block is decompressed and handed to `onChunk` as it
-// completes, so the decompressed output is never fully buffered (bounded to one
-// block at a time). Partial trailing bytes are retained until the next push.
-// `onChunk` must consume its argument synchronously (it aliases the internal
-// buffer for RAW blocks and is not retained across the next push).
+// Streaming counterpart to decodeLz4Block: push byte slices, each complete LZ4Block block is
+// decompressed and handed to `onChunk` as it completes, so output is never fully buffered. Partial
+// trailing bytes are retained until the next push. `onChunk` must consume synchronously (it aliases
+// the internal buffer for RAW blocks, not retained across the next push).
 export function createLz4BlockDecoder(
   onChunk                             ,
 )                                                 {

package/vendor-core/mcp-server-factory.js

@@ -2,8 +2,9 @@ import { z } from 'zod';
 import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
                                                                          
 import {
-  resolveOrCreateRun, diagnoseRun, getRunSummary, compareRuns, getFindingEvidence, evaluateBudgetsForRun,
+  resolveOrCreateRun, diagnoseRun, getRunSummary, compareRuns, getFindingEvidence, getFindingDocumentation, getReferenceDoc, evaluateBudgetsForRun,
 } from './mcp-tools.js';
+import { listRuns } from './list-runs.js';
 
 const sourceSchema = z.union([
   z.object({ path: z.string() }),
@@ -14,6 +15,19 @@ const runRefSchema = { source: sourceSchema.optional(), runId: z.string().option
 const secondRunRefSchema = { sourceB: sourceSchema.optional(), runIdB: z.string().optional() };
 const formatSchema = { format: z.enum(['json', 'md']).optional() };
 
+// list_runs takes a bare shsBaseUrl (no appId/attemptId, unlike sourceSchema's SHS branch, since
+// it's listing candidates rather than resolving one), so it can't reuse sourceSchema: refine
+// against both dir and shsBaseUrl being set instead, so one doesn't silently win over the other.
+const listRunsInputSchema = z.object({
+  dir: z.string().optional(),
+  shsBaseUrl: z.string().optional(),
+  namePattern: z.string().optional(),
+  minDate: z.string().optional(),
+  maxDate: z.string().optional(),
+  maxResults: z.number().int().positive().optional(),
+  redact: z.boolean().optional(),
+}).refine((v) => !(v.dir && v.shsBaseUrl), { message: 'Provide only one of dir or shsBaseUrl, not both.' });
+
 function toCallToolResult(text        , structuredContent                         )                 {
   return { content: [{ type: 'text'         , text }], structuredContent };
 }
@@ -26,11 +40,9 @@ function toolErrorResult(error                                    )
   };
 }
 
-// diagnoseRun/compareRuns may additionally carry a `markdown` field (only
-// when the caller asked for `format: 'md'`); that field never belongs in
-// structuredContent (its shape must stay stable regardless of `format`), so
-// it's stripped here, and — when present — becomes content[0].text instead
-// of the usual JSON.stringify.
+// diagnoseRun/compareRuns may carry a `markdown` field (only when the caller asked for
+// format: 'md'); it never belongs in structuredContent (whose shape must stay stable regardless of
+// format), so it's stripped here and, when present, becomes content[0].text instead of JSON.stringify.
 function toolResultWithMarkdown                  (promise                                    )                          {
   return promise.then(
     (value) => {
@@ -41,8 +53,7 @@ function toolResultWithMarkdown                  (promise
   );
 }
 
-// Counterpart for the 3 tools that never produce a markdown field: its own
-// mapping (not a delegation through toolResultWithMarkdown), so there's no
+// Counterpart for the tools that never produce a markdown field: its own mapping, so there's no
 // need to cast T to pretend it might carry a `markdown` field it never does.
 function toolResult                  (promise            )                          {
   return promise.then((value) => toCallToolResult(JSON.stringify(value), value                           ), toolErrorResult);
@@ -51,6 +62,11 @@ function toolResult                  (promise            )
 export function createMcpServer()            {
   const server = new McpServer({ name: 'sparkforensics', version: '1.0.0' });
 
+  server.registerTool('list_runs', {
+    description: 'List candidate Spark event-log runs from a local directory or a Spark History Server, before diagnosing one with the other tools.',
+    inputSchema: listRunsInputSchema,
+  }, (params) => toolResult(listRuns(params)));
+
   server.registerTool('diagnose_run', {
     description: 'Diagnose a Spark run: thresholded findings with remediation text, an impact-ranked fix recommendation rollup, and clean-check status.',
     inputSchema: {
@@ -111,5 +127,19 @@ export function createMcpServer()            {
     Promise.resolve().then(() => getFindingEvidence(runId, findingId, { redact })),
   ));
 
+  server.registerTool('get_finding_documentation', {
+    description: 'Detection and tuning reference documentation for one finding type (not tied to a specific run), so a client without browser access can read the same background material the web view links to.',
+    inputSchema: { type: z.string() },
+  }, ({ type }) => toolResult(
+    Promise.resolve().then(() => getFindingDocumentation(type)),
+  ));
+
+  server.registerTool('get_reference_doc', {
+    description: 'Full tuning-reference chapter or bottleneck markdown by doc anchor (e.g. "#joins", "#bottleneck-skew", "#metric-task-duration"), so a client without browser access can read the same reference the web view shows.',
+    inputSchema: { anchor: z.string() },
+  }, ({ anchor }) => toolResult(
+    Promise.resolve().then(() => getReferenceDoc(anchor)),
+  ));
+
   return server;
 }