sparkforensics-cli 0.1.0

package/bin/sparkforensics-analyze.mjs

@@ -0,0 +1,288 @@
+#!/usr/bin/env node
+import { parseArgs } from 'node:util';
+import { writeFileSync, existsSync, realpathSync } from 'node:fs';
+import { join, dirname } from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+
+const binDir = dirname(fileURLToPath(import.meta.url));
+const pkgDir = dirname(binDir);
+
+// vendor-core/ exists only in a published/standalone install (populated by
+// scripts/vendor-core.mjs at pack time); in the monorepo it falls back to
+// the real packages/core/src/ sibling. load-vendored.js itself is the one
+// module this bootstrap has to locate by hand (see its own header comment);
+// every other core module then loads through its exported loadVendored().
+const vendoredHelper = join(pkgDir, 'vendor-core', 'load-vendored.js');
+const helperPath = existsSync(vendoredHelper) ? vendoredHelper : join(pkgDir, '..', 'core', 'src', 'load-vendored.js');
+const { loadVendored } = await import(pathToFileURL(helperPath).href);
+const loadCore = (moduleName) => loadVendored(pkgDir, moduleName);
+
+const { collectRun } = await loadCore('cli/collect-run');
+const { resolveFromShs } = await loadCore('shs-load');
+const { analyze } = await loadCore('analyzer');
+const { deriveEvidenceAvailability } = await loadCore('evidence-availability');
+const { buildEvidenceReport, toFindingsFilter } = await loadCore('evidence-report');
+const { evaluateBudgets } = await loadCore('cli/budgets');
+const { buildComparison, renderComparisonMarkdown } = await loadCore('run-comparison');
+const { redactComparison } = await loadCore('redact');
+
+const USAGE = `Usage: sparkforensics-analyze <event-log-file|rolling-log-dir> [options]
+       sparkforensics-analyze --shs-base-url <url> --app-id <id> [--attempt-id <id>] [options]
+
+Options:
+  --format md|json                 Output format (default: json).
+  --out <path>                    Write output to a file instead of stdout.
+  --max-runtime <ms>               Fail if app runtime exceeds this many ms.
+  --max-spill <gb>                  Fail if any stage spills more than this many GB.
+  --max-skew <ratio>                Fail if any stage's P95/median duration ratio exceeds this.
+  --max-failed-task-rate <pct>      Fail if the task failure rate exceeds this percent.
+  --min-efficiency <pct>            Fail if compute efficiency falls below this percent.
+  --shs-base-url <url>              Fetch the run from a Spark History Server instead of a
+                                    local file (mutually exclusive with the positional argument).
+  --app-id <id>                     Spark application ID to fetch. Required with --shs-base-url.
+  --attempt-id <id>                 Optional attempt ID, used with --shs-base-url.
+  --baseline <path>                 Compare the run against a baseline local event-log file or
+                                    rolling-log directory. Baseline is local-path-only (no SHS
+                                    support). Adds a comparison section to the output.
+  --max-regression-pct <pct>        Requires --baseline. Fail if the regression metric (see
+                                    --regression-metric) regressed by more than this percent.
+  --regression-metric <key>         Metric key to check with --max-regression-pct (default:
+                                    wallClock). Requires --baseline and --max-regression-pct.
+  --fail-on-introduced <band|all>   Requires --baseline. Fail if any finding was introduced by
+                                    the candidate matching this impact band (or any, with "all").
+  --redact                          Pseudonymize the app id and any host/IP tokens in the output
+                                    (app-1, host-1, ...), so a report can be shared outside the
+                                    environment that produced it.
+  --impact <band[,band]>            Filter the output's findings array to these impact bands
+                                    (critical, warning, info). recommendations/cleanChecks and
+                                    the summary counts stay on the full, unfiltered set.
+  --type <type[,type]>              Filter the output's findings array to these finding types.
+  --stage <id>                      Filter the output's findings array to this stage id.
+
+Exit codes: 0 pass, 1 budget violated, 2 bad arguments, the local input could not be parsed, or the --shs-base-url fetch failed, 3 a budget was inconclusive.
+`;
+
+function parseCliArgs(argv) {
+  const { values, positionals } = parseArgs({
+    args: argv,
+    allowPositionals: true,
+    options: {
+      out: { type: 'string' },
+      format: { type: 'string' },
+      'max-runtime': { type: 'string' },
+      'max-spill': { type: 'string' },
+      'max-skew': { type: 'string' },
+      'max-failed-task-rate': { type: 'string' },
+      'min-efficiency': { type: 'string' },
+      'shs-base-url': { type: 'string' },
+      'app-id': { type: 'string' },
+      'attempt-id': { type: 'string' },
+      baseline: { type: 'string' },
+      'max-regression-pct': { type: 'string' },
+      'regression-metric': { type: 'string' },
+      'fail-on-introduced': { type: 'string' },
+      redact: { type: 'boolean' },
+      impact: { type: 'string' },
+      type: { type: 'string' },
+      stage: { type: 'string' },
+      help: { type: 'boolean' },
+    },
+  });
+  return { values, positionals };
+}
+
+function bail(message, code) {
+  process.stderr.write(message);
+  process.exitCode = code;
+}
+
+function splitCsv(value) {
+  if (value === undefined) return undefined;
+  return value.split(',').map((s) => s.trim()).filter(Boolean);
+}
+
+async function collectWithEvidence(path) {
+  const { appModel, skippedLines } = await collectRun(path);
+  appModel.evidenceAvailability = deriveEvidenceAvailability(appModel, { skippedLines });
+  return appModel;
+}
+
+export async function main(argv, { fetchImpl } = {}) {
+  const { values, positionals } = parseCliArgs(argv);
+  if (values.help) {
+    process.stderr.write(USAGE);
+    process.exitCode = 0;
+    return;
+  }
+
+  const usingShs = values['shs-base-url'] !== undefined;
+  if (usingShs) {
+    if (positionals.length > 0) {
+      return bail(`Pass either <event-log-file|rolling-log-dir> or --shs-base-url, not both.\n${USAGE}`, 2);
+    }
+    if (values['app-id'] === undefined) {
+      return bail(`--shs-base-url requires --app-id.\n${USAGE}`, 2);
+    }
+  } else {
+    if (positionals.length !== 1) {
+      return bail(USAGE, 2);
+    }
+    if (values['app-id'] !== undefined || values['attempt-id'] !== undefined) {
+      return bail(`--app-id/--attempt-id require --shs-base-url.\n${USAGE}`, 2);
+    }
+  }
+
+  if (values.format !== undefined && values.format !== 'json' && values.format !== 'md') {
+    return bail(`Invalid value for --format (expected "json" or "md").\n${USAGE}`, 2);
+  }
+
+  const usingBaseline = values.baseline !== undefined;
+  // Single source of truth for which flags need --baseline: a future flag
+  // just gets appended here, instead of also needing its own OR-condition
+  // above (easy to forget, and forgetting it means the flag is silently
+  // accepted without --baseline instead of erroring).
+  const BASELINE_DEPENDENT_FLAGS = ['max-regression-pct', 'regression-metric', 'fail-on-introduced'];
+  if (!usingBaseline && BASELINE_DEPENDENT_FLAGS.some((flag) => values[flag] !== undefined)) {
+    return bail(`--max-regression-pct/--regression-metric/--fail-on-introduced require --baseline.\n${USAGE}`, 2);
+  }
+  if (values['regression-metric'] !== undefined && values['max-regression-pct'] === undefined) {
+    return bail(`--regression-metric requires --max-regression-pct.\n${USAGE}`, 2);
+  }
+
+  const budgets = {
+    maxRuntimeMs: values['max-runtime'] != null ? Number(values['max-runtime']) : undefined,
+    maxSpillGb: values['max-spill'] != null ? Number(values['max-spill']) : undefined,
+    maxSkewRatio: values['max-skew'] != null ? Number(values['max-skew']) : undefined,
+    maxFailedTaskRatePct: values['max-failed-task-rate'] != null ? Number(values['max-failed-task-rate']) : undefined,
+    minEfficiencyPct: values['min-efficiency'] != null ? Number(values['min-efficiency']) : undefined,
+    maxRegressionPct: values['max-regression-pct'] != null ? Number(values['max-regression-pct']) : undefined,
+  };
+  // Derived from `budgets`, not hardcoded: every key on that object literal
+  // above is numeric (regressionMetric/failOnIntroduced are only added below,
+  // after this validation loop runs), so a future numeric budget field can't
+  // silently skip this check by only being added to one of the two places.
+  const NUMERIC_BUDGET_FLAGS = Object.keys(budgets);
+  for (const flag of NUMERIC_BUDGET_FLAGS) {
+    const value = budgets[flag];
+    if (value !== undefined && !Number.isFinite(value)) {
+      return bail(`Invalid numeric value for ${flag}.\n${USAGE}`, 2);
+    }
+  }
+  // Only set when the user actually passed --regression-metric: evaluateBudgets()
+  // now treats "regressionMetric is set" as "the caller asked for a regression
+  // check" (guarding against it being set without --max-regression-pct), so
+  // defaulting it here unconditionally would trip that guard on every run that
+  // never touched either flag. checkRegression already falls back to
+  // 'wallClock' itself (`budgets.regressionMetric ?? 'wallClock'`) once
+  // maxRegressionPct is actually set, so no default is needed here.
+  if (values['regression-metric'] !== undefined) budgets.regressionMetric = values['regression-metric'];
+  if (values['fail-on-introduced'] !== undefined) budgets.failOnIntroduced = values['fail-on-introduced'];
+
+  const impactBand = splitCsv(values.impact);
+  const type = splitCsv(values.type);
+  let stageId;
+  if (values.stage !== undefined) {
+    stageId = Number(values.stage);
+    if (!Number.isInteger(stageId)) {
+      return bail(`Invalid integer value for --stage.\n${USAGE}`, 2);
+    }
+  }
+  const findingsFilter = toFindingsFilter(impactBand, type, stageId);
+
+  let appModel;
+  let baselineAppModel;
+  try {
+    if (usingShs) {
+      const shsPromise = resolveFromShs(
+        values['shs-base-url'], values['app-id'], values['attempt-id'],
+        fetchImpl !== undefined ? { fetchImpl } : {},
+      );
+      // The SHS fetch (network) and a local --baseline parse (disk) are
+      // independent, so run them concurrently. Not applied to the
+      // local-candidate+local-baseline combination below: both sides there
+      // are synchronous-equivalent CPU work reading the same way, so
+      // parallelizing wouldn't help and would just add indirection.
+      if (usingBaseline) {
+        [appModel, baselineAppModel] = await Promise.all([shsPromise, collectWithEvidence(values.baseline)]);
+      } else {
+        appModel = await shsPromise;
+      }
+    } else {
+      appModel = await collectWithEvidence(positionals[0]);
+      if (usingBaseline) baselineAppModel = await collectWithEvidence(values.baseline);
+    }
+  } catch (e) {
+    process.stderr.write(`${e.message}\n`);
+    process.exitCode = 2;
+    return;
+  }
+
+  const analyzeModel = (model) => analyze(
+    model.app, model.stages, model.executors.added, model.executors.removed,
+    model.jobs, model.sql, model.runAggregates,
+  );
+  const catalog = analyzeModel(appModel);
+
+  let comparison;
+  if (usingBaseline) {
+    const baselineCatalog = analyzeModel(baselineAppModel);
+    comparison = buildComparison(
+      { label: 'baseline', appModel: baselineAppModel, catalog: baselineCatalog },
+      { label: 'candidate', appModel, catalog },
+    );
+    // --redact must also scrub the comparison section: stage names in
+    // comparison.findings/baseStages/candStages carry raw Spark stage text,
+    // which is exactly what --redact promises to pseudonymize.
+    if (values.redact) comparison = redactComparison(comparison);
+  }
+
+  const { markdown, json } = buildEvidenceReport(appModel, { redact: values.redact, findingsFilter, markdown: values.format === 'md' });
+  let output;
+  if (values.format === 'md') {
+    output = comparison ? `${markdown}${renderComparisonMarkdown(comparison)}\n` : `${markdown}\n`;
+  } else {
+    const payload = comparison
+      ? {
+        candidate: json,
+        comparison: {
+          confidence: comparison.confidence,
+          reason: comparison.reason,
+          matchedCoverage: comparison.matchedCoverage,
+          metrics: comparison.metrics,
+          findings: comparison.findings,
+        },
+      }
+      : json;
+    output = `${JSON.stringify(payload, null, 2)}\n`;
+  }
+  if (values.out) writeFileSync(values.out, output);
+  else process.stdout.write(output);
+
+  const { results, violated, inconclusive } = evaluateBudgets({ appModel, catalog, budgets, comparison });
+  for (const r of results) {
+    if (r.status === 'inconclusive') process.stderr.write(`[inconclusive] ${r.name}: ${r.detail}\n`);
+    else if (r.status === 'violation') process.stderr.write(`[violation] ${r.name}: ${r.detail}\n`);
+  }
+
+  // Always checked, unlike the opt-in budgets above: a run whose event log
+  // never recorded an ApplicationEnd is inconclusive by default, not just
+  // when the caller happens to pass --max-runtime.
+  const incompleteRunFinding = catalog.find((f) => f.type === 'incompleteRun');
+  if (incompleteRunFinding) process.stderr.write(`[inconclusive] run-complete: ${incompleteRunFinding.recommendation}\n`);
+
+  if (violated) process.exitCode = 1;
+  else if (inconclusive || incompleteRunFinding) process.exitCode = 3;
+  else process.exitCode = 0;
+}
+
+// Guarded so tests can import `main` for in-process runs (e.g. mocking the
+// global `fetch` that resolveFromShs falls back to) without also triggering
+// a second, real invocation as a side effect of the import. A published
+// install's `npm install` bin creates a node_modules/.bin/ symlink (see
+// server/index.js's invokedDirectly for the same fix), so argv[1] must be
+// realpath-resolved before comparing against this module's real path:
+// otherwise the guard never matches when invoked through that symlink and
+// the CLI silently does nothing (exit 0, no output).
+if (process.argv[1] && realpathSync(process.argv[1]) === fileURLToPath(import.meta.url)) {
+  await main(process.argv.slice(2));
+}

package/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "sparkforensics-cli",
+  "version": "0.1.0",
+  "type": "module",
+  "license": "MIT",
+  "description": "CLI to analyze Apache Spark event logs for performance bottlenecks: skew, spill, GC pressure, stragglers, and more.",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/shuffle-works/sparkforensics.git"
+  },
+  "bin": {
+    "sparkforensics-analyze": "./bin/sparkforensics-analyze.mjs"
+  },
+  "engines": {
+    "node": ">=18"
+  },
+  "files": ["bin/", "vendor-core/"],
+  "scripts": {
+    "prepack": "node ../../scripts/vendor-core.mjs .",
+    "test": "vitest run"
+  },
+  "dependencies": {
+    "zod": "^4.4.3"
+  },
+  "devDependencies": {
+    "@sparkforensics/core": "*",
+    "vitest": "^4.1.10"
+  }
+}

package/vendor-core/analyzer.js

@@ -0,0 +1,167 @@
+import { DETECTORS,               } from './detectors.js';
+import { computePeakConcurrentCores } from './core-count.js';
+import { assertNever } from './assert-never.js';
+import { estimateImpact } from './impact-estimator.js';
+import { computeOccupancy,                     } from './occupancy.js';
+import { deriveImpactBand } from './impact-band.js';
+import { IMPACT_BAND_ORDER } from './format-utils.js';
+             
+                                                                                
+                    
+
+// FNV-1a 32-bit: a small, dependency-free stable string hash. Used to derive
+// a deterministic finding `id` at the single choke point below so the same
+// logical finding keeps the same id across runs (no timestamps, no randomness).
+function fnv1a(str        )         {
+  let h = 0x811c9dc5;
+  for (let i = 0; i < str.length; i++) {
+    h ^= str.charCodeAt(i);
+    h = Math.imul(h, 0x01000193);
+  }
+  return (h >>> 0).toString(16).padStart(8, '0');
+}
+
+export function findingId(f         )         {
+  // Location key mirrors the report's stable identity: stage, else SQL
+  // execution, else the audited config property.
+  const locKey = f.stageId ?? f.executionId ?? f.property ?? '';
+  // Discriminators for detectors that intentionally emit multiple findings on
+  // the same location+metric; without them these siblings hash to one id:
+  //   stage-scope: slowHost (host), memoryUtilization (executorId + dimension),
+  //     partitionSizing (rule);
+  //   app-scope: cacheUtilization (rddId + variant, stageId is always null),
+  //     cachingOpportunity (stageId/executionId both null too: `relation`/
+  //     `format` distinguish leaf findings, `operator`+`relation` distinguish
+  //     composite findings, `executionIds` as a last resort when everything
+  //     else matches);
+  //   SQL plan-advisor: smallFiles (direction/nodeName),
+  //     duplicatePlanSubtree (rootName/subtreeSize/groupIndex: two unrelated
+  //     duplicate-subtree groups can share rootName+subtreeSize, e.g. the same
+  //     "BroadcastExchange over a Project/Filter/Scan" shape repeated once per
+  //     dimension table; groupIndex (the group's deterministic position within
+  //     one execution's plan) is the actual uniqueness guarantee, since the
+  //     informational `sampleRelation` field can be null or coincide for two
+  //     groups),
+  //     broadcastSizing (per node/side, distinguished by value + largerSideBytes).
+  // The metric `value` folds in the per-node magnitude that broadcastSizing
+  // siblings carry no other field for; it is deterministic for a given log, so
+  // the id stays stable across re-parses (no timestamp/randomness).
+  // memoryUtilization's `rule` (heapNearCapacity/heapOverProvisioned) is excluded here: an
+  // executor falls in exactly one heap band, so `executorId` alone already guarantees
+  // uniqueness and folding `rule` in too would just add gratuitous id-rotation risk if the
+  // band logic ever changes. partitionSizing's `rule` (maxPartitionTooBig/
+  // shufflePartitionSkew/lowShuffleParallelism) IS load-bearing here: a stage can emit more
+  // than one of those rules at once, sharing the same stageId+metric.
+  const rule = f.type === 'memoryUtilization' ? undefined : f.rule;
+  const disc = [
+    f.host, f.executorId, rule, f.variant, f.dimension,
+    f.direction, f.nodeName, f.rootName, f.subtreeSize, f.groupIndex, f.largerSideBytes,
+    f.rddId, f.relation, f.format, f.operator,
+    f.executionIds ? f.executionIds.join(',') : '',
+  ].map((v) => v ?? '').join('|');
+  return fnv1a(`${f.type}|${locKey}|${f.metric ?? ''}|${f.value ?? ''}|${disc}`);
+}
+
+function push(out           , entry          , result                            )       {
+  if (!result) return;
+  const detectorVersion = entry.version ?? 1;
+  for (const f of (Array.isArray(result) ? result : [result])) {
+    if (!f) continue;
+    if (entry.suppressWhen && entry.suppressWhen(f, out)) continue;
+    const stamped = { ...f, docAnchor: f.docAnchor ?? entry.docAnchor, detectorVersion };
+    const id = findingId(stamped);
+    // Guardrail against any detector (this one or a future one) emitting two
+    // structurally-identical findings for what should be one logical
+    // occurrence: same id => same finding, keep only the first. Added after a
+    // real-log bug where duplicatePlanSubtree emitted two distinct findings
+    // sharing one id (see findingId's groupIndex discriminator above and
+    // tests/detectors-plan.test.js's duplicatePlanSubtree regression tests);
+    // this guard's correctness depends on findingId's discriminator list
+    // actually being unique per distinct finding, not on this line itself.
+    if (out.some((existing) => existing.id === id)) continue;
+    out.push({ ...stamped, id });
+  }
+}
+
+// `app` is widened to `SparkAppInfo | null` rather than the plan's literal
+// non-nullable `SparkAppInfo`: real callers (evidence-report.ts's
+// `buildJson`, useIngest.ts's `runDone`/`snapshotParsedRun`) pass
+// `AppModel.app`, which is genuinely `SparkAppInfo | null` at the type
+// level even though by the time `analyze()` runs in practice a full parse
+// has completed and `app` is always populated (the same reasoning
+// detectors.ts's `DetectorCtx.app` comment gives for treating its own,
+// separate `app` field as non-nullable). Widened to match the real caller
+// type rather than forcing a cast at every call site.
+export function analyze(
+  app                     ,
+  stages                    ,
+  executorsAdded                 ,
+  executorsRemoved                 ,
+  jobs                  ,
+  sql                            = new Map(),
+  runAggregates                       = null,
+)            {
+  // `app ?? {}`: every detector below tolerates a null `app` (malformed logs
+  // with no ApplicationStart), so this call must too; computePeakConcurrentCores
+  // just falls back to the executor-derived core sum when `resources` is absent.
+  // computePeakConcurrentCores (not computeTotalCores) here specifically: the
+  // ceiling this feeds (src/occupancy.ts's computeCeiling) needs a genuine
+  // concurrent-capacity bound, and computeTotalCores's cumulative sum over
+  // every addition overstates that under dynamic allocation/executor
+  // replacement (a churned-through executor's cores were never actually
+  // concurrent with its replacement's).
+  // `executorsAdded`/`executorsRemoved` casts: computePeakConcurrentCores only
+  // reads `executorId`/`timestamp`/`totalCores`, all present on
+  // ExecutorAddedEvent/ExecutorRemovedEvent, but `totalCores` isn't on
+  // ExecutorRemovedEvent, so the `ExecutorEvent` union as a whole is a
+  // structural mismatch against its parameter shapes.
+  const totalCores = computePeakConcurrentCores(
+    app ?? {},
+    executorsAdded                                                                         ,
+    executorsRemoved                                                    ,
+  );
+  // Computed once, up front, so detectors can gate impact band on the same
+  // occupancy-clipped waste figure estimateImpact() below displays as that
+  // finding's savings (src/detectors.ts's `clippedWasteMs`), not a raw
+  // pre-clip delta the two passes would otherwise disagree on.
+  const occupancy = computeOccupancy(stages                                          , totalCores);
+  const ctx = {
+    app, stages, executorsAdded, executorsRemoved, jobs, sql, runAggregates, occupancy,
+  };
+  const out            = [];
+  for (const d of DETECTORS) {
+    if (d.inScorecard === false) continue;
+    switch (d.scope) {
+      case 'stage':
+        for (const s of stages.values()) push(out, d, d.detect(s, ctx));
+        break;
+      case 'sql':
+        for (const e of sql.values()) push(out, d, d.detect(e, ctx));
+        break;
+      case 'app':
+      case 'config':
+        push(out, d, d.detect(ctx));
+        break;
+      default:
+        assertNever(d.scope);
+    }
+  }
+  estimateImpact(out, stages, totalCores);
+  deriveImpactBand(out, app);
+  // Ascending IMPACT_BAND_ORDER (critical 0 → info 2) puts the worst band
+  // first, matching the old descending 3/2/1 rank this replaced; `sort` is
+  // stable, so findings sharing a band keep their DETECTORS declaration order.
+  out.sort((a, b) => IMPACT_BAND_ORDER[a.impactBand] - IMPACT_BAND_ORDER[b.impactBand]);
+  return out;
+}
+
+export function auditConfig(app                     )            {
+  const out            = [];
+  for (const d of DETECTORS) if (d.scope === 'config') push(out, d, d.detect({ app }));
+  // configAudit's own impact-estimator case (src/impact-estimator.ts) needs neither
+  // `stages` nor `totalCores`: it's unconditionally `costOnly('none')`. An empty stages
+  // map is enough for parity with the same finding type produced via analyze().
+  estimateImpact(out, new Map());
+  deriveImpactBand(out, app);
+  return out.map((f) => ({ ...f, stageId: f.stageId ?? null }));
+}

package/vendor-core/assert-never.js

@@ -0,0 +1,3 @@
+export function assertNever(x       )        {
+  throw new Error(`Unreachable case: ${JSON.stringify(x)}`);
+}