sparkforensics-cli 0.1.0

package/vendor-core/docs-config.js

@@ -0,0 +1,72 @@
+// Single source of the docs-panel URL surface. DOCS_BASE_URL is a repo-relative
+// path to the vendored, prebuilt spark-doc index.html committed directly in this
+// repo (refresh it with `npm run update-docs`). The app is frequently opened via
+// file://, where the iframe origin is opaque (see DocsSheet.tsx).
+export const DOCS_BASE_URL         = 'vendor/spark-doc/index.html';
+
+// Build a docs URL for an anchor like '#bottleneck-skew'. The leading '#' is
+// stripped and re-added so the fragment is never percent-encoded away. `tok`
+// (ELV-043's per-load handshake nonce) goes in the query string, not the hash
+// since the hash is reserved for #anchor navigation. Omitted for callers with no
+// postMessage channel to protect (e.g. the plain <a href> fallback).
+export function docsUrl(anchor        , tok         )         {
+  const frag = String(anchor).replace(/^#/, '');
+  const query = tok ? `?tok=${encodeURIComponent(tok)}` : '';
+  return `${DOCS_BASE_URL}${query}#${encodeURIComponent(frag)}`;
+}
+
+// Internal metric key → docs '#metric-*' anchor. A metric with no entry renders
+// as plain text (no broken link). Keys are the app's own metric identifiers.
+export const METRIC_ANCHORS                         = {
+  'task-duration': '#metric-task-duration',
+  'shuffle-read-bytes': '#metric-shuffle-read-bytes',
+  'shuffle-write-bytes': '#metric-shuffle-write-bytes',
+  'memory-bytes-spilled': '#metric-memory-bytes-spilled',
+  'disk-bytes-spilled': '#metric-disk-bytes-spilled',
+  'jvm-gc-time': '#metric-jvm-gc-time',
+  'gcpct': '#metric-gcpct',
+  'executor-run-time': '#metric-executor-run-time',
+  'fetch-wait-time-ratio': '#metric-fetch-wait-time-ratio',
+  'input-bytes': '#metric-input-bytes',
+  'output-bytes': '#metric-output-bytes',
+  'io-ratio': '#metric-io-ratio',
+  'peak-execution-memory': '#metric-peak-execution-memory',
+  'failed-tasks': '#metric-failed-tasks',
+  'speculative-tasks': '#metric-speculative-tasks',
+  'first-stage-submitted-at': '#metric-first-stage-submitted-at',
+  'executor-count': '#metric-executor-count',
+  'stage-duration': '#metric-stage-duration',
+};
+
+// Allowlist of every anchor that actually exists in the vendored spark-doc
+// bundle. `DocsLink` renders a link only for anchors in this set: a detector
+// may declare a `docAnchor` for a section that has not been written yet (the
+// analyzer contract requires every finding to carry one), and gating here keeps
+// that from becoming a dead "Learn more" link that scrolls nowhere. When a new
+// section lands (via `npm run update-docs`), add its anchor here and the link
+// lights up automatically. `tests/docs-config.test.js` asserts this set stays a
+// subset of the real ids in vendor/spark-doc/index.html so it cannot drift.
+export const KNOWN_DOC_ANCHORS              = new Set([
+  // Page sections
+  '#intro', '#spark-architecture', '#memory-model', '#partitioning', '#joins',
+  '#shuffle', '#data-formats', '#table-formats', '#caching', '#pyspark', '#aqe',
+  '#cluster-config', '#anti-patterns', '#metrics', '#config',
+  // Bottleneck sections
+  '#bottleneck-cold-start', '#bottleneck-failures', '#bottleneck-gc',
+  '#bottleneck-job-failure-rate', '#bottleneck-retry-waste', '#bottleneck-shuffle',
+  '#bottleneck-skew', '#bottleneck-slow-host', '#bottleneck-spill',
+  '#bottleneck-straggler', '#bottleneck-tiny-tasks', '#bottleneck-utilization',
+  '#bottleneck-memory-utilization', '#bottleneck-broadcast-sizing',
+  '#bottleneck-duplicate-plan-subtree', '#bottleneck-small-files',
+  '#bottleneck-stage-shape', '#bottleneck-stage-slowness',
+  // Config-audit sections
+  '#config-autoscale-bounds', '#config-memory-overhead', '#config-serializer',
+  '#config-shuffle-service',
+  // Metric glossary entries
+  ...Object.values(METRIC_ANCHORS),
+]);
+
+// True when `anchor` resolves to a real section in the vendored docs.
+export function isKnownDocAnchor(anchor         )          {
+  return KNOWN_DOC_ANCHORS.has(String(anchor));
+}

package/vendor-core/docs-site-config.js

@@ -0,0 +1,23 @@
+import { typeTag } from './format-utils.js';
+
+// docs-config.ts's own header comment scopes it to "the single source of the
+// [vendor] docs-panel URL surface"; this is the equivalent single source for
+// docs-site (VitePress) links, kept in its own file so the two unrelated doc
+// systems don't blur into one.
+//
+// The `.html` extension matters: it's what makes this link resolve in the
+// packaged `server/` deploy mode. That static file server
+// (packages/server/lib/static-files.js) matches a request path to a file exactly, with
+// no directory-index or extension-guessing fallback (unlike the Vite
+// dev-server's docs-site middleware in vite.config.ts, which tries
+// bare/`.html`/`index.html` candidates). VitePress's default build
+// (`cleanUrls` unset, i.e. false) names each page's output file `<slug>.html`,
+// so the extension-less form would 404 there.
+//
+// No allowlist/gating function is needed here, unlike `isKnownDocAnchor` for
+// the vendor link: `tests/docs-site-tag-coverage.test.js` already fails CI if
+// any `TYPE_TAG_MAP` value lacks a documented `{#tag}` heading, and both the
+// map and the page live in this same repo, so a mismatch can't ship past CI.
+export function findingGuideUrl(type        )         {
+  return `/docs/user-guide/understanding-findings.html#${typeTag(type).toLowerCase()}`;
+}

package/vendor-core/efficiency-model.js

@@ -0,0 +1,62 @@
+// §5 Efficiency/wastage model (SparkLens EfficiencyStatisticsAnalyzer).
+// DESIGN SPIKE: decomposes available compute-hours into driver-bound vs.
+// executor-bound waste, plus two theoretical floors. Mapped onto computeWallClock.
+import { computeWallClock } from './wall-clock.js';
+import { computeTotalCores } from './core-count.js';
+                                                                             
+
+export function computeEfficiencyModel({ app, stages, executorsAdded, runAggregates }   
+                           
+                               
+                                  
+                                      
+ )   
+                                
+                           
+                             
+                            
+                          
+                                              
+  {
+  // `app ?? {}`: computeTotalCores just falls back to the executor-derived core sum when
+  // `resources` is absent, and every caller here already tolerates a null `app` (malformed
+  // logs with no ApplicationStart); `app!` would crash on `app.resources` in that case.
+  // `executorsAdded` cast: computeTotalCores only reads `totalCores`, present on
+  // ExecutorAddedEvent (the only kind real callers pass here) but not ExecutorRemovedEvent,
+  // so the union as a whole is a structural mismatch against computeTotalCores's
+  // `{totalCores?}` shape.
+  const totalCores = computeTotalCores(app ?? {}, executorsAdded                                  );
+  const appDurationMs = (app?.endTime ?? 0) - (app?.startTime ?? 0);
+  const wc = computeWallClock(app, stages                                                               );
+
+  const availableComputeHours = totalCores * (appDurationMs / 3600000);
+
+  // Driver-bound: cores idle during startup + gaps + idle (no active stage).
+  const driverIdleMs = wc.startup + wc.gaps + wc.idle;
+  const driverWasteHours = totalCores * (driverIdleMs / 3600000);
+
+  // Executor-bound: during stagesActive, allocated capacity beyond busy cores.
+  // Tasks only run during active windows, so whole-run busyCoreMs already lives
+  // inside stagesActive; no separate per-window sweep needed.
+  const allocatedActiveCoreMs = totalCores * wc.stagesActive;
+  const busyCoreMs = runAggregates?.busyCoreMs ?? 0;
+  const executorWasteHours = Math.max(0, allocatedActiveCoreMs - busyCoreMs) / 3600000;
+
+  const wastageHours = driverWasteHours + executorWasteHours;
+  const wastagePct = availableComputeHours > 0 ? Math.round(wastageHours / availableComputeHours * 100) : null;
+
+  let totalTaskMs = 0;
+  const perStage = runAggregates?.perStage ?? {};
+  for (const k of Object.keys(perStage)) totalTaskMs += perStage[k].totalTaskDurationSum;
+  const floorZeroSkewMs = totalCores > 0 ? totalTaskMs / totalCores : 0;
+
+  let dominantWaste                               = null;
+  if (driverWasteHours > 0 || executorWasteHours > 0) {
+    dominantWaste = driverWasteHours >= executorWasteHours ? 'driver' : 'executor';
+  }
+
+  return {
+    availableComputeHours, driverWasteHours, executorWasteHours, wastagePct,
+    floorZeroSkewMs, dominantWaste,
+  };
+}

package/vendor-core/etl-phases.js

@@ -0,0 +1,28 @@
+// §7 ETL-phase time attribution (Onehouse Spark Analyzer). Heuristic and
+// approximate: classify each stage by byte-flow shape. A stage can be both
+// Transform and Load (shuffle + write), so buckets overlap and need not sum to
+// wall-clock. Storage-format-aware attribution (Hudi/Delta/Iceberg) is NOT
+// portable (no table-format metadata in event logs) and is out of scope.
+                         
+                       
+                       
+                            
+                             
+                      
+                       
+ 
+
+export function attributeEtlPhases(
+  stages                            ,
+)                                                       {
+  const acc = { extract: 0, transform: 0, load: 0 };
+  for (const s of stages.values()) {
+    const dur = Math.max(0, (s.completedAt ?? 0) - (s.submittedAt ?? 0));
+    if (dur === 0) continue;
+    const shuffled = (s.shuffleReadBytes ?? 0) > 0 || (s.shuffleWriteBytes ?? 0) > 0;
+    if ((s.inputBytes ?? 0) > 0 && (s.shuffleReadBytes ?? 0) === 0) acc.extract += dur; // scan-only
+    if (shuffled) acc.transform += dur;
+    if ((s.outputBytes ?? 0) > 0) acc.load += dur;
+  }
+  return acc;
+}