sparkforensics-cli 0.1.0

package/vendor-core/event-schemas.js

@@ -0,0 +1,405 @@
+import { z } from 'zod';
+
+// ---------------------------------------------------------------------------
+// sparkPlanInfo: iterative (non-recursive-schema) tree parser
+//
+// Validates the raw, unbounded-depth `sparkPlanInfo` tree carried on
+// SparkListenerSQLExecutionStart/End events (consumed today by
+// `resolvePlanTree` in event-handlers.js:233-273). Per the spec, this must
+// NEVER be validated with z.lazy(): it uses an explicit heap-allocated stack,
+// the same iterative-over-recursive approach src/plan-tree-walk.js's
+// walkPlanTree already uses for the *resolved* tree (that one walks an
+// already-built tree; this one builds one from raw JSON).
+// ---------------------------------------------------------------------------
+
+// `resolvePlanTree` (event-handlers.ts) reads `m.accumulatorId` to look up
+// the accumulator's live value in accumMap; this field was missing from the
+// original (Task 6) schema, so it was silently stripped by object-schema
+// validation before resolvePlanTree ever saw it, breaking metric resolution
+// for every plan node once real events start flowing through
+// SparkEventSchema.safeParse. Declared optional (not required) to keep
+// accepting tests/event-schemas.test.js's existing 'value'-only fixture;
+// resolvePlanTree already tolerates a missing accumulatorId (accumMap.has
+// just resolves false, the metric contributes no value).
+const SparkPlanMetricSchema = z.object({
+  name: z.string(),
+  accumulatorId: z.number().optional(),
+  value: z.union([z.string(), z.number()]).optional(),
+  metricType: z.string().optional(),
+});
+
+// Shallow shape only. `children` is intentionally z.array(z.unknown()) here:
+// each child gets its own shallow parse inside the iterative walk below, so
+// this schema never recurses into itself and zod never sees the full tree
+// depth in one call.
+//
+// `metadata` (file-scan nodes carry Location/Format/ReadSchema/PushedFilters/
+// PartitionFilters/DataFilters/Batched here) was missing from the original
+// schema, so it was silently stripped before ever reaching the posted `sql`
+// message's `sparkPlanInfo` (same data-loss class as the accumulatorId gap
+// above). Nothing reads it yet, but it's real data crossing the worker
+// boundary, so it's kept rather than dropped.
+const SparkPlanInfoNodeSchema = z.object({
+  nodeName: z.string(),
+  simpleString: z.string().optional(),
+  metrics: z.array(SparkPlanMetricSchema).optional(),
+  metadata: z.record(z.string(), z.unknown()).optional(),
+  children: z.array(z.unknown()),
+});
+
+                                
+                   
+                        
+                                                                                                     
+                                     
+                            
+ 
+
+const MAX_PLAN_DEPTH = 500;
+
+                 
+                                                   
+                            
+                        
+                
+ 
+
+export function parseSparkPlanInfoTree(raw         )                {
+  const rootShallow = SparkPlanInfoNodeSchema.parse(raw);
+  const rootFrame        = { shallow: rootShallow, children: [], pendingRaw: [...rootShallow.children], depth: 0 };
+  const stack          = [rootFrame];
+  const parentOf = new Map              ();
+
+  while (stack.length > 0) {
+    const top = stack[stack.length - 1];
+    if (top.pendingRaw.length === 0) {
+      stack.pop();
+      const built                = { ...top.shallow, children: top.children };
+      const parent = parentOf.get(top);
+      if (parent) {
+        parent.children.push(built);
+        continue;
+      }
+      return built;
+    }
+    if (top.depth > MAX_PLAN_DEPTH) {
+      throw new Error(`sparkPlanInfo tree exceeds max depth of ${MAX_PLAN_DEPTH}`);
+    }
+    const nextRaw = top.pendingRaw.shift();
+    const shallow = SparkPlanInfoNodeSchema.parse(nextRaw);
+    const frame        = { shallow, children: [], pendingRaw: [...shallow.children], depth: top.depth + 1 };
+    parentOf.set(frame, top);
+    stack.push(frame);
+  }
+  throw new Error('unreachable: sparkPlanInfo tree stack exhausted without resolving root');
+}
+
+// A zod field schema wrapping parseSparkPlanInfoTree, for use inside the
+// SQLExecutionStart event schema. Nullable/optional to mirror
+// `event.sparkPlanInfo ?? null` in startSqlExecution (event-handlers.js:448).
+// Deliberately NOT z.lazy(): defers to the iterative parser above, so the
+// full tree depth is still never handed to zod's own recursion.
+const SparkPlanInfoFieldSchema = z
+  .unknown()
+  .nullable()
+  .optional()
+  .transform((val, ctx) => {
+    if (val == null) return null;
+    try {
+      return parseSparkPlanInfoTree(val);
+    } catch (err) {
+      ctx.addIssue({ code: z.ZodIssueCode.custom, message: err instanceof Error ? err.message : String(err) });
+      return z.NEVER;
+    }
+  });
+
+// ---------------------------------------------------------------------------
+// Six fully-specified event schemas (verbatim per task-6-brief.md Step 3;
+// backed by exact quoted source cited in each comment)
+// ---------------------------------------------------------------------------
+
+// event-handlers.js:516-518 (`SparkListenerLogStart` case, inline: sets
+// state.pendingSparkVersion)
+export const LogStartEventSchema = z.object({
+  Event: z.literal('SparkListenerLogStart'),
+  'Spark Version': z.string().optional(),
+});
+
+// event-handlers.js:293-306 (startApplication)
+export const ApplicationStartEventSchema = z.object({
+  Event: z.literal('SparkListenerApplicationStart'),
+  'App ID': z.string().optional(),
+  'App Name': z.string().optional(),
+  Timestamp: z.number().optional(),
+  'Spark Version': z.string().optional(),
+});
+
+// event-handlers.js:308-323 (updateEnvironment): 'Spark Properties' is fed
+// straight into normalizeSparkProperties(props), which already accepts
+// object | array | null (src/format-utils... no, src/event-handlers.js:53-62)
+//
+// Property VALUES were previously required to be z.string(), but
+// normalizeSparkProperties (event-handlers.ts:211-226) just spread each
+// value through (`{ ...(props ?? {}) }` / `map[pair[0]] = pair[1]`) with no
+// requirement that it already be a string, so a real Spark config carrying a
+// non-string-looking JSON value (a bare number or boolean) was rejecting the
+// entire EnvironmentUpdate event instead of parsing it. Widened to the
+// realistic set of raw JSON scalar types; normalizeSparkProperties now
+// coerces each value to a string at the boundary so its `Record<string,
+// string>` return contract (and every downstream reader of `app.config`)
+// is unaffected.
+const SparkPropertyValueSchema = z.union([z.string(), z.number(), z.boolean()]);
+export const EnvironmentUpdateEventSchema = z.object({
+  Event: z.literal('SparkListenerEnvironmentUpdate'),
+  'Spark Properties': z.union([
+    z.record(z.string(), SparkPropertyValueSchema),
+    z.array(z.tuple([z.string(), SparkPropertyValueSchema])),
+    z.null(),
+  ]).optional(),
+});
+
+// event-handlers.js:526-532 (inline ApplicationEnd case)
+export const ApplicationEndEventSchema = z.object({
+  Event: z.literal('SparkListenerApplicationEnd'),
+  Timestamp: z.number().optional(),
+});
+
+// event-handlers.js:490-502 (addExecutor). `Resource Profile Id` is read from
+// BOTH inside 'Executor Info' (the real Spark placement) and top-level (a
+// fallback for a variant that hoists it) via `event['Executor Info']?.[...] ??
+// event['Resource Profile Id']`, so both locations must be declared or the
+// nested (primary) one is silently stripped by object-schema validation.
+// `Timestamp` is read as `event['Timestamp']` with no defensive operator, so
+// per the optionality rule it's required, not guessed-optional (it also
+// backs ExecutorAddedEvent.timestamp: number, a non-optional field).
+export const ExecutorAddedEventSchema = z.object({
+  Event: z.literal('SparkListenerExecutorAdded'),
+  Timestamp: z.number(),
+  'Executor ID': z.string(),
+  'Executor Info': z.object({
+    Host: z.string().optional(),
+    'Total Cores': z.number().optional(),
+    'Resource Profile Id': z.number().optional(),
+  }).optional(),
+  'Resource Profile Id': z.number().optional(),
+});
+
+// event-handlers.js:504-512 (removeExecutor). `Timestamp` is read as
+// `event['Timestamp']` with no defensive operator, so per the optionality
+// rule it's required (backs ExecutorRemovedEvent.timestamp: number).
+export const ExecutorRemovedEventSchema = z.object({
+  Event: z.literal('SparkListenerExecutorRemoved'),
+  Timestamp: z.number(),
+  'Executor ID': z.string(),
+  'Removed Reason': z.string().optional(),
+});
+
+// ---------------------------------------------------------------------------
+// Nine schemas transcribed from their handlers (task-6-brief.md Step 4). See
+// task-6-report.md for the exact field-by-field derivation from each handler.
+// ---------------------------------------------------------------------------
+
+// event-handlers.js:325-345 (startJob). 'spark.sql.execution.id' is read via
+// `event['Properties']?.['spark.sql.execution.id']` then
+// `parseInt(sqlExecIdStr, 10)` (startJob, event-handlers.ts:498-501):
+// parseInt coerces its argument to a string internally, so it tolerates the
+// raw value already being a number just as well as a string; requiring
+// z.string() here rejected the whole JobStart event for a log variant that
+// emits this property as a JSON number instead of a numeric string.
+export const JobStartEventSchema = z.object({
+  Event: z.literal('SparkListenerJobStart'),
+  'Job ID': z.number().optional(),
+  'Submission Time': z.number().optional(),
+  'Stage IDs': z.array(z.number()).optional(),
+  Properties: z.object({
+    'spark.sql.execution.id': z.union([z.string(), z.number()]).optional(),
+  }).optional(),
+});
+
+// event-handlers.js:347-360 (endJob)
+export const JobEndEventSchema = z.object({
+  Event: z.literal('SparkListenerJobEnd'),
+  'Job ID': z.number(),
+  'Completion Time': z.number().optional(),
+  'Job Result': z.object({
+    Result: z.string().optional(),
+    Exception: z.object({
+      Message: z.string().optional(),
+    }).optional(),
+  }).optional(),
+});
+
+// event-handlers.js:362-389 (submitStage) + mergeStageRddInfo
+// (event-handlers.js:391-422, called from submitStage with the same 'Stage
+// Info' object, so its 'RDD Info' reads are part of this event's shape too)
+const RddInfoSchema = z.object({
+  'RDD ID': z.number().optional(),
+  Name: z.string().optional(),
+  Callsite: z.string().optional(),
+  'Storage Level': z.object({
+    'Use Disk': z.boolean().optional(),
+    'Use Memory': z.boolean().optional(),
+    Deserialized: z.boolean().optional(),
+    Replication: z.number().optional(),
+  }).optional(),
+  'Number of Partitions': z.number().optional(),
+  // 'Number of Cached Partitions' / 'Memory Size' / 'Disk Size' are read via
+  // `rdd[field] || prev?.field || 0` (mergeStageRddInfo, event-handlers.ts:
+  // 594-596): a `||` fallback is tolerant of the key being absent
+  // (undefined || ... just falls through to the next fallback), so a real
+  // event-log variant that omits one of these keys must still parse.
+  'Number of Cached Partitions': z.number().optional(),
+  'Memory Size': z.number().optional(),
+  'Disk Size': z.number().optional(),
+});
+
+export const StageSubmittedEventSchema = z.object({
+  Event: z.literal('SparkListenerStageSubmitted'),
+  'Stage Info': z.object({
+    'Stage ID': z.number(),
+    'Stage Name': z.string().optional(),
+    Details: z.string().optional(),
+    'Submission Time': z.number().optional(),
+    'Parent IDs': z.array(z.number()).optional(),
+    'RDD Info': z.array(RddInfoSchema).optional(),
+  }),
+});
+
+// event-handlers.js:543-551 (inline StageCompleted case)
+export const StageCompletedEventSchema = z.object({
+  Event: z.literal('SparkListenerStageCompleted'),
+  'Stage Info': z.object({
+    'Stage ID': z.number(),
+    'Completion Time': z.number().optional(),
+    'Failure Reason': z.string().optional(),
+  }),
+});
+
+// event-handlers.js:431-444 (recordStageExecutorMetrics)
+export const StageExecutorMetricsEventSchema = z.object({
+  Event: z.literal('SparkListenerStageExecutorMetrics'),
+  'Stage ID': z.number(),
+  'Executor ID': z.string(),
+  'Executor Metrics': z.record(z.string(), z.number()).optional(),
+});
+
+const MAX_ACCUMULABLES_PER_TASK = 10_000;
+
+// event-handlers.js:147-231 (accumulateTask)
+export const TaskEndEventSchema = z.object({
+  Event: z.literal('SparkListenerTaskEnd'),
+  'Stage ID': z.number(),
+  'Stage Attempt ID': z.number().optional(),
+  'Task End Reason': z.object({
+    Reason: z.string().optional(),
+  }).optional(),
+  'Task Info': z.object({
+    'Launch Time': z.number().optional(),
+    'Finish Time': z.number().optional(),
+    // Failed/Killed are read via `info['Failed'] || info['Killed']`
+    // (accumulateTask, event-handlers.ts:340): a `||` fallback is tolerant
+    // of either key being absent (undefined || undefined is falsy, no
+    // crash), so real event-log variants that omit these keys must still
+    // parse rather than have the whole TaskEnd event rejected.
+    Failed: z.boolean().optional(),
+    Killed: z.boolean().optional(),
+    // Speculative is read via `info['Speculative'] === true`
+    // (accumulateTask, event-handlers.ts:347): a strict-equality check is
+    // also tolerant of the key being absent (undefined === true is just
+    // false), so this is optional too.
+    Speculative: z.boolean().optional(),
+    Host: z.string().optional(),
+    'Executor ID': z.string().optional(),
+    Locality: z.string().optional(),
+    Index: z.number().optional(),
+    // Bounded well above any real plan's per-task metric count (see
+    // MAX_PLAN_DEPTH above for the same defensive intent): caps how much a
+    // single crafted TaskEnd can grow event-handlers.ts's taskAccumStages,
+    // which is never pruned for the life of the parse.
+    Accumulables: z.array(z.object({
+      ID: z.number(),
+      Update: z.union([z.string(), z.number()]).optional(),
+      Value: z.union([z.string(), z.number()]).optional(),
+    })).max(MAX_ACCUMULABLES_PER_TASK).optional(),
+  }).optional(),
+  'Task Metrics': z.object({
+    'Peak Execution Memory': z.number().optional(),
+    'JVM GC Time': z.number().optional(),
+    'Memory Bytes Spilled': z.number().optional(),
+    'Disk Bytes Spilled': z.number().optional(),
+    'Executor Run Time': z.number().optional(),
+    'Executor CPU Time': z.number().optional(),
+    'Shuffle Read Metrics': z.object({
+      'Remote Bytes Read': z.number().optional(),
+      'Local Bytes Read': z.number().optional(),
+      'Fetch Wait Time': z.number().optional(),
+    }).optional(),
+    'Shuffle Write Metrics': z.object({
+      'Shuffle Bytes Written': z.number().optional(),
+    }).optional(),
+    'Input Metrics': z.object({
+      'Bytes Read': z.number().optional(),
+    }).optional(),
+    'Output Metrics': z.object({
+      'Bytes Written': z.number().optional(),
+    }).optional(),
+  }).optional(),
+});
+
+// event-handlers.js:446-460 (startSqlExecution)
+export const SqlExecutionStartEventSchema = z.object({
+  Event: z.literal('org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart'),
+  executionId: z.number(),
+  description: z.string().optional(),
+  time: z.number(),
+  physicalPlanDescription: z.string().optional(),
+  sparkPlanInfo: SparkPlanInfoFieldSchema,
+});
+
+// event-handlers.js (applyAdaptiveExecutionUpdate): AQE re-plans mid-query
+// and re-emits sparkPlanInfo for the same executionId; last write wins.
+export const SqlAdaptiveExecutionUpdateEventSchema = z.object({
+  Event: z.literal('org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate'),
+  executionId: z.number(),
+  physicalPlanDescription: z.string().optional(),
+  sparkPlanInfo: SparkPlanInfoFieldSchema,
+});
+
+// event-handlers.js:462-478 (endSqlExecution)
+export const SqlExecutionEndEventSchema = z.object({
+  Event: z.literal('org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd'),
+  executionId: z.number(),
+  time: z.number(),
+});
+
+// event-handlers.js:480-488 (applyDriverAccumUpdates)
+export const DriverAccumUpdatesEventSchema = z.object({
+  Event: z.literal('org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates'),
+  executionId: z.number(),
+  accumUpdates: z.array(z.tuple([z.number(), z.number()])),
+});
+
+// ---------------------------------------------------------------------------
+// Discriminated union: exactly 16 entries, one per `case` in processEvent's
+// switch (event-handlers.js:514-577).
+// ---------------------------------------------------------------------------
+
+export const SparkEventSchema = z.discriminatedUnion('Event', [
+  LogStartEventSchema,
+  ApplicationStartEventSchema,
+  EnvironmentUpdateEventSchema,
+  ApplicationEndEventSchema,
+  JobStartEventSchema,
+  JobEndEventSchema,
+  StageSubmittedEventSchema,
+  StageCompletedEventSchema,
+  StageExecutorMetricsEventSchema,
+  TaskEndEventSchema,
+  SqlExecutionStartEventSchema,
+  SqlAdaptiveExecutionUpdateEventSchema,
+  SqlExecutionEndEventSchema,
+  DriverAccumUpdatesEventSchema,
+  ExecutorAddedEventSchema,
+  ExecutorRemovedEventSchema,
+]);
+                                                          

package/vendor-core/evidence-availability.js

@@ -0,0 +1,121 @@
+                                                                                                                                                                            
+
+// Not annotated `: number` on purpose: `const X = 1` infers the literal type
+// `1`, which is assignable both wherever `number` is expected (e.g. equality
+// checks below) and wherever the `EvidenceAvailability.schemaVersion: 1`
+// literal field is expected (the `return` at the bottom of this file);
+// widening it to `number` would break that assignment.
+export const EVIDENCE_AVAILABILITY_SCHEMA_VERSION = 1;
+
+const SUMMARIES                                     = {
+  observed: 'Observed in this event log.',
+  explicitlyDisabled: 'Explicitly disabled in this event log.',
+  noObservedExecutorMetrics: 'Not emitted by this event log.',
+  noObservedStageSubmission: 'Not emitted by this event log.',
+  noRddStorageSnapshot: 'Stages were submitted without RDD storage snapshots.',
+  noResolvedSqlPlan: 'Not emitted by this event log.',
+  noSqlExecution: 'Not applicable to this event log.',
+  noEnvironmentUpdate: 'Not emitted by this event log.',
+  noTaskRecords: 'Not emitted by this event log.',
+  noUsableCoreTimeAggregate: 'No usable aggregate was emitted.',
+  outsideEventLogScope: 'Available outside local event-log scope.',
+  parseIncomplete: 'Cannot determine from an incomplete parse.',
+};
+
+function observed(key             , eventType                   , count        )                            {
+  return { key, state: 'present', reasonCode: 'observed', summary: SUMMARIES.observed, evidence: { eventType, count } };
+}
+
+function entry(key             , state               , reasonCode                    )                            {
+  return { key, state, reasonCode, summary: SUMMARIES[reasonCode] };
+}
+
+function count(inputs                         , key                      )         {
+  const value = inputs[key];
+  return typeof value === 'number' && Number.isFinite(value) && value > 0 ? value : 0;
+}
+
+/** True when the serialized value is a ledger this build understands. Fails
+ * closed on any unknown `schemaVersion` so a future, incompatible ledger is
+ * never trusted as V1 data; used at read/restore time, not just write time. */
+export function isSupportedEvidenceAvailability(value         )          {
+  return value != null && (value                               ).schemaVersion === EVIDENCE_AVAILABILITY_SCHEMA_VERSION;
+}
+
+function isPositiveFinite(value         )                  {
+  return typeof value === 'number' && Number.isFinite(value) && value > 0;
+}
+
+/** A stage contributes usable core-time evidence only when it both ran tasks
+ * and produced a positive duration sum: a degenerate aggregate (every task
+ * with `finish <= launch`) carries `totalTaskDurationSum <= 0` and proves
+ * nothing, so it must not report `present`. Shared with ScalingSim so the
+ * two consumers cannot drift. */
+export function hasUsableRunAggregates(runAggregates         )          {
+  const perStage = (runAggregates                                             )?.perStage;
+  return perStage != null
+    && typeof perStage === 'object'
+    && !Array.isArray(perStage)
+    && Object.values(perStage                                                                           ).some(
+      (stage) => isPositiveFinite(stage?.taskCount) && isPositiveFinite(stage?.totalTaskDurationSum),
+    );
+}
+
+/** Matches the parser's config normalization (`.toLowerCase() === 'true'`),
+ * so a real boolean or mixed-case `'False'` is handled identically. */
+function configIs(value         , expected        )          {
+  return value != null && String(value).toLowerCase() === expected;
+}
+
+function absent(key             , reasonCode                    , trustworthy         )                            {
+  return trustworthy ? entry(key, reasonCode === 'noSqlExecution' ? 'notApplicable' : 'notEmitted', reasonCode) : entry(key, 'unknown', 'parseIncomplete');
+}
+
+export function deriveEvidenceAvailability(appModel          , { skippedLines = 0 }                            = {})                       {
+  const app = appModel?.app;
+  const inputs                          = app?.evidenceInputs ?? {};
+  const trustworthy = skippedLines === 0 && count(inputs, 'applicationEnds') > 0;
+  const metricRows = count(inputs, 'executorMetricRows');
+  const submissions = count(inputs, 'stageSubmissions');
+  const snapshots = count(inputs, 'rddStorageSnapshots');
+  const plans = count(inputs, 'resolvedSqlPlans');
+  const executions = count(inputs, 'sqlExecutions');
+  const environments = count(inputs, 'environmentUpdates');
+  const taskRecords = count(inputs, 'taskRecords');
+
+  const executorMetrics = metricRows > 0
+    ? observed('executorMetrics', 'executorMetricRows', metricRows)
+    : configIs(app?.config?.['spark.eventLog.logStageExecutorMetrics'], 'false')
+      ? entry('executorMetrics', 'disabled', 'explicitlyDisabled')
+      : absent('executorMetrics', 'noObservedExecutorMetrics', trustworthy);
+  const rddStorageSnapshots = snapshots > 0
+    ? observed('rddStorageSnapshots', 'rddStorageSnapshots', snapshots)
+    : absent('rddStorageSnapshots', submissions === 0 ? 'noObservedStageSubmission' : 'noRddStorageSnapshot', trustworthy);
+  const sqlPlan = plans > 0
+    ? observed('sqlPlan', 'resolvedSqlPlans', plans)
+    : executions > 0 && trustworthy
+      ? entry('sqlPlan', 'notEmitted', 'noResolvedSqlPlan')
+      : absent('sqlPlan', 'noSqlExecution', trustworthy);
+  const sparkConfiguration = environments > 0
+    ? observed('sparkConfiguration', 'environmentUpdates', environments)
+    : absent('sparkConfiguration', 'noEnvironmentUpdate', trustworthy);
+  const taskCoreTime = taskRecords === 0
+    ? absent('taskCoreTime', 'noTaskRecords', trustworthy)
+    : hasUsableRunAggregates(appModel?.runAggregates)
+      ? observed('taskCoreTime', 'taskRecords', taskRecords)
+      : absent('taskCoreTime', 'noUsableCoreTimeAggregate', trustworthy);
+
+  return {
+    schemaVersion: EVIDENCE_AVAILABILITY_SCHEMA_VERSION,
+    entries: [
+      executorMetrics,
+      rddStorageSnapshots,
+      sqlPlan,
+      sparkConfiguration,
+      taskCoreTime,
+      entry('infrastructureContext', 'outsideEventLog', 'outsideEventLogScope'),
+      entry('sourceContext', 'outsideEventLog', 'outsideEventLogScope'),
+      entry('costContext', 'outsideEventLog', 'outsideEventLogScope'),
+    ],
+  };
+}