sparkforensics-cli 0.1.0 → 0.2.2

package/vendor-core/event-schemas.js

@@ -1,26 +1,14 @@
 import { z } from 'zod';
 
-// ---------------------------------------------------------------------------
-// sparkPlanInfo: iterative (non-recursive-schema) tree parser
+// sparkPlanInfo: iterative (non-recursive-schema) tree parser.
 //
-// Validates the raw, unbounded-depth `sparkPlanInfo` tree carried on
-// SparkListenerSQLExecutionStart/End events (consumed today by
-// `resolvePlanTree` in event-handlers.js:233-273). Per the spec, this must
-// NEVER be validated with z.lazy(): it uses an explicit heap-allocated stack,
-// the same iterative-over-recursive approach src/plan-tree-walk.js's
-// walkPlanTree already uses for the *resolved* tree (that one walks an
-// already-built tree; this one builds one from raw JSON).
-// ---------------------------------------------------------------------------
+// Validates the raw, unbounded-depth sparkPlanInfo tree on SQLExecutionStart/End events. Must
+// NEVER be validated with z.lazy(): it uses an explicit heap-allocated stack so zod never sees the
+// full tree depth in one call.
 
-// `resolvePlanTree` (event-handlers.ts) reads `m.accumulatorId` to look up
-// the accumulator's live value in accumMap; this field was missing from the
-// original (Task 6) schema, so it was silently stripped by object-schema
-// validation before resolvePlanTree ever saw it, breaking metric resolution
-// for every plan node once real events start flowing through
-// SparkEventSchema.safeParse. Declared optional (not required) to keep
-// accepting tests/event-schemas.test.js's existing 'value'-only fixture;
-// resolvePlanTree already tolerates a missing accumulatorId (accumMap.has
-// just resolves false, the metric contributes no value).
+// resolvePlanTree reads m.accumulatorId to look up the accumulator's live value in accumMap.
+// Optional (not required): resolvePlanTree tolerates a missing accumulatorId (accumMap.has
+// resolves false, the metric contributes no value).
 const SparkPlanMetricSchema = z.object({
   name: z.string(),
   accumulatorId: z.number().optional(),
@@ -28,17 +16,10 @@ const SparkPlanMetricSchema = z.object({
   metricType: z.string().optional(),
 });
 
-// Shallow shape only. `children` is intentionally z.array(z.unknown()) here:
-// each child gets its own shallow parse inside the iterative walk below, so
-// this schema never recurses into itself and zod never sees the full tree
-// depth in one call.
-//
-// `metadata` (file-scan nodes carry Location/Format/ReadSchema/PushedFilters/
-// PartitionFilters/DataFilters/Batched here) was missing from the original
-// schema, so it was silently stripped before ever reaching the posted `sql`
-// message's `sparkPlanInfo` (same data-loss class as the accumulatorId gap
-// above). Nothing reads it yet, but it's real data crossing the worker
-// boundary, so it's kept rather than dropped.
+// Shallow shape only. `children` is z.array(z.unknown()): each child gets its own shallow parse
+// in the iterative walk below, so this schema never recurses and zod never sees the full depth.
+// `metadata` (file-scan Location/Format/ReadSchema/...) is kept though nothing reads it yet: real
+// data crossing the worker boundary.
 const SparkPlanInfoNodeSchema = z.object({
   nodeName: z.string(),
   simpleString: z.string().optional(),
@@ -94,11 +75,8 @@ export function parseSparkPlanInfoTree(raw         )                {
   throw new Error('unreachable: sparkPlanInfo tree stack exhausted without resolving root');
 }
 
-// A zod field schema wrapping parseSparkPlanInfoTree, for use inside the
-// SQLExecutionStart event schema. Nullable/optional to mirror
-// `event.sparkPlanInfo ?? null` in startSqlExecution (event-handlers.js:448).
-// Deliberately NOT z.lazy(): defers to the iterative parser above, so the
-// full tree depth is still never handed to zod's own recursion.
+// A zod field wrapping parseSparkPlanInfoTree for the SQLExecutionStart schema. Nullable/optional
+// to mirror `event.sparkPlanInfo ?? null`. Deliberately NOT z.lazy(): defers to the iterative parser.
 const SparkPlanInfoFieldSchema = z
   .unknown()
   .nullable()
@@ -113,19 +91,15 @@ const SparkPlanInfoFieldSchema = z
     }
   });
 
-// ---------------------------------------------------------------------------
-// Six fully-specified event schemas (verbatim per task-6-brief.md Step 3;
-// backed by exact quoted source cited in each comment)
-// ---------------------------------------------------------------------------
+// Fully-specified event schemas.
 
-// event-handlers.js:516-518 (`SparkListenerLogStart` case, inline: sets
-// state.pendingSparkVersion)
+// SparkListenerLogStart case (inline: sets state.pendingSparkVersion).
 export const LogStartEventSchema = z.object({
   Event: z.literal('SparkListenerLogStart'),
   'Spark Version': z.string().optional(),
 });
 
-// event-handlers.js:293-306 (startApplication)
+// startApplication.
 export const ApplicationStartEventSchema = z.object({
   Event: z.literal('SparkListenerApplicationStart'),
   'App ID': z.string().optional(),
@@ -134,20 +108,9 @@ export const ApplicationStartEventSchema = z.object({
   'Spark Version': z.string().optional(),
 });
 
-// event-handlers.js:308-323 (updateEnvironment): 'Spark Properties' is fed
-// straight into normalizeSparkProperties(props), which already accepts
-// object | array | null (src/format-utils... no, src/event-handlers.js:53-62)
-//
-// Property VALUES were previously required to be z.string(), but
-// normalizeSparkProperties (event-handlers.ts:211-226) just spread each
-// value through (`{ ...(props ?? {}) }` / `map[pair[0]] = pair[1]`) with no
-// requirement that it already be a string, so a real Spark config carrying a
-// non-string-looking JSON value (a bare number or boolean) was rejecting the
-// entire EnvironmentUpdate event instead of parsing it. Widened to the
-// realistic set of raw JSON scalar types; normalizeSparkProperties now
-// coerces each value to a string at the boundary so its `Record<string,
-// string>` return contract (and every downstream reader of `app.config`)
-// is unaffected.
+// updateEnvironment: 'Spark Properties' is fed to normalizeSparkProperties, which accepts
+// object | array | null. Property VALUES are string|number|boolean (a real config can carry a
+// bare JSON number/boolean); normalizeSparkProperties coerces each to a string at the boundary.
 const SparkPropertyValueSchema = z.union([z.string(), z.number(), z.boolean()]);
 export const EnvironmentUpdateEventSchema = z.object({
   Event: z.literal('SparkListenerEnvironmentUpdate'),
@@ -158,20 +121,15 @@ export const EnvironmentUpdateEventSchema = z.object({
   ]).optional(),
 });
 
-// event-handlers.js:526-532 (inline ApplicationEnd case)
+// inline ApplicationEnd case.
 export const ApplicationEndEventSchema = z.object({
   Event: z.literal('SparkListenerApplicationEnd'),
   Timestamp: z.number().optional(),
 });
 
-// event-handlers.js:490-502 (addExecutor). `Resource Profile Id` is read from
-// BOTH inside 'Executor Info' (the real Spark placement) and top-level (a
-// fallback for a variant that hoists it) via `event['Executor Info']?.[...] ??
-// event['Resource Profile Id']`, so both locations must be declared or the
-// nested (primary) one is silently stripped by object-schema validation.
-// `Timestamp` is read as `event['Timestamp']` with no defensive operator, so
-// per the optionality rule it's required, not guessed-optional (it also
-// backs ExecutorAddedEvent.timestamp: number, a non-optional field).
+// addExecutor. `Resource Profile Id` is read from BOTH inside 'Executor Info' (real Spark
+// placement) and top-level (a variant that hoists it), so both must be declared or the nested one
+// is silently stripped. `Timestamp` is read with no defensive operator, so it's required.
 export const ExecutorAddedEventSchema = z.object({
   Event: z.literal('SparkListenerExecutorAdded'),
   Timestamp: z.number(),
@@ -184,9 +142,7 @@ export const ExecutorAddedEventSchema = z.object({
   'Resource Profile Id': z.number().optional(),
 });
 
-// event-handlers.js:504-512 (removeExecutor). `Timestamp` is read as
-// `event['Timestamp']` with no defensive operator, so per the optionality
-// rule it's required (backs ExecutorRemovedEvent.timestamp: number).
+// removeExecutor. `Timestamp` is read with no defensive operator, so it's required.
 export const ExecutorRemovedEventSchema = z.object({
   Event: z.literal('SparkListenerExecutorRemoved'),
   Timestamp: z.number(),
@@ -194,18 +150,10 @@ export const ExecutorRemovedEventSchema = z.object({
   'Removed Reason': z.string().optional(),
 });
 
-// ---------------------------------------------------------------------------
-// Nine schemas transcribed from their handlers (task-6-brief.md Step 4). See
-// task-6-report.md for the exact field-by-field derivation from each handler.
-// ---------------------------------------------------------------------------
+// Schemas transcribed from their handlers.
 
-// event-handlers.js:325-345 (startJob). 'spark.sql.execution.id' is read via
-// `event['Properties']?.['spark.sql.execution.id']` then
-// `parseInt(sqlExecIdStr, 10)` (startJob, event-handlers.ts:498-501):
-// parseInt coerces its argument to a string internally, so it tolerates the
-// raw value already being a number just as well as a string; requiring
-// z.string() here rejected the whole JobStart event for a log variant that
-// emits this property as a JSON number instead of a numeric string.
+// startJob. 'spark.sql.execution.id' is read then parseInt'd, which coerces to string internally,
+// so it tolerates the raw value being a number; widened to string|number (some logs emit a JSON number).
 export const JobStartEventSchema = z.object({
   Event: z.literal('SparkListenerJobStart'),
   'Job ID': z.number().optional(),
@@ -216,7 +164,7 @@ export const JobStartEventSchema = z.object({
   }).optional(),
 });
 
-// event-handlers.js:347-360 (endJob)
+// endJob.
 export const JobEndEventSchema = z.object({
   Event: z.literal('SparkListenerJobEnd'),
   'Job ID': z.number(),
@@ -229,9 +177,8 @@ export const JobEndEventSchema = z.object({
   }).optional(),
 });
 
-// event-handlers.js:362-389 (submitStage) + mergeStageRddInfo
-// (event-handlers.js:391-422, called from submitStage with the same 'Stage
-// Info' object, so its 'RDD Info' reads are part of this event's shape too)
+// submitStage + mergeStageRddInfo (called from submitStage with the same 'Stage Info', so its
+// 'RDD Info' reads are part of this event's shape too).
 const RddInfoSchema = z.object({
   'RDD ID': z.number().optional(),
   Name: z.string().optional(),
@@ -243,11 +190,8 @@ const RddInfoSchema = z.object({
     Replication: z.number().optional(),
   }).optional(),
   'Number of Partitions': z.number().optional(),
-  // 'Number of Cached Partitions' / 'Memory Size' / 'Disk Size' are read via
-  // `rdd[field] || prev?.field || 0` (mergeStageRddInfo, event-handlers.ts:
-  // 594-596): a `||` fallback is tolerant of the key being absent
-  // (undefined || ... just falls through to the next fallback), so a real
-  // event-log variant that omits one of these keys must still parse.
+  // 'Number of Cached Partitions'/'Memory Size'/'Disk Size' are read via `rdd[field] || prev || 0`,
+  // tolerant of the key being absent, so a variant that omits one must still parse.
   'Number of Cached Partitions': z.number().optional(),
   'Memory Size': z.number().optional(),
   'Disk Size': z.number().optional(),
@@ -265,7 +209,7 @@ export const StageSubmittedEventSchema = z.object({
   }),
 });
 
-// event-handlers.js:543-551 (inline StageCompleted case)
+// inline StageCompleted case.
 export const StageCompletedEventSchema = z.object({
   Event: z.literal('SparkListenerStageCompleted'),
   'Stage Info': z.object({
@@ -275,7 +219,7 @@ export const StageCompletedEventSchema = z.object({
   }),
 });
 
-// event-handlers.js:431-444 (recordStageExecutorMetrics)
+// recordStageExecutorMetrics.
 export const StageExecutorMetricsEventSchema = z.object({
   Event: z.literal('SparkListenerStageExecutorMetrics'),
   'Stage ID': z.number(),
@@ -285,7 +229,7 @@ export const StageExecutorMetricsEventSchema = z.object({
 
 const MAX_ACCUMULABLES_PER_TASK = 10_000;
 
-// event-handlers.js:147-231 (accumulateTask)
+// accumulateTask.
 export const TaskEndEventSchema = z.object({
   Event: z.literal('SparkListenerTaskEnd'),
   'Stage ID': z.number(),
@@ -296,26 +240,20 @@ export const TaskEndEventSchema = z.object({
   'Task Info': z.object({
     'Launch Time': z.number().optional(),
     'Finish Time': z.number().optional(),
-    // Failed/Killed are read via `info['Failed'] || info['Killed']`
-    // (accumulateTask, event-handlers.ts:340): a `||` fallback is tolerant
-    // of either key being absent (undefined || undefined is falsy, no
-    // crash), so real event-log variants that omit these keys must still
-    // parse rather than have the whole TaskEnd event rejected.
+    // Failed/Killed are read via `info['Failed'] || info['Killed']`, tolerant of either being
+    // absent, so variants that omit them must still parse.
     Failed: z.boolean().optional(),
     Killed: z.boolean().optional(),
-    // Speculative is read via `info['Speculative'] === true`
-    // (accumulateTask, event-handlers.ts:347): a strict-equality check is
-    // also tolerant of the key being absent (undefined === true is just
-    // false), so this is optional too.
+    // Speculative is read via `=== true`, also tolerant of the key being absent, so it's optional too.
     Speculative: z.boolean().optional(),
     Host: z.string().optional(),
     'Executor ID': z.string().optional(),
     Locality: z.string().optional(),
     Index: z.number().optional(),
-    // Bounded well above any real plan's per-task metric count (see
-    // MAX_PLAN_DEPTH above for the same defensive intent): caps how much a
-    // single crafted TaskEnd can grow event-handlers.ts's taskAccumStages,
-    // which is never pruned for the life of the parse.
+    'Task ID': z.number().optional(),
+    'Attempt Number': z.number().optional(),
+    // Bounded well above any real per-task metric count: caps how much a crafted TaskEnd can grow
+    // taskAccumStages, which is never pruned for the life of the parse.
     Accumulables: z.array(z.object({
       ID: z.number(),
       Update: z.union([z.string(), z.number()]).optional(),
@@ -346,7 +284,7 @@ export const TaskEndEventSchema = z.object({
   }).optional(),
 });
 
-// event-handlers.js:446-460 (startSqlExecution)
+// startSqlExecution.
 export const SqlExecutionStartEventSchema = z.object({
   Event: z.literal('org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionStart'),
   executionId: z.number(),
@@ -356,8 +294,8 @@ export const SqlExecutionStartEventSchema = z.object({
   sparkPlanInfo: SparkPlanInfoFieldSchema,
 });
 
-// event-handlers.js (applyAdaptiveExecutionUpdate): AQE re-plans mid-query
-// and re-emits sparkPlanInfo for the same executionId; last write wins.
+// applyAdaptiveExecutionUpdate: AQE re-plans mid-query and re-emits sparkPlanInfo for the same
+// executionId; last write wins.
 export const SqlAdaptiveExecutionUpdateEventSchema = z.object({
   Event: z.literal('org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate'),
   executionId: z.number(),
@@ -365,25 +303,21 @@ export const SqlAdaptiveExecutionUpdateEventSchema = z.object({
   sparkPlanInfo: SparkPlanInfoFieldSchema,
 });
 
-// event-handlers.js:462-478 (endSqlExecution)
+// endSqlExecution.
 export const SqlExecutionEndEventSchema = z.object({
   Event: z.literal('org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd'),
   executionId: z.number(),
   time: z.number(),
 });
 
-// event-handlers.js:480-488 (applyDriverAccumUpdates)
+// applyDriverAccumUpdates.
 export const DriverAccumUpdatesEventSchema = z.object({
   Event: z.literal('org.apache.spark.sql.execution.ui.SparkListenerDriverAccumUpdates'),
   executionId: z.number(),
   accumUpdates: z.array(z.tuple([z.number(), z.number()])),
 });
 
-// ---------------------------------------------------------------------------
-// Discriminated union: exactly 16 entries, one per `case` in processEvent's
-// switch (event-handlers.js:514-577).
-// ---------------------------------------------------------------------------
-
+// Discriminated union: one entry per `case` in processEvent's switch.
 export const SparkEventSchema = z.discriminatedUnion('Event', [
   LogStartEventSchema,
   ApplicationStartEventSchema,

package/vendor-core/evidence-availability.js

@@ -1,10 +1,7 @@
                                                                                                                                                                             
 
-// Not annotated `: number` on purpose: `const X = 1` infers the literal type
-// `1`, which is assignable both wherever `number` is expected (e.g. equality
-// checks below) and wherever the `EvidenceAvailability.schemaVersion: 1`
-// literal field is expected (the `return` at the bottom of this file);
-// widening it to `number` would break that assignment.
+// Not annotated `: number` on purpose: `const X = 1` infers the literal type `1`, assignable to
+// the EvidenceAvailability.schemaVersion:1 literal field; widening to `number` would break that.
 export const EVIDENCE_AVAILABILITY_SCHEMA_VERSION = 1;
 
 const SUMMARIES                                     = {
@@ -46,11 +43,9 @@ function isPositiveFinite(value         )                  {
   return typeof value === 'number' && Number.isFinite(value) && value > 0;
 }
 
-/** A stage contributes usable core-time evidence only when it both ran tasks
- * and produced a positive duration sum: a degenerate aggregate (every task
- * with `finish <= launch`) carries `totalTaskDurationSum <= 0` and proves
- * nothing, so it must not report `present`. Shared with ScalingSim so the
- * two consumers cannot drift. */
+/** A stage contributes usable core-time evidence only when it both ran tasks and produced a
+ * positive duration sum: a degenerate aggregate (every task finish <= launch) proves nothing.
+ * Shared with ScalingSim so the two consumers can't drift. */
 export function hasUsableRunAggregates(runAggregates         )          {
   const perStage = (runAggregates                                             )?.perStage;
   return perStage != null