showdar-skills 0.6.0 → 0.7.0

package/CHANGELOG.md

@@ -4,6 +4,37 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 
+## [0.7.0]
+
+### Added
+
+- Deterministic workflow trace projection (`src/workflow-trace.js`): pure
+  state-diff observation over workflow transitions with 10 closed semantic
+  event types, no timestamps, no authority content, and no automatic
+  persistence.
+- Workflow benchmark scenario schema and loader
+  (`benchmark/schema/workflow-scenario.schema.json`,
+  `benchmark/lib/workflow-scenario-loader.js`).
+- Deterministic workflow benchmark corpus
+  (`benchmark/scenarios/workflows/`, 16 scenarios): exact-match M1–M10
+  invariants covering selection, skip policy, verification preservation,
+  stale-resume blocking, authority invariance, and completion.
+- `npm run eval:workflows` driver
+  (`scripts/workflow-observability-eval.mjs`).
+
+### Changed
+
+- `npm run eval` now runs retrieval evaluation (`eval:retrieval`) followed
+  by workflow evaluation (`eval:workflows`); release evaluation blocks on
+  exact workflow invariants.
+- `npm run check` remains test/validate/package correctness only.
+
+### Safety
+
+- Traces contain normalized semantic data only (IDs, enums, receipt
+  summaries, statuses): no authority state, raw prompts, logs, telemetry,
+  or network reporting.
+
 ## [0.6.0]
 
 ### Added

package/MIGRATION.md

@@ -1,3 +1,20 @@
+# Migrating to 0.7.0
+
+0.7.0 adds deterministic workflow trace projection
+(`src/workflow-trace.js`) and a workflow benchmark corpus over the
+unchanged 0.6 runtime. No user action is required.
+
+- Existing 0.6 installs and configs remain valid: no workflow-state
+  schema migration (still schemaVersion 1), no config migration, no
+  adapter migration, no checkpoint migration.
+- Runtime workflow semantics are unchanged; traces observe released
+  behavior only and never affect execution, routing, or authority.
+- The new `src/workflow-trace.js` module ships in the package; benchmark
+  scenarios, schema, loader, and eval driver are development/release
+  tooling and do not ship.
+- No telemetry, network reporting, automatic storage, or tracking is
+  introduced.
+
 # Migrating to 0.6.0
 
 0.6.0 adds portable workflow execution state (`src/workflow-state.js`,

package/README.md

@@ -443,6 +443,39 @@ Per-workflow skip policy (evidence-backed, never severity or wording alone):
 - Incident: severity never grants production mutation; recovery and
   verification remain gated by current authority.
 
+### Workflow traces (observability, benchmark-only)
+
+Workflow traces are pure projections of before/after workflow states
+(`src/workflow-trace.js`): ordered events such as `stage-entered`,
+`stage-completed`, `stage-skipped`, `workflow-blocked`, `workflow-resumed`,
+and `workflow-completed`. Traces carry stage IDs, skip reasons, receipt
+summaries, and statuses only — no timestamps, no prompts, no secrets, no
+authority content. Nothing persists them automatically; the benchmark
+corpus (`npm run eval:workflows`) uses them to verify selection, skip,
+resume, and completion behavior deterministically.
+
+A workflow trace does not mutate workflow state, affect routing or
+authority, persist automatically, or send telemetry. The 10 event
+categories are `workflow-created`, `stages-selected`, `stage-entered`,
+`evidence-recorded`, `stage-completed`, `stage-skipped`,
+`workflow-blocked`, `workflow-interrupted`, `workflow-resumed`, and
+`workflow-completed`.
+
+### Evaluation
+
+```bash
+npm run eval:retrieval  # retrieval evaluation (unchanged behavior)
+npm run eval:workflows  # deterministic workflow semantic benchmark
+npm run eval            # both, sequentially (release-blocking)
+```
+
+Workflow evaluation asserts exact M1–M10 invariants (selection accuracy,
+invalid-skip rejection, verification preservation, stale-resume blocking,
+authority invariance, completion, trace equality, revision monotonicity,
+checkpoint round-trip, skip-evidence backing) with no fuzzy score
+thresholds. `npm run check` (test/validate/pack) does not run the
+benchmark; the release pipeline runs `npm run eval`, gating both suites.
+
 ## A typical software workflow
 
 ```text

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "showdar-skills",
-  "version": "0.6.0",
+  "version": "0.7.0",
   "description": "Production-grade software engineering lifecycle skills for coding agents.",
   "type": "module",
   "bin": { "showdar": "./bin/showdar.js" },
@@ -8,7 +8,9 @@
     "test": "node --test",
     "validate": "node bin/showdar.js validate",
     "sync:runtime": "node scripts/sync-runtime.mjs",
-    "eval": "node scripts/retrieval-eval.mjs",
+    "eval": "npm run eval:retrieval && npm run eval:workflows",
+    "eval:retrieval": "node scripts/retrieval-eval.mjs",
+    "eval:workflows": "node scripts/workflow-observability-eval.mjs",
     "check": "npm test && npm run validate && npm pack --dry-run",
     "smoke": "node scripts/package-smoke.mjs",
     "release:check": "node scripts/check-release-version.mjs"

package/src/validate.js

@@ -266,6 +266,64 @@ export async function validateWorkflowStatePolicy(packageRoot) {
   for (const storage of ['.showdar/state', '~/.showdar', 'sqlite', 'telemetry']) {
     if (moduleText.toLowerCase().includes(storage.toLowerCase())) errors.push(`workflow-state: no storage layer reference ${storage}`);
   }
+  for (const error of (await validateWorkflowScenarioCoverage(packageRoot, policy)).errors) errors.push(error);
+  return { ok: errors.length === 0, errors };
+}
+
+export async function validateWorkflowScenarioCoverage(packageRoot, policy = null) {
+  const errors = [];
+  const primitives = new Set(SKILLS.map((skill) => skill.id));
+  const workflows = new Set(WORKFLOW_SKILLS.map((w) => w.id));
+  let trace;
+  try {
+    trace = await import('./workflow-trace.js');
+  } catch (error) {
+    return { ok: false, errors: [`workflow-trace unavailable: ${error.message}`] };
+  }
+  const eventTypes = new Set(trace.WORKFLOW_EVENT_TYPES);
+  let files;
+  try {
+    files = await readdir(path.join(packageRoot, 'benchmark', 'scenarios', 'workflows'));
+  } catch {
+    return { ok: false, errors: ['workflow-scenarios: benchmark/scenarios/workflows missing'] };
+  }
+  const scenarioFiles = files.filter((f) => f.endsWith('.json')).sort();
+  if (!scenarioFiles.length) return { ok: false, errors: ['workflow-scenarios: no scenario files'] };
+  const seenIds = new Set();
+  const coverage = new Map();
+  for (const file of scenarioFiles) {
+    let scenario;
+    try {
+      scenario = JSON.parse(await readFile(path.join(packageRoot, 'benchmark', 'scenarios', 'workflows', file), 'utf8'));
+    } catch (error) {
+      errors.push(`workflow-scenarios: ${file} is not valid JSON (${error.message})`);
+      continue;
+    }
+    if (seenIds.has(scenario.id)) errors.push(`workflow-scenarios: duplicate id ${scenario.id}`);
+    seenIds.add(scenario.id);
+    if (!workflows.has(scenario.workflow)) {
+      errors.push(`workflow-scenarios: ${file} references unknown workflow ${scenario.workflow}`);
+      continue;
+    }
+    coverage.set(scenario.workflow, (coverage.get(scenario.workflow) ?? 0) + 1);
+    const catalogStages = new Set((policy?.selectableStages?.(scenario.workflow)) ?? []);
+    for (const stage of scenario.selection?.selectedStages ?? []) {
+      if (!primitives.has(stage)) errors.push(`workflow-scenarios: ${file} selects unknown stage ${stage}`);
+      else if (catalogStages.size && !catalogStages.has(stage)) errors.push(`workflow-scenarios: ${file} selects non-catalog stage ${stage}`);
+    }
+    for (const step of scenario.steps ?? []) {
+      if (step.stage && !primitives.has(step.stage)) errors.push(`workflow-scenarios: ${file} step references unknown stage ${step.stage}`);
+    }
+    for (const entry of scenario.expected?.trace ?? []) {
+      if (!eventTypes.has(entry.type)) errors.push(`workflow-scenarios: ${file} expects unknown event type ${entry.type}`);
+      if (entry.stage !== null && entry.stage !== undefined && !primitives.has(entry.stage)) {
+        errors.push(`workflow-scenarios: ${file} expects unknown event stage ${entry.stage}`);
+      }
+    }
+  }
+  for (const workflow of WORKFLOW_SKILLS) {
+    if (!coverage.get(workflow.id)) errors.push(`workflow-scenarios: ${workflow.id} has no scenario coverage`);
+  }
   return { ok: errors.length === 0, errors };
 }
 

package/src/workflow-trace.js

@@ -0,0 +1,183 @@
+import {
+  validateWorkflowState,
+  isWorkflowComplete,
+  FORBIDDEN_AUTHORITY_KEYS,
+} from './workflow-state.js';
+
+export const WORKFLOW_EVENT_TYPES = Object.freeze([
+  'workflow-created',
+  'stages-selected',
+  'stage-entered',
+  'evidence-recorded',
+  'stage-completed',
+  'stage-skipped',
+  'workflow-blocked',
+  'workflow-interrupted',
+  'workflow-resumed',
+  'workflow-completed',
+]);
+
+const EVENT_TYPE_SET = new Set(WORKFLOW_EVENT_TYPES);
+const EVENT_KEYS = new Set(['seq', 'type', 'workflowId', 'revision', 'stage', 'detail']);
+
+function sortedKinds(receipts) {
+  return [...receipts].map((r) => r.kind).sort();
+}
+
+function sortedQualities(receipts) {
+  return [...receipts].map((r) => r.quality).sort();
+}
+
+function freezeEvent(event) {
+  return Object.freeze({ ...event, detail: Object.freeze({ ...event.detail }) });
+}
+
+function makeEvent(seq, type, state, stage, detail) {
+  return freezeEvent({
+    seq,
+    type,
+    workflowId: state.workflowId,
+    revision: state.revision,
+    stage: stage ?? null,
+    detail: detail ?? {},
+  });
+}
+
+export function assertNoAuthority(event) {
+  const serialized = JSON.stringify(event).toLowerCase().replace(/[\s_-]+/g, '');
+  for (const forbidden of FORBIDDEN_AUTHORITY_KEYS) {
+    if (serialized.includes(forbidden)) {
+      throw new Error(`workflow trace event must not contain authority-derived content (found ${forbidden})`);
+    }
+  }
+  return true;
+}
+
+export function validateWorkflowEvent(event) {
+  const errors = [];
+  if (event === null || typeof event !== 'object' || Array.isArray(event)) return { ok: false, errors: ['workflow event must be an object'] };
+  for (const key of Object.keys(event)) {
+    if (!EVENT_KEYS.has(key)) errors.push(`event contains unknown key: ${key}`);
+  }
+  if (!Number.isInteger(event.seq) || event.seq < 0) errors.push('event seq must be a non-negative integer');
+  if (!EVENT_TYPE_SET.has(event.type)) errors.push(`event type must be one of: ${WORKFLOW_EVENT_TYPES.join(', ')}`);
+  if (typeof event.workflowId !== 'string' || !event.workflowId) errors.push('event workflowId must be a non-empty string');
+  if (!Number.isInteger(event.revision) || event.revision < 0) errors.push('event revision must be a non-negative integer');
+  if (event.stage !== null && typeof event.stage !== 'string') errors.push('event stage must be a string or null');
+  if (event.detail === null || typeof event.detail !== 'object' || Array.isArray(event.detail)) errors.push('event detail must be an object');
+  if (!errors.length) {
+    try {
+      assertNoAuthority(event);
+    } catch (error) {
+      errors.push(error.message);
+    }
+  }
+  return { ok: errors.length === 0, errors };
+}
+
+export function eventKey(event) {
+  return JSON.stringify([event.type, event.workflowId, event.stage, event.detail]);
+}
+
+export function normalizeTrace(events) {
+  return events.map(eventKey);
+}
+
+function requireValidState(state, label) {
+  const validation = validateWorkflowState(state);
+  if (!validation.ok) throw new Error(`Invalid ${label} workflow state: ${validation.errors.join('; ')}`);
+}
+
+export function projectWorkflowEvents(prevState, nextState, input = {}) {
+  if (prevState !== null) requireValidState(prevState, 'prev');
+  requireValidState(nextState, 'next');
+  const op = input.op ?? null;
+  if (typeof op !== 'string' || !op) throw new Error('projectWorkflowEvents requires input.op');
+  const events = [];
+  const emit = (type, stage, detail) => {
+    const event = makeEvent(events.length, type, nextState, stage, detail);
+    const validation = validateWorkflowEvent(event);
+    if (!validation.ok) throw new Error(`Projected invalid event: ${validation.errors.join('; ')}`);
+    events.push(event);
+  };
+  switch (op) {
+    case 'create': {
+      emit('workflow-created', null, {
+        selectedStages: [...nextState.selectedStages],
+        candidateStages: [...nextState.candidateStages],
+      });
+      const sameOrder = nextState.selectedStages.length === nextState.candidateStages.length
+        && nextState.selectedStages.every((s, i) => s === nextState.candidateStages[i]);
+      if (!sameOrder) emit('stages-selected', null, { selectedStages: [...nextState.selectedStages] });
+      break;
+    }
+    case 'start':
+      emit('stage-entered', input.stage ?? nextState.activeStage, {});
+      break;
+    case 'record': {
+      const receipts = input.receipts ?? nextState.evidenceReceipts.slice(-1);
+      emit('evidence-recorded', null, { receiptKinds: sortedKinds(receipts), receiptQualities: sortedQualities(receipts) });
+      break;
+    }
+    case 'complete': {
+      const entry = nextState.completedStages[nextState.completedStages.length - 1];
+      if (!entry) throw new Error('complete projection requires a completed stage entry');
+      emit('stage-completed', entry.stage, {
+        receiptKinds: sortedKinds(entry.evidenceReceipts),
+        receiptQualities: sortedQualities(entry.evidenceReceipts),
+      });
+      if (nextState.nextStage === null && isWorkflowComplete(nextState)) {
+        emit('workflow-completed', null, {
+          completedStages: nextState.completedStages.map((c) => c.stage),
+          skippedStages: nextState.skippedStages.map((s) => s.stage),
+        });
+      }
+      break;
+    }
+    case 'skip': {
+      const entry = nextState.skippedStages[nextState.skippedStages.length - 1];
+      if (!entry) throw new Error('skip projection requires a skipped stage entry');
+      emit('stage-skipped', entry.stage, { reason: entry.reason, policy: entry.policy, evidence: [...entry.evidence] });
+      break;
+    }
+    case 'block':
+      emit('workflow-blocked', null, {
+        blockerIds: [...nextState.blockers].map((b) => b.id).sort(),
+        blockerTypes: [...nextState.blockers].map((b) => b.type).sort(),
+      });
+      break;
+    case 'unblock':
+      break;
+    case 'interrupt':
+      emit('workflow-interrupted', null, {});
+      break;
+    case 'resume': {
+      const outcome = input.resolutionOutcome ?? (nextState.status === 'BLOCKED' ? 'blocked' : 'ready');
+      emit('workflow-resumed', null, { outcome, staleReason: input.staleReason ?? null });
+      if (outcome === 'blocked') {
+        emit('workflow-blocked', null, {
+          blockerIds: [...nextState.blockers].map((b) => b.id).sort(),
+          blockerTypes: [...nextState.blockers].map((b) => b.type).sort(),
+        });
+      }
+      break;
+    }
+    case 'finalize':
+      emit('workflow-completed', null, {
+        completedStages: nextState.completedStages.map((c) => c.stage),
+        skippedStages: nextState.skippedStages.map((s) => s.stage),
+      });
+      break;
+    default:
+      throw new Error(`Unknown workflow trace op: ${op}`);
+  }
+  return Object.freeze(events);
+}
+
+export const workflowTraceAPI = {
+  projectWorkflowEvents,
+  eventKey,
+  normalizeTrace,
+  assertNoAuthority,
+  validateWorkflowEvent,
+};