scripts-orchestrator 2.13.0 → 2.14.0

package/README.md

@@ -36,6 +36,10 @@ npm install --save-dev scripts-orchestrator
 - **Optional Phases**: Mark phases as optional and run them selectively
 - **Git-Based Caching**: Automatically skips execution when git state is unchanged
 - **Comprehensive Logging**: Detailed logging of command execution and results
+- **Incremental JSON results**: Live-updating `json_results` file as commands complete (v2.14+)
+- **NDJSON event stream**: Machine-readable per-command events for dashboards (v2.14+)
+- **Post-run hook**: Run a shell command after results are written via `post_run` config (v2.14+)
+- **Run-state file**: Library-owned in-progress indicator for live dashboard integration (v2.14+)
 
 ## Configuration
 
@@ -410,6 +414,84 @@ export default {
 
 All logs (command logs, main orchestrator logs, and git cache) will be stored in the specified folder.
 
+## Live Dashboard Integration (v2.14+)
+
+### Incremental JSON results
+
+By default `json_results` is written only at the end of a run. From v2.14 onward the file is
+updated atomically (write-to-temp + rename) after **each command starts or completes**, so
+watchers always see a consistent snapshot:
+
+```json
+{
+  "success": null,
+  "timestamp": "2026-06-04T07:13:21.000Z",
+  "commands": [
+    { "command": "lint-ci", "phase": "lint", "success": true, "durationMs": 4200 },
+    { "command": "playwright_ci", "phase": "tests", "success": null, "startedAt": "2026-06-04T07:13:25.000Z" }
+  ]
+}
+```
+
+`"success": null` at the top level is the in-progress sentinel. It is replaced with `true` or
+`false` when `writeJsonResults` writes the final result.
+
+### NDJSON event stream
+
+Alongside `json_results`, the library writes one NDJSON line per event to
+`<json_results_basename>-events.ndjson`:
+
+```jsonl
+{"type":"command_start","timestamp":"...","command":"lint-ci","phase":"lint","scope":"workspace"}
+{"type":"command_end","timestamp":"...","command":"lint-ci","phase":"lint","success":true,"durationMs":4200}
+{"type":"run_end","timestamp":"...","success":true,"durationMs":12800}
+```
+
+Dashboard tools can `tail -f` this file or watch it with `fs.watch` to get real-time updates
+without parsing human-readable log lines.
+
+### Run-state file (A4)
+
+When `--logFolder` is specified, the library writes `{logFolder}/.scripts-orchestrator-run.json`
+at run start and removes it on run end:
+
+```json
+{
+  "startedAt": "2026-06-04T07:13:17.000Z",
+  "pid": 12345,
+  "phase": "tests",
+  "activeCommand": "playwright_ci"
+}
+```
+
+This file is the authoritative in-progress signal for live dashboards. Its absence means the run
+has finished (or never started).
+
+### Post-run hook (A7)
+
+Add `post_run` to your config to run a shell command **after** `json_results` is written and the
+run-state file is cleared, but **before** `process.exit()`:
+
+```javascript
+export default {
+  json_results: './logs/scripts-orchestrator-results.json',
+  post_run: 'node scripts/generate-report.js',  // called after every run
+  phases: [ /* ... */ ]
+};
+```
+
+The hook receives two environment variables:
+- `SCRIPTS_ORCHESTRATOR_SUCCESS=1` (or `0`) — whether the run succeeded
+- `SCRIPTS_ORCHESTRATOR_EXIT_CODE=0` (or `1`) — same, as a numeric exit code
+
+The hook runs synchronously and its exit code is logged but does not change the orchestrator's
+own exit code.
+
+**Typical use case:** trigger a monorepo rollup report after each workspace finishes:
+```javascript
+post_run: 'node ../../scripts/merge-orchestrator-report.js'
+```
+
 ## Git-Based Caching
 
 The orchestrator automatically tracks the git commit hash and repository state to optimize execution:

package/index.js

@@ -111,6 +111,9 @@ const htmlResultsPath =
     ? argv.htmlResults
     : (commandsConfig.html_results ?? commandsConfig.html_results_path ?? null);
 
+// A7: post-run hook — shell command run after json_results written
+const postRun = commandsConfig.post_run ?? null;
+
 // Set the log folder for the main orchestrator logs if specified
 if (logFolder) {
   log.setLogFolder(logFolder);
@@ -128,6 +131,8 @@ const orchestrator = new Orchestrator(
   jsonResultsPath,
   htmlResultsPath,
 );
+// A7: wire post-run hook from config
+orchestrator.postRun = postRun;
 
 // Enhanced signal handlers
 const handleSignal = async (signal) => {
@@ -137,6 +142,8 @@ const handleSignal = async (signal) => {
   } catch (error) {
     log.error(`Cleanup failed: ${error.message}`);
   }
+  // A4: clear run-state so dashboards know the run ended
+  orchestrator._clearRunState();
   process.exit(1);
 };
 

package/lib/orchestrator.js

@@ -1,4 +1,6 @@
 import fs from 'fs';
+import path from 'path';
+import { spawnSync } from 'child_process';
 import { processManager } from './process-manager.js';
 import { healthCheck } from './health-check.js';
 import { log } from './logger.js';
@@ -35,6 +37,16 @@ export class Orchestrator {
     this.commandTimings = new Map(); // command -> { durationMs, memoryKb? }
     this.phaseResults = []; // { name, success, durationMs } per phase run
     this.gitCache = new GitCache(logFolder);
+    // A1: track per-command start times for incremental JSON
+    this.commandStartTimes = new Map(); // command -> ISO start string
+    // A2: events file path derived from jsonResultsPath
+    this.eventsPath = this._deriveEventsPath(jsonResultsPath);
+    // A4: library-owned run-state file
+    this.runStatePath = logFolder
+      ? path.join(path.resolve(logFolder), '.scripts-orchestrator-run.json')
+      : null;
+    // A7: post-run hook command (shell string)
+    this.postRun = null; // set from config in index.js
 
     // Set the log folder in process manager
     if (logFolder) {
@@ -58,6 +70,118 @@ export class Orchestrator {
     return [];
   }
 
+  _deriveEventsPath(jsonResultsPath) {
+    if (!jsonResultsPath || jsonResultsPath === '-') return null;
+    return jsonResultsPath.replace(/\.json$/, '') + '-events.ndjson';
+  }
+
+  // A4: write current run state atomically
+  _writeRunState(extra = {}) {
+    if (!this.runStatePath) return;
+    const state = {
+      startedAt: this.runStartedAt ? new Date(this.runStartedAt).toISOString() : new Date().toISOString(),
+      pid: process.pid,
+      ...extra,
+    };
+    const tmp = this.runStatePath + '.tmp';
+    try {
+      fs.mkdirSync(path.dirname(this.runStatePath), { recursive: true });
+      fs.writeFileSync(tmp, JSON.stringify(state, null, 2) + '\n', 'utf8');
+      fs.renameSync(tmp, this.runStatePath);
+    } catch { /* non-fatal */ }
+  }
+
+  // A4: remove run-state file on run end
+  _clearRunState() {
+    if (!this.runStatePath) return;
+    try { fs.unlinkSync(this.runStatePath); } catch { /* ignore */ }
+  }
+
+  // A2: append a structured NDJSON event
+  _appendEvent(type, data = {}) {
+    if (!this.eventsPath) return;
+    const line = JSON.stringify({ type, timestamp: new Date().toISOString(), ...data });
+    try {
+      fs.appendFileSync(this.eventsPath, line + '\n', 'utf8');
+    } catch {
+      // non-fatal: don't let event logging break the run
+    }
+  }
+
+  // A1: atomically write current run state (completed + in-flight commands) to json_results
+  _writePartialResults() {
+    if (this.jsonResultsPath == null || this.jsonResultsPath === '-') return;
+    const outPath = this.jsonResultsPath || './scripts-orchestrator-results.json';
+
+    const commands = [];
+
+    const buildEntry = (command, phaseName) => {
+      const timing = this.commandTimings.get(command);
+      const startedAt = this.commandStartTimes.get(command);
+      const skipped = this.skippedCommands.includes(command);
+      const done = timing != null || skipped;
+
+      if (!done && startedAt) {
+        return { command, ...(phaseName ? { phase: phaseName } : {}), success: null, startedAt };
+      }
+      if (!done) return null; // not yet started — omit
+
+      const skipReason = skipped ? (this.skipReasons.get(command) ?? null) : null;
+      const success =
+        !this.failedCommands.includes(command) &&
+        (skipReason === null ||
+          skipReason === 'disabled' ||
+          skipReason === 'optional_phase_not_requested' ||
+          skipReason === 'before_start_phase');
+      return {
+        command,
+        ...(phaseName ? { phase: phaseName } : {}),
+        success,
+        ...(timing?.durationMs != null && this.metrics.includes('time') ? { durationMs: timing.durationMs } : {}),
+        ...(this.metrics.includes('memory') ? { memoryKb: timing?.memoryKb ?? null } : {}),
+        ...(skipReason ? { skipReason } : {}),
+      };
+    };
+
+    if (Array.isArray(this.config)) {
+      for (const { command } of this.config) {
+        const entry = buildEntry(command, null);
+        if (entry) commands.push(entry);
+      }
+    } else if (this.config.phases) {
+      for (const phase of this.config.phases) {
+        for (const { command } of (phase.parallel || [])) {
+          const entry = buildEntry(command, phase.name);
+          if (entry) commands.push(entry);
+        }
+      }
+    }
+
+    const payload = {
+      success: null, // in-progress sentinel; replaced by writeJsonResults on completion
+      timestamp: new Date().toISOString(),
+      commands,
+      ...(this.config.phases && this.phaseResults.length > 0 ? { phases: this.phaseResults } : {}),
+    };
+
+    const tmpPath = outPath + '.tmp';
+    try {
+      fs.mkdirSync(path.dirname(path.resolve(outPath)), { recursive: true });
+      fs.writeFileSync(tmpPath, JSON.stringify(payload, null, 2), 'utf8');
+      fs.renameSync(tmpPath, outPath);
+    } catch (err) {
+      this.logger.verbose(`Partial results write failed: ${err.message}`);
+    }
+
+    // A4: keep run-state file in sync with current active commands and phase
+    const inFlight = commands.filter(c => c.success === null).map(c => c.command);
+    const currentPhase = commands.length > 0 ? (commands[commands.length - 1].phase ?? null) : null;
+    this._writeRunState({
+      activeCommand: inFlight.length === 1 ? inFlight[0] : (inFlight.length > 1 ? inFlight : null),
+      phase: currentPhase,
+    });
+  }
+
   formatDuration(ms) {
     if (ms < 1000) return `${ms}ms`;
     const seconds = Math.floor(ms / 1000);
@@ -69,7 +193,7 @@ export class Orchestrator {
     return `${seconds}s`;
   }
 
-  async executeCommand(commandConfig, visited = new Set()) {
+  async executeCommand(commandConfig, visited = new Set(), phaseName = null) {
     const {
       command,
       dependencies = [],
@@ -184,6 +308,11 @@ export class Orchestrator {
       }
     }
 
+    // A1/A2: record start and emit event
+    this.commandStartTimes.set(command, new Date().toISOString());
+    this._appendEvent('command_start', { command, phase: phaseName, scope: 'workspace' });
+    this._writePartialResults();
+
     // Execute the main command with retries
     let result = false;
     let commandOutput = '';
@@ -258,6 +387,9 @@ export class Orchestrator {
 
     const totalDurationMs = Date.now() - startTime;
     setTiming(totalDurationMs, lastRunResult?.memoryKb ?? null);
+    // A1/A2: emit completion event and write incremental results
+    this._appendEvent('command_end', { command, phase: phaseName, success: result, durationMs: totalDurationMs });
+    this._writePartialResults();
     visited.delete(command);
     return result;
   }
@@ -500,6 +632,9 @@ export class Orchestrator {
 
   async run() {
     this.startTime = Date.now();
+    this.runStartedAt = this.startTime;
+    // A4: write initial run-state at start
+    this._writeRunState({ phase: null, activeCommand: null });
     try {
       // Check if we should skip execution based on git state (unless forced)
       if (!this.force) {
@@ -598,7 +733,7 @@ export class Orchestrator {
             // Run commands sequentially
             results = [];
             for (const commandConfig of phase.parallel) {
-              const result = await this.executeCommand(commandConfig);
+              const result = await this.executeCommand(commandConfig, new Set(), phase.name);
               results.push(result);
               if (!result) {
                 // In sequential mode, stop phase execution on first failure
@@ -608,7 +743,7 @@ export class Orchestrator {
           } else {
             // Run commands in parallel
             const tasks = phase.parallel.map((commandConfig) =>
-              this.executeCommand(commandConfig),
+              this.executeCommand(commandConfig, new Set(), phase.name),
             );
             results = await Promise.all(tasks);
           }
@@ -698,11 +833,21 @@ export class Orchestrator {
         );
       }
 
+      // A2: emit run_end event; A1: final JSON written by writeJsonResults (replaces partial)
+      const runDurationMs = this.startTime ? Date.now() - this.startTime : undefined;
+      this._appendEvent('run_end', { success: !hasFailures, ...(runDurationMs != null ? { durationMs: runDurationMs } : {}) });
+
       // Write JSON results if requested
       if (this.jsonResultsPath != null) {
         this.writeJsonResults(hasFailures);
       }
 
+      // A4: clear run-state file — run is done
+      this._clearRunState();
+
+      // A7: run post_run hook after results are written
+      this._runPostRunHook(hasFailures);
+
       // Update git cache on successful execution
       if (!hasFailures) {
         await this.gitCache.updateCache();
@@ -719,6 +864,9 @@ export class Orchestrator {
     } catch (error) {
       this.logger.error(`Orchestrator failed: ${error.message}`);
 
+      // A4: clear run-state on error too
+      this._clearRunState();
+
       // Cleanup on error
       try {
         await this.processManager.cleanup();
@@ -729,4 +877,23 @@ export class Orchestrator {
       process.exit(1);
     }
   }
+
+  // A7: run user-configured post_run shell command
+  _runPostRunHook(hasFailures) {
+    if (!this.postRun) return;
+    this.logger.info(`[post_run] ${this.postRun}`);
+    const result = spawnSync(this.postRun, {
+      shell: true,
+      stdio: 'inherit',
+      cwd: process.cwd(),
+      env: {
+        ...process.env,
+        SCRIPTS_ORCHESTRATOR_SUCCESS: hasFailures ? '0' : '1',
+        SCRIPTS_ORCHESTRATOR_EXIT_CODE: hasFailures ? '1' : '0',
+      },
+    });
+    if (result.status !== 0) {
+      this.logger.warn(`[post_run] hook exited with code ${result.status}`);
+    }
+  }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "scripts-orchestrator",
-  "version": "2.13.0",
+  "version": "2.14.0",
   "description": "A powerful script orchestrator for running parallel commands with dependency management, background processes, and health checks",
   "main": "lib/index.js",
   "type": "module",