scripts-orchestrator 2.14.0 → 2.15.1

package/README.md

@@ -40,6 +40,7 @@ npm install --save-dev scripts-orchestrator
 - **NDJSON event stream**: Machine-readable per-command events for dashboards (v2.14+)
 - **Post-run hook**: Run a shell command after results are written via `post_run` config (v2.14+)
 - **Run-state file**: Library-owned in-progress indicator for live dashboard integration (v2.14+)
+- **Phase recommendations**: Memory-aware `--recommend` mode that proposes an optimal phase layout from a run's time/memory metrics (advisory, v2.15+)
 
 ## Configuration
 
@@ -450,7 +451,7 @@ Alongside `json_results`, the library writes one NDJSON line per event to
 Dashboard tools can `tail -f` this file or watch it with `fs.watch` to get real-time updates
 without parsing human-readable log lines.
 
-### Run-state file (A4)
+### Run-state file
 
 When `--logFolder` is specified, the library writes `{logFolder}/.scripts-orchestrator-run.json`
 at run start and removes it on run end:
@@ -467,7 +468,7 @@ at run start and removes it on run end:
 This file is the authoritative in-progress signal for live dashboards. Its absence means the run
 has finished (or never started).
 
-### Post-run hook (A7)
+### Post-run hook
 
 Add `post_run` to your config to run a shell command **after** `json_results` is written and the
 run-state file is cleared, but **before** `process.exit()`:
@@ -522,6 +523,59 @@ This is useful when you want to:
 - Debug issues without modifying the codebase
 - Override the cache in CI/CD pipelines
 
+## Phase Recommendations (advisory)
+
+When a run is executed with `metrics: ['time', 'memory']`, the results JSON records each command's
+`durationMs` and peak `memoryKb`. The `--recommend` mode reads that JSON and reports a **memory-aware
+phase recommendation**: it never runs anything and changes no run state.
+
+```bash
+# Analyse an existing results JSON and print a suggested phase layout to the console
+scripts-orchestrator --recommend ./logs/scripts-orchestrator-results.json
+
+# Write the report to a plain-text log file instead of the console (only a pointer line is printed)
+scripts-orchestrator --recommend ./logs/results.json --recommend-out ./logs/recommendation.log
+
+# Size the budget for a machine running N gates in parallel (each gets 1/N of RAM and cores)
+scripts-orchestrator --recommend ./logs/scripts-orchestrator-results.json --fanout 3
+
+# Override the memory budget explicitly (MB) or change the RAM safety fraction
+scripts-orchestrator --recommend ./logs/results.json --budget-mb 8192
+scripts-orchestrator --recommend ./logs/results.json --mem-safety 0.7
+```
+
+It reports three things:
+
+1. **Observed timeline** — each phase's wall-clock (the longest step in it) and the concurrent peak
+   memory (Σ of member peaks), flagging any phase whose concurrent peak exceeds the host budget.
+2. **Recommended layout** — a [First-Fit-Decreasing](https://en.wikipedia.org/wiki/Bin_packing_problem)
+   bin-packing by duration that groups steps into sequential phases so each phase's concurrent peak
+   memory stays under `budget = totalmem × memSafety ÷ fanout` and its step count stays under
+   `coreShare = (cores − 2) ÷ fanout`. Long steps seed phases; short steps fill the gaps beneath them,
+   so the estimated makespan stays near the theoretical floor (the single longest step) without
+   oversubscribing RAM.
+3. **Verdict** — a single yes/no line: whether re-grouping is worth it (it must trim ≥5% and ≥5s off
+   the makespan), or — when one step is ≥95% of the makespan — that the only remaining lever is to
+   split that step into smaller commands the orchestrator can schedule separately.
+
+The same logic is exported for programmatic use:
+
+```js
+import { recommendPhases, formatRecommendationReport } from 'scripts-orchestrator';
+
+const payload = JSON.parse(fs.readFileSync('./logs/results.json', 'utf8'));
+const rec = recommendPhases(payload, { fanout: 3 });
+console.log(formatRecommendationReport(rec));
+// rec.verdict.worthwhile, rec.verdict.reason, rec.recommended.bins, rec.observed, rec.budgetBytes, …
+```
+
+A natural place to wire it is the `post_run` config hook, so each run prints a recommendation for its
+own results JSON when it finishes.
+
+This is advisory only — the budget is conservative (per-process peaks summed as if they coincide) and
+the packing does not model inter-phase data dependencies, so validate any suggested layout against a
+real run before adopting it.
+
 ## Exit Codes
 
 - `0`: All commands executed successfully
@@ -535,6 +589,7 @@ See [versions](./docs/versions.md)
 - Better UX to indicate what is happening
 - Tests to avoid regression
 - Run any shell command rather than assume the command is specified in package.json (? tentative)
+- Promote the advisory `--recommend` phase recommender into an opt-in automatic scheduler that packs each phase under a per-host memory budget at run time
 
 
 ## Disclaimer

package/index.js

@@ -51,10 +51,103 @@ const argv = yargs(hideBin(process.argv))
     type: 'string',
     description: 'Write HTML report to this path; use "-" for stdout only',
   })
+  .option('render', {
+    type: 'string',
+    description: 'Render an existing results JSON file to HTML (no run). Use with --html-results.',
+  })
+  .option('recommend', {
+    type: 'string',
+    description:
+      'Analyse an existing results JSON and print a memory-aware phase recommendation (no run).',
+  })
+  .option('fanout', {
+    type: 'number',
+    description: 'Workspace fan-out (parallel gates sharing the host) used to size the --recommend budget. Default 1.',
+  })
+  .option('mem-safety', {
+    type: 'number',
+    description: 'Fraction of total RAM the --recommend budget may use (default 0.8).',
+  })
+  .option('budget-mb', {
+    type: 'number',
+    description: 'Override the --recommend memory budget with a fixed value in MB.',
+  })
+  .option('recommend-out', {
+    type: 'string',
+    description: 'Write the --recommend report to this file (plain text) instead of the console.',
+  })
   .help()
   .alias('h', 'help')
   .parse();
 
+// --render mode: turn an existing results JSON into HTML and exit (no orchestration run).
+// Keeps all HTML rendering in the library so consumers never reimplement it.
+if (argv.render != null) {
+  const { renderReportHtml } = await import('./lib/index.js');
+  const srcPath = path.resolve(process.cwd(), argv.render);
+  if (!fs.existsSync(srcPath)) {
+    log.error(`Error: --render source not found at ${srcPath}`);
+    process.exit(1);
+  }
+  let payload;
+  try {
+    payload = JSON.parse(fs.readFileSync(srcPath, 'utf8'));
+  } catch (err) {
+    log.error(`Error: failed to parse --render JSON: ${err.message}`);
+    process.exit(1);
+  }
+  const html = renderReportHtml(payload);
+  const out = argv.htmlResults ?? null;
+  if (out == null || out === '-') {
+    console.log(html);
+  } else {
+    const outPath = path.resolve(process.cwd(), out);
+    const tmpPath = outPath + '.tmp';
+    fs.mkdirSync(path.dirname(outPath), { recursive: true });
+    fs.writeFileSync(tmpPath, html, 'utf8');
+    fs.renameSync(tmpPath, outPath);
+    log.info(`📄 Rendered ${path.relative(process.cwd(), srcPath)} → ${path.relative(process.cwd(), outPath)}`);
+  }
+  process.exit(0);
+}
+
+// --recommend mode: analyse an existing results JSON and print a memory-aware phase
+// recommendation. Advisory only — no orchestration run. Output goes to the console, or to a
+// plain-text log file when --recommend-out is given.
+if (argv.recommend != null) {
+  const { recommendPhases, formatRecommendationReport } = await import('./lib/index.js');
+  const srcPath = path.resolve(process.cwd(), argv.recommend);
+  if (!fs.existsSync(srcPath)) {
+    log.error(`Error: --recommend source not found at ${srcPath}`);
+    process.exit(1);
+  }
+  let payload;
+  try {
+    payload = JSON.parse(fs.readFileSync(srcPath, 'utf8'));
+  } catch (err) {
+    log.error(`Error: failed to parse --recommend JSON: ${err.message}`);
+    process.exit(1);
+  }
+  const rec = recommendPhases(payload, {
+    fanout: argv.fanout,
+    memSafety: argv.memSafety,
+    budgetMb: argv.budgetMb,
+  });
+  const report = formatRecommendationReport(rec, { sourcePath: path.relative(process.cwd(), srcPath) });
+  if (argv.recommendOut != null && argv.recommendOut !== '-') {
+    // Strip ANSI colour codes so the log file stays plain text. The escape char is built at runtime
+    // so the regex carries no literal control character (keeps eslint's no-control-regex happy).
+    const ansi = new RegExp(String.fromCharCode(27) + '\\[[0-9;]*m', 'g');
+    const outPath = path.resolve(process.cwd(), argv.recommendOut);
+    fs.mkdirSync(path.dirname(outPath), { recursive: true });
+    fs.writeFileSync(outPath, report.replace(ansi, '') + '\n', 'utf8');
+    log.info(`📄 Phase recommendation written to ${path.relative(process.cwd(), outPath)}`);
+  } else {
+    console.log(report);
+  }
+  process.exit(0);
+}
+
 // Extract arguments
 const args = argv._;
 const configPath = args[0] || './scripts-orchestrator.config.js';
@@ -111,9 +204,16 @@ const htmlResultsPath =
     ? argv.htmlResults
     : (commandsConfig.html_results ?? commandsConfig.html_results_path ?? null);
 
-// A7: post-run hook — shell command run after json_results written
+// post-run hook — shell command run after json_results written
 const postRun = commandsConfig.post_run ?? null;
 
+// Periodic hook — shell command run on an interval WHILE the run is in flight (e.g. to roll up
+// results into an aggregate report). Library owns only the cadence; the command is project-specific.
+const periodicHook = commandsConfig.periodic_hook ?? null;
+const periodicIntervalMs = Number(commandsConfig.periodic_interval_ms) > 0
+  ? Number(commandsConfig.periodic_interval_ms)
+  : 45000;
+
 // Set the log folder for the main orchestrator logs if specified
 if (logFolder) {
   log.setLogFolder(logFolder);
@@ -131,18 +231,22 @@ const orchestrator = new Orchestrator(
   jsonResultsPath,
   htmlResultsPath,
 );
-// A7: wire post-run hook from config
+// wire post-run hook from config
 orchestrator.postRun = postRun;
+// Wire periodic hook (cadence owned by the library)
+orchestrator.periodicHook = periodicHook;
+orchestrator.periodicIntervalMs = periodicIntervalMs;
 
 // Enhanced signal handlers
 const handleSignal = async (signal) => {
   log.warn(`\nReceived ${signal} signal. Cleaning up...`);
+  orchestrator._stopPeriodicHook();
   try {
     await orchestrator.processManager.cleanup();
   } catch (error) {
     log.error(`Cleanup failed: ${error.message}`);
   }
-  // A4: clear run-state so dashboards know the run ended
+  // clear run-state so dashboards know the run ended
   orchestrator._clearRunState();
   process.exit(1);
 };

package/lib/index.js

@@ -3,6 +3,30 @@ import { ProcessManager } from './process-manager.js';
 import { HealthCheck } from './health-check.js';
 import { Logger } from './logger.js';
 import { GitCache } from './git-cache.js';
+import { renderReportHtml } from './report-html.js';
+import {
+  recommendPhases,
+  decideVerdict,
+  formatRecommendationReport,
+  computeBudget,
+  usableSteps,
+  observedTimeline,
+  packPhases,
+} from './recommend-phases.js';
 
-export { Orchestrator, ProcessManager, HealthCheck, Logger, GitCache };
-export default Orchestrator; 
+export {
+  Orchestrator,
+  ProcessManager,
+  HealthCheck,
+  Logger,
+  GitCache,
+  renderReportHtml,
+  recommendPhases,
+  decideVerdict,
+  formatRecommendationReport,
+  computeBudget,
+  usableSteps,
+  observedTimeline,
+  packPhases,
+};
+export default Orchestrator;