yaml-flow 5.4.0 → 6.0.0

package/examples/example-board/demo-task-executor.js

@@ -32,7 +32,10 @@
  *     "boardId":          "<board id>",        // e.g. "default"
  *     "boardRuntimeDir":  "<relative>",        // e.g. "runtime"
  *     "runtimeStatusDir": "<relative>",        // e.g. "runtime-out"
- *     "cardsDir":         "<relative>"         // e.g. "surface/tmp-cards"
+ *     "cardsDir":         "<relative>",        // e.g. "surface/tmp-cards"
+ *     "serverUrl":        "<base url>",        // optional; e.g. "http://127.0.0.1:7799"
+ *     "boardLiveCardsCliJs":"<abs path>",      // optional; path to board-live-cards-cli.js
+ *     "stepMachineCliPath":"<abs path>"        // optional; path to step-machine-cli.js
  *   }
  *
  * Supported source kinds (based on custom fields in --in):
@@ -58,6 +61,7 @@ import os from 'node:os';
 import crypto from 'node:crypto';
 import { execFileSync } from 'node:child_process';
 import { fileURLToPath } from 'node:url';
+import { parseRef, blobStorageForRef, reportComplete, reportFailed } from 'yaml-flow/storage-refs';
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
@@ -103,7 +107,7 @@ function readCache(key, ttlMs = CACHE_TTL_MS) {
 
 // Shared single-URL fetch helper used by both url and url-list.
 // cacheTimeoutSec: override TTL in seconds (null → use CACHE_TTL_MS default).
-function doFetchApi(url, method, headers, cacheTimeoutSec, errFile) {
+function doFetchApi(url, method, headers, cacheTimeoutSec) {
   const ttlMs = cacheTimeoutSec != null ? cacheTimeoutSec * 1000 : CACHE_TTL_MS;
   const k = cacheKey(`url:${method}:${url}`);
   const cached = readCache(k, ttlMs);
@@ -265,14 +269,14 @@ function fail(msg, errFile) {
 }
 
 function runSourceFetchSubcommand(argv) {
-  const inIdx = argv.indexOf('--in');
-  const outIdx = argv.indexOf('--out');
-  const errIdx = argv.indexOf('--err');
+  const inIdx = argv.indexOf('--in-ref');
+  const outIdx = argv.indexOf('--out-ref');
+  const errIdx = argv.indexOf('--err-ref');
   const extraIdx = argv.indexOf('--extra');
-  const inFile = inIdx !== -1 ? argv[inIdx + 1] : undefined;
-  const outFile = outIdx !== -1 ? argv[outIdx + 1] : undefined;
-  const errFile = errIdx !== -1 ? argv[errIdx + 1] : undefined;
-  const extraB64 = extraIdx !== -1 ? argv[extraIdx + 1] : undefined;
+  const inRefStr  = inIdx  !== -1 ? argv[inIdx + 1]  : undefined;
+  const outRefStr = outIdx !== -1 ? argv[outIdx + 1] : undefined;
+  const errRefStr = errIdx !== -1 ? argv[errIdx + 1] : undefined;
+  const extraB64  = extraIdx !== -1 ? argv[extraIdx + 1] : undefined;
 
   let extra = {};
   if (extraB64) {
@@ -280,19 +284,50 @@ function runSourceFetchSubcommand(argv) {
     catch { console.warn('[demo-task-executor] bad --extra base64, ignoring'); }
   }
 
-  if (!inFile || !outFile) {
-    fail('Usage: run-source-fetch --in <source.json> --out <result.json> [--err <error.txt>]', errFile);
+  if (!inRefStr || !outRefStr) {
+    fail('Usage: run-source-fetch --in-ref <::kind::value> --out-ref <::kind::value> [--err-ref <::kind::value>]');
   }
 
-  if (!fs.existsSync(inFile)) {
-    fail(`Input file not found: ${inFile}`, errFile);
+  let inRef, outRef, errRef;
+  try {
+    inRef  = parseRef(inRefStr);
+    outRef = parseRef(outRefStr);
+    if (errRefStr) errRef = parseRef(errRefStr);
+  } catch (e) {
+    fail(`invalid ref argument: ${e.message}`);
+  }
+
+  const inStorage  = blobStorageForRef(inRef);
+  const outStorage = blobStorageForRef(outRef);
+  const errStorage = errRef ? blobStorageForRef(errRef) : undefined;
+
+  // Local error reporter — writes to errStorage and calls back to board if callback present.
+  const failRef = (msg, callback) => {
+    if (errStorage && errRef) { try { errStorage.write(errRef.value, msg); } catch {} }
+    console.error(`[demo-task-executor] ${msg}`);
+    if (callback) { try { reportFailed(callback, msg); } catch {} }
+    process.exit(1);
+  };
+
+  const rawIn = inStorage.read(inRef.value);
+  if (rawIn === null) {
+    failRef(`Input not found: ${inRefStr}`);
   }
 
+  // Payload may be { source_def, callback } (new protocol) or raw source def (legacy).
+  let envelope;
+  try {
+    envelope = JSON.parse(rawIn);
+  } catch (err) {
+    failRef(`Cannot parse input: ${String(err && err.message || err)}`);
+  }
+
+  const callback = envelope.source_def ? envelope.callback : undefined;
   let sourceDef;
   try {
-    sourceDef = readJson(inFile);
+    sourceDef = envelope.source_def ?? envelope;
   } catch (err) {
-    fail(`Cannot parse source file: ${String(err && err.message || err)}`, errFile);
+    failRef(`Cannot resolve source_def: ${String(err && err.message || err)}`, callback);
   }
 
   let resultValue;
@@ -321,14 +356,14 @@ function runSourceFetchSubcommand(argv) {
       }
     }
     if (sourceDef.tickersFrom && !fetchArgs.tickers) {
-      fail('url: tickersFrom resolved to empty list — skipping fetch', errFile);
+      failRef('url: tickersFrom resolved to empty list — skipping fetch', callback);
     }
     const urlContext = { ...(sourceDef._projections || {}), ...fetchArgs };
     const url = interpolatePrompt(cfg.url, urlContext);
     try {
-      resultValue = doFetchApi(url, method, headers, cacheTimeoutSec, errFile);
+      resultValue = doFetchApi(url, method, headers, cacheTimeoutSec);
     } catch (err) {
-      fail(`url failed: ${err.message}`, errFile);
+      failRef(`url failed: ${err.message}`, callback);
     }
 
   } else if (sourceDef['url-list']) {
@@ -346,15 +381,15 @@ function runSourceFetchSubcommand(argv) {
       ? sourceDef._projections.url_list : null;
 
     if (!urlList || urlList.length === 0) {
-      fail('url-list: _projections.url_list must be a non-empty string array', errFile);
+      failRef('url-list: _projections.url_list must be a non-empty string array', callback);
     }
 
     const results = [];
     for (const u of urlList) {
       try {
-        results.push(doFetchApi(u, method, headers, cacheTimeoutSec, errFile));
+        results.push(doFetchApi(u, method, headers, cacheTimeoutSec));
       } catch (err) {
-        fail(`url-list fetch failed for ${u}: ${err.message}`, errFile);
+        failRef(`url-list fetch failed for ${u}: ${err.message}`, callback);
       }
     }
     resultValue = results;
@@ -362,7 +397,7 @@ function runSourceFetchSubcommand(argv) {
   } else if (sourceDef.copilot || sourceDef.prompt_template) {
     const prompt = resolveCopilotPrompt(sourceDef);
     if (!prompt) {
-      fail('Source definition missing copilot.prompt_template (or prompt_template)', errFile);
+      failRef('Source definition missing copilot.prompt_template (or prompt_template)', callback);
     }
 
     // Use boardSetupRoot (from --extra) as copilot working directory
@@ -382,11 +417,11 @@ function runSourceFetchSubcommand(argv) {
         'copilot-sessions',
         String(sourceDef.bindTo || 'default').replace(/[^a-zA-Z0-9_-]/g, '_'),
       );
-      const wrapperOutFile = outFile + '.wrapper-out.json';
+      const wrapperOutFile = outRef.value + '.wrapper-out.json';
       try {
         resultValue = runCopilotViaWrapper(prompt, sourceDef, wrapperOutFile, sessionDir, copilotCwd);
       } catch (err) {
-        fail(`copilot invocation failed: ${String(err && err.message || err)}`, errFile);
+        failRef(`copilot invocation failed: ${String(err && err.message || err)}`, callback);
       } finally {
         try { fs.unlinkSync(wrapperOutFile); } catch {}
       }
@@ -402,7 +437,7 @@ function runSourceFetchSubcommand(argv) {
           ...(copilotCwd ? { cwd: copilotCwd } : {}),
         });
       } catch (err) {
-        fail(`copilot invocation failed: ${String(err && err.message || err)}`, errFile);
+        failRef(`copilot invocation failed: ${String(err && err.message || err)}`, callback);
       }
       // Basic JSON extraction: find first { or [ in output
       const firstBrace = rawOutput.indexOf('{');
@@ -424,17 +459,17 @@ function runSourceFetchSubcommand(argv) {
   } else if (sourceDef.workiq) {
     const cfg = typeof sourceDef.workiq === 'object' ? sourceDef.workiq : {};
     if (!cfg.query_template || typeof cfg.query_template !== 'string') {
-      fail('Source definition missing workiq.query_template', errFile);
+      failRef('Source definition missing workiq.query_template', callback);
     }
     const interpolationContext = { ...sourceDef._projections, ...(cfg.args ?? {}) };
     const query = interpolatePrompt(cfg.query_template, interpolationContext);
 
     const wrapperPath = path.join(__dirname, 'scripts', 'workiq_wrapper.mjs');
     if (!fs.existsSync(wrapperPath)) {
-      fail('workiq source kind requires workiq_wrapper.js in scripts/', errFile);
+      failRef('workiq source kind requires workiq_wrapper.js in scripts/', callback);
     }
     try {
-      execFileSync(process.execPath, [wrapperPath, outFile], {
+      execFileSync(process.execPath, [wrapperPath, outRef.value], {
         encoding: 'utf-8',
         stdio: ['inherit', 'pipe', 'pipe'],
         maxBuffer: 10 * 1024 * 1024,
@@ -444,25 +479,34 @@ function runSourceFetchSubcommand(argv) {
           ...(extra.serverUrl ? { WORKIQ_SERVER_URL: extra.serverUrl } : {}),
         },
       });
-      return; // wrapper wrote directly to outFile
+      return; // wrapper wrote directly to out file
     } catch (err) {
-      fail(`workiq invocation failed: ${String(err && err.message || err)}`, errFile);
+      failRef(`workiq invocation failed: ${String(err && err.message || err)}`, callback);
     }
   } else if (sourceDef.mock) {
     // MOCK_DB lookup — data hardcoded at the top of this file
     resultValue = MOCK_DB[sourceDef.mock];
     if (resultValue === undefined) {
-      fail(`Key "${sourceDef.mock}" not found in MOCK_DB`, errFile);
+      failRef(`Key "${sourceDef.mock}" not found in MOCK_DB`, callback);
     }
   } else {
-    fail('Source definition has no recognised kind (url, url-list, copilot, workiq, mock)', errFile);
+    failRef('Source definition has no recognised kind (url, url-list, copilot, workiq, mock)', callback);
   }
 
-  // Write result to --out as JSON payload, same contract as current mock mode.
+  // Write result to --out via storage abstraction, then report back to board.
   try {
-    fs.writeFileSync(outFile, JSON.stringify(resultValue, null, 2));
+    outStorage.write(outRef.value, JSON.stringify(resultValue, null, 2));
   } catch (err) {
-    fail(`Cannot write output file: ${String(err && err.message || err)}`, errFile);
+    failRef(`Cannot write output: ${String(err && err.message || err)}`, callback);
+  }
+
+  if (callback) {
+    try {
+      reportComplete(callback, outRef);
+    } catch (err) {
+      console.error(`[demo-task-executor] reportComplete failed: ${String(err && err.message || err)}`);
+      process.exit(1);
+    }
   }
 
 }
@@ -650,6 +694,8 @@ const CAPABILITIES = {
       runtimeStatusDir: { type: 'string', description: 'Relative path to runtime-out dir.' },
       cardsDir:         { type: 'string', description: 'Relative path to cards dir.' },
       serverUrl:        { type: 'string', description: 'Base URL of the hosting server (e.g. http://127.0.0.1:7799). Used by source kinds that call server-side proxy endpoints.' },
+      boardLiveCardsCliJs: { type: 'string', description: 'Absolute path to board-live-cards-cli.js when configured by the runtime.' },
+      stepMachineCliPath: { type: 'string', description: 'Absolute path to step-machine-cli.js when configured by the runtime.' },
     },
   },
 };

package/examples/index.html

@@ -454,19 +454,6 @@
             <div class="command"><code>Open browser/step-machine-browser/index.html</code><button class="copy-btn" data-copy="browser/step-machine-browser/index.html">Copy path</button></div>
           </article>
 
-          <article class="example-card" data-surface="browser" data-model="livecards" data-search="portfolio tracker browser boards portfolio-tracker node javascript board-live-cards cli">
-            <div class="chip-row"><span class="chip">Browser support</span><span class="chip">LiveCards</span><span class="chip advanced">Advanced</span></div>
-            <h3>Portfolio Tracker Board Driver</h3>
-            <p>Node-driven portfolio tracker demo that exercises end-to-end board-live-cards orchestration for a browser-facing board workflow.</p>
-            <div class="command"><code>node examples/browser/boards/portfolio-tracker/portfolio-tracker.js</code><button class="copy-btn" data-copy="node examples/browser/boards/portfolio-tracker/portfolio-tracker.js">Copy</button></div>
-          </article>
-
-          <article class="example-card" data-surface="browser" data-model="livecards" data-search="portfolio tracker task-executor external source cli delegation custom">
-            <div class="chip-row"><span class="chip">Browser support</span><span class="chip">LiveCards</span><span class="chip">Task-Executor</span></div>
-            <h3>Portfolio Tracker with Task-Executor</h3>
-            <p>Portfolio tracker with external source execution via task-executor. Demonstrates delegating source.cli execution to a custom executor script.</p>
-            <div class="command"><code>node examples/browser/boards/portfolio-tracker/portfolio-tracker.js --task-executor "node C:\path\to\portfolio-tracker-task-executor.cjs"</code><button class="copy-btn" data-copy="node examples/browser/boards/portfolio-tracker/portfolio-tracker.js --task-executor &quot;node C:\path\to\portfolio-tracker-task-executor.cjs&quot;">Copy</button></div>
-          </article>
         </div>
       </div>
     </section>
@@ -708,7 +695,6 @@
           <tbody>
             <tr data-surface="browser" data-model="livecards"><td><a href="example-board/demo-shell.html"><code>examples/example-board/demo-shell.html</code></a></td><td>Browser</td><td>LiveCards</td><td><code>Open example-board/demo-shell.html</code></td><td>Maintained example-board browser demo for board/canvas rendering and compute-backed cards</td></tr>
             <tr data-surface="browser" data-model="step-machine"><td><a href="browser/step-machine-browser/index.html"><code>examples/browser/step-machine-browser/index.html</code></a></td><td>Browser</td><td>Step Machine</td><td><code>Open browser/step-machine-browser/index.html</code></td><td>Simple browser-side step-machine demo</td></tr>
-            <tr data-surface="browser" data-model="livecards"><td><code>examples/browser/boards/portfolio-tracker/portfolio-tracker.js</code></td><td>Browser support</td><td>LiveCards</td><td><code>node examples/browser/boards/portfolio-tracker/portfolio-tracker.js</code></td><td>End-to-end board-live-cards portfolio driver</td></tr>
             <tr data-surface="cli" data-model="step-machine"><td><code>examples/cli/step-machine-cli/portfolio-tracker/run-portfolio-tracker.bat</code></td><td>CLI</td><td>Step Machine</td><td><code>run-portfolio-tracker.bat</code></td><td>Portfolio tracker YAML flow orchestration</td></tr>
             <tr data-surface="cli" data-model="step-machine"><td><code>examples/cli/step-machine-cli/portfolio-tracker/portfolio-tracker.flow.yaml</code></td><td>CLI</td><td>Step Machine</td><td><code>node step-machine-cli.js examples/cli/step-machine-cli/portfolio-tracker/portfolio-tracker.flow.yaml --initial-data '{...baseline...}'</code></td><td>Direct baseline portfolio tracker invocation</td></tr>
             <tr data-surface="cli" data-model="step-machine"><td><code>examples/cli/step-machine-cli/portfolio-tracker/portfolio-tracker.flow.yaml</code></td><td>CLI</td><td>Step Machine</td><td><code>node step-machine-cli.js examples/cli/step-machine-cli/portfolio-tracker/portfolio-tracker.flow.yaml --initial-data '{...price-shock...}'</code></td><td>Direct price-shock portfolio tracker invocation</td></tr>

package/examples/step-machine-cli/portfolio-tracker/handlers/_board-cli.js

@@ -16,7 +16,6 @@ export function runBoardCli(args, options = {}) {
     stdio: capture ? 'pipe' : 'pipe',
     env: {
       ...process.env,
-      BOARD_LIVE_CARDS_NO_SPAWN: process.env.BOARD_LIVE_CARDS_NO_SPAWN ?? '1',
       BOARD_DIR: process.env.BOARD_DIR ?? '',
     },
   });

package/examples/step-machine-cli/portfolio-tracker/run-portfolio-tracker.bat

@@ -2,7 +2,6 @@
 setlocal
 cd /d "%~dp0"
 
-set "BOARD_LIVE_CARDS_NO_SPAWN=1"
 set "EXAMPLE_TEMP_ROOT=%TEMP%\yaml-flow-step-machine-portfolio-tracker"
 set "STORE_DIR=%EXAMPLE_TEMP_ROOT%\store"
 set "RUNTIME_ROOT=%EXAMPLE_TEMP_ROOT%\runtime"
@@ -25,5 +24,5 @@ node ..\..\..\step-machine-cli.js --store file --store-dir "%STORE_DIR%" --statu
 exit /b %ERRORLEVEL%
 
 :run
-	node -e "const fs=require('fs');const cp=require('child_process');const raw=JSON.parse(fs.readFileSync('portfolio-tracker.input.json','utf8'));raw.runtime_root=(process.env.RUNTIME_ROOT||'').replace(/\\\\/g,'/');const input=JSON.stringify(raw);const r=cp.spawnSync(process.execPath,['..\\..\\..\\step-machine-cli.js','portfolio-tracker.flow.yaml','--store','file','--store-dir',process.env.STORE_DIR,'--initial-data',input],{stdio:'inherit',windowsHide:true,env:{...process.env,BOARD_LIVE_CARDS_NO_SPAWN:'1'}});process.exit(r.status??1);"
+	node -e "const fs=require('fs');const cp=require('child_process');const raw=JSON.parse(fs.readFileSync('portfolio-tracker.input.json','utf8'));raw.runtime_root=(process.env.RUNTIME_ROOT||'').replace(/\\\\/g,'/');const input=JSON.stringify(raw);const r=cp.spawnSync(process.execPath,['..\\..\\..\\step-machine-cli.js','portfolio-tracker.flow.yaml','--store','file','--store-dir',process.env.STORE_DIR,'--initial-data',input],{stdio:'inherit',windowsHide:true});process.exit(r.status??1);"
 exit /b %ERRORLEVEL%

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "yaml-flow",
-  "version": "5.4.0",
+  "version": "6.0.0",
   "description": "Unified workflow engine: step-machine (sequential) + event-graph (stateless DAG) with pluggable storage",
   "author": "",
   "license": "MIT",
@@ -10,6 +10,7 @@
   "types": "./dist/index.d.ts",
   "bin": {
     "board-live-cards-cli": "./board-live-cards-cli.js",
+    "card-store": "./card-store.js",
     "step-machine-cli": "./step-machine-cli.js"
   },
   "exports": {
@@ -83,6 +84,28 @@
       "import": "./dist/runtime-artifacts/index.js",
       "require": "./dist/runtime-artifacts/index.cjs"
     },
+    "./storage-refs": {
+      "types": "./dist/storage-refs.d.ts",
+      "import": "./dist/storage-refs.js",
+      "require": "./dist/storage-refs.cjs"
+    },
+    "./execution-refs": {
+      "types": "./dist/execution-refs.d.ts",
+      "import": "./dist/execution-refs.js",
+      "require": "./dist/execution-refs.cjs"
+    },
+    "./board-live-cards-browser": {
+      "types": "./dist/cli/browser-api/board-live-cards-browser-adapter.d.ts",
+      "import": "./dist/cli/browser-api/board-live-cards-browser-adapter.js"
+    },
+    "./board-live-cards-node": {
+      "types": "./dist/cli/node/fs-board-adapter.d.ts",
+      "import": "./dist/cli/node/fs-board-adapter.js"
+    },
+    "./card-store-browser": {
+      "types": "./dist/cli/browser-api/card-store-browser-api.d.ts",
+      "import": "./dist/cli/browser-api/card-store-browser-api.js"
+    },
     "./board-livecards-server-runtime": "./board-livecards-server-runtime.js",
     "./package.json": "./package.json"
   },
@@ -91,6 +114,7 @@
   },
   "files": [
     "board-live-cards-cli.js",
+    "storage-refs.js",
     "step-machine-cli.js",
     "board-livecards-server-runtime.js",
     "dist",
@@ -100,17 +124,29 @@
   ],
   "scripts": {
     "build": "tsup",
-    "build:browser": "tsup --config tsup.browser.config.ts",
+    "build:browser": "tsup --config tsup.browser.config.ts && node scripts/generate-browser-integrity.mjs",
+    "build:quickjs": "tsup --config tsup.quickjs.config.ts",
+    "check:bundle-budget": "node scripts/check-bundle-budgets.mjs",
+    "check:browser-bundle": "node scripts/check-browser-bundle-safety.mjs",
+    "release:gate": "npm run -s build && npm run -s build:browser && npm run -s build:quickjs && npm run -s test:run && npm run -s check:bundle-budget && npm run -s check:browser-bundle",
+    "standalone": "npm run -s test:cli-parity && npm run -s build && npm run -s build:quickjs && node scripts/build-standalone.mjs",
+    "pycli:install": "python -m pip install -r pycli/requirements.txt",
+    "pycli:install:venv": "c:/Users/sreenaga/ADO/ai-tool-evolver/.venv/Scripts/python.exe -m pip install -r pycli/requirements.txt",
+    "pycli:install:py312": "py -3.12 -m pip install -r pycli/requirements.txt",
+    "pycli:install:venv312": "./.venv312/Scripts/python.exe -m pip install -r pycli/requirements.txt",
     "build:example": "node demo-src/example-board/build.js",
     "dev": "tsup --watch",
     "test": "vitest",
     "test:run": "vitest run",
+    "test:cli-parity": "npm run -s build:quickjs && python scripts/cli_parity_check.py",
+    "test:cli-parity:full": "npm run -s build:quickjs && python scripts/cli_parity_check.py --include-portfolio",
+    "test:e2e": "npm run -s build:quickjs && .\\.venv312\\Scripts\\python.exe examples/browser/boards/portfolio-tracker/portfolio-tracker.py --run-pycli",
     "test:example-board": "vitest run tests/examples/example-board.test.ts",
     "check:example-board": "npm run test:example-board && npm run validate:cards -- \"examples/example-board/cards/*.json\"",
     "validate:cards": "tsx scripts/validate-live-cards.ts",
     "lint": "eslint src/",
     "typecheck": "tsc --noEmit",
-    "prepublishOnly": "npm run build && npm run build:browser"
+    "prepublishOnly": "npm run build && npm run build:browser && npm run build:quickjs && npm run check:bundle-budget && npm run check:browser-bundle"
   },
   "devDependencies": {
     "@types/node": "^20.10.0",