yaml-flow 5.2.6 → 5.3.0

package/examples/example-board/scripts/copilot_wrapper.bat

@@ -126,6 +126,22 @@ for /f "skip=50 tokens=*" %%f in ('dir /b /o-d "!LOG_DIR!\!LOG_AGENT!_*.log" 2^>
 
 powershell -NoProfile -ExecutionPolicy Bypass -File "%~dp0copilot_wrapper_helper.ps1" "%OUTPUT_FILE%" "!RESULT_TYPE!" "!RESULT_SHAPE_FILE!"
 
+REM If JSON extraction failed (exit 2) and result_type is json, retry once with a
+REM correction prompt in the same session (--resume SESSION_UUID continues the conversation).
+SET "PS1_EXIT=!ERRORLEVEL!"
+if "!PS1_EXIT!"=="2" (
+    SET "RETRY_PROMPT_FILE=%TEMP%\copilot-retry-!RANDOM!.txt"
+    (
+        echo Your previous response did not contain a valid JSON object.
+        echo Please respond with ONLY the JSON object — no markdown, no explanation, no preamble.
+        echo Start your response with { and end with }.
+    ) > "!RETRY_PROMPT_FILE!"
+    type "!RETRY_PROMPT_FILE!" | call copilot --allow-all --resume !SESSION_UUID! !MODEL_FLAG! > "%OUTPUT_FILE%" 2>&1
+    del "!RETRY_PROMPT_FILE!" 2>nul
+    REM Re-run PS1 with -IsRetry so it writes skeleton on second failure rather than exit 2
+    powershell -NoProfile -ExecutionPolicy Bypass -File "%~dp0copilot_wrapper_helper.ps1" "%OUTPUT_FILE%" "!RESULT_TYPE!" "!RESULT_SHAPE_FILE!" "-IsRetry"
+)
+
 if exist "!CACHE_SESSION_PATH!" (
     for %%f in ("!CACHE_SESSION_PATH!\*") do (
         move /y "%%f" "%SESSION_DIR%\" >nul 2>&1

package/examples/example-board/scripts/copilot_wrapper_helper.ps1

@@ -9,13 +9,16 @@
 #                       if result_shape_file is absent, accepts any valid JSON object
 #   result_shape_file - (json result_type only) JSON file whose top-level keys are required in output
 #
-# raw  result_type: right for chat responses and task executor sources.
+# raw  result_type: right for chat responses and task executor source_defs.
 # json result_type: right for structured calls where the input contained {prompt, result_shape}.
 
 param(
     [Parameter(Mandatory)][string]$OutputFile,
     [Parameter(Mandatory)][string]$ResultType,
-    [string]$ResultShapeFile = ''
+    [string]$ResultShapeFile = '',
+    # When $true (retry pass), write shape-skeleton fallback instead of exiting with code 2.
+    # On first pass, exit 2 signals the caller (.bat) to retry with a correction prompt.
+    [switch]$IsRetry
 )
 
 if (-not (Test-Path $OutputFile)) { exit 0 }
@@ -47,7 +50,7 @@ foreach ($line in ($cleaned -split "`n")) {
         $t -match '^X ' -or                          # X failed tool ops
         $t -match '^\$ ' -or                         # $ shell commands
         $t -match '^[\u2514\u251c]' -or              # └ ├ tree lines
-        $t -match 'session-state.*\.json' -or        # session-state file refs
+        $t -match 'session-state.*\.json' -or        # session-state file projections
         $t -match 'agent.decision has been simulated' -or
         $t -match 'has been simulated and saved' -or
         $t -match '^\d+ (files?|lines?|matches?) found$' -or  # "3 files found"
@@ -163,7 +166,7 @@ if (-not $foundJson) {
 if ($foundJson) {
     [IO.File]::WriteAllText($OutputFile, $foundJson, [Text.Encoding]::UTF8)
 } else {
-    # No matching JSON found — record raw in noise file, write shape-skeleton fallback
+    # No matching JSON found — record raw in noise file for upstream visibility
     $NoiseFile = $OutputFile + '.noise'
     $fallbackNoise = "FALLBACK=no_json_match`nSHAPE_KEYS=$($shapeKeys -join ',')`nRAW_LENGTH=$($cleaned.Length)`n---`n$cleaned"
     if (Test-Path $NoiseFile) {
@@ -172,10 +175,16 @@ if ($foundJson) {
     } else {
         [IO.File]::WriteAllText($NoiseFile, $fallbackNoise, [Text.Encoding]::UTF8)
     }
-    $fallback = if ($shapeKeys.Count -gt 0) {
-        $obj = [ordered]@{}
-        foreach ($k in $shapeKeys) { $obj[$k] = $null }
-        $obj | ConvertTo-Json -Depth 2 -Compress
-    } else { '{}' }
-    [IO.File]::WriteAllText($OutputFile, $fallback, [Text.Encoding]::UTF8)
+    if ($IsRetry) {
+        # Second pass — write shape-skeleton so the card gets a structured (empty) result
+        $fallback = if ($shapeKeys.Count -gt 0) {
+            $obj = [ordered]@{}
+            foreach ($k in $shapeKeys) { $obj[$k] = $null }
+            $obj | ConvertTo-Json -Depth 2 -Compress
+        } else { '{}' }
+        [IO.File]::WriteAllText($OutputFile, $fallback, [Text.Encoding]::UTF8)
+    } else {
+        # First pass — exit 2 to signal the caller (.bat) to retry with a correction prompt
+        exit 2
+    }
 }

package/examples/example-board/scripts/workiq_wrapper.mjs

@@ -0,0 +1,66 @@
+#!/usr/bin/env node
+/**
+ * workiq_wrapper.mjs — Calls the demo-server /api/workiq/ask proxy endpoint.
+ *
+ * Usage: node workiq_wrapper.mjs <out_file>
+ *   WORKIQ_QUERY env var:       the interpolated query string
+ *   WORKIQ_SERVER_URL env var:  base URL of demo-server (default: http://127.0.0.1:7799)
+ *
+ * The demo-server has a TTY so workiq can produce output there.
+ * Writes raw WorkIQ response text to <out_file>.
+ */
+
+import http from 'node:http';
+import fs from 'node:fs';
+
+const outFile = process.argv[2];
+const query = process.env.WORKIQ_QUERY;
+const serverBase = (process.env.WORKIQ_SERVER_URL || 'http://127.0.0.1:7799').replace(/\/$/, '');
+
+if (!outFile) { console.error('workiq_wrapper: missing <out_file> argument'); process.exit(1); }
+if (!query)   { console.error('workiq_wrapper: WORKIQ_QUERY env var not set'); process.exit(1); }
+
+const body = JSON.stringify({ query });
+const url = new URL('/api/workiq/ask', serverBase);
+
+const reqOptions = {
+  hostname: url.hostname,
+  port: url.port || 80,
+  path: url.pathname,
+  method: 'POST',
+  headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(body) },
+};
+
+const req = http.request(reqOptions, (res) => {
+  let data = '';
+  res.on('data', chunk => { data += chunk; });
+  res.on('end', () => {
+    try {
+      const json = JSON.parse(data);
+      if (json.error) {
+        fs.writeFileSync(outFile, `workiq error: ${json.error}`, 'utf8');
+        process.exit(1);
+      }
+      fs.writeFileSync(outFile, json.response || '', 'utf8');
+      process.exit(0);
+    } catch {
+      fs.writeFileSync(outFile, data, 'utf8');
+      process.exit(0);
+    }
+  });
+});
+
+req.on('error', (err) => {
+  fs.writeFileSync(outFile, `workiq proxy error: ${err.message}\nIs demo-server running at ${serverBase}?`, 'utf8');
+  process.exit(1);
+});
+
+req.setTimeout(60_000, () => {
+  req.destroy();
+  fs.writeFileSync(outFile, 'workiq proxy timeout after 60s', 'utf8');
+  process.exit(1);
+});
+
+req.write(body);
+req.end();
+

package/examples/npm-libs/continuous-event-graph/live-cards-board.ts

@@ -46,9 +46,9 @@ const cards: LiveCard[] = [
     card_data: {},
     data: { requires: ['market-feed'] },
     compute: {
-      total: { fn: 'sum', input: 'sources.market-feed.prices' },
-      avg: { fn: 'avg', input: 'sources.market-feed.prices' },
-      count: { fn: 'count', input: 'sources.market-feed.prices' },
+      total: { fn: 'sum', input: 'source_defs.market-feed.prices' },
+      avg: { fn: 'avg', input: 'source_defs.market-feed.prices' },
+      count: { fn: 'count', input: 'source_defs.market-feed.prices' },
     },
   },
   {
@@ -132,8 +132,8 @@ const board: LiveBoard = {
       card_data: {},
       data: { requires: ['equity-feed', 'bond-feed'] },
       compute: {
-        equity_total: { fn: 'sum', input: 'sources.equity-feed.prices' },
-        bond_total: { fn: 'sum', input: 'sources.bond-feed.yields' },
+        equity_total: { fn: 'sum', input: 'source_defs.equity-feed.prices' },
+        bond_total: { fn: 'sum', input: 'source_defs.bond-feed.yields' },
       },
     },
     {

package/examples/npm-libs/continuous-event-graph/soc-incident-board.ts

@@ -2,7 +2,7 @@
  * SOC Incident Board — Correct Model
  *
  * A Security Operations Center dashboard demonstrating:
- *   • External data sources (alert-feed, threat-intel) — NO handlers
+ *   • External data source_defs (alert-feed, threat-intel) — NO handlers
  *   • Handler-driven compute (severity-score, blast-radius)
  *   • Fire-and-forget side effects (slack-alert, create-ticket)
  *   • Per-task refreshStrategy: 'once' on create-ticket (no duplicate tickets)
@@ -76,7 +76,7 @@ const config: GraphConfig = {
     refreshStrategy: 'data-changed',
   },
   tasks: {
-    // External sources — no handler, data pushed externally
+    // External source_defs — no handler, data pushed externally
     'alert-feed': {
       provides: ['alert-feed'],
       description: 'SIEM alert feed (external push)',
@@ -203,7 +203,7 @@ console.log(`create-ticket: refreshStrategy 'once' (no duplicate tickets)\n`);
 
 console.log(`--- T0: Initial incident (${siemAlerts.length} alerts, ${threatIntelV1.length} IOCs) ---\n`);
 
-// Push both sources simultaneously — engine stores data, auto-hash computed
+// Push both source_defs simultaneously — engine stores data, auto-hash computed
 graph.pushAll([
   {
     type: 'task-completed',

package/examples/npm-libs/event-graph/research-pipeline.ts

@@ -29,18 +29,18 @@ const graph: GraphConfig = {
   },
   tasks: {
     fetch_sources: {
-      provides: ['raw-sources'],
+      provides: ['raw-asources'],
       description: 'Fetch source documents from the web',
     },
     analyse_sentiment: {
-      requires: ['raw-sources'],
+      requires: ['raw-asources'],
       provides: ['sentiment-result'],
-      description: 'Run sentiment analysis on sources',
+      description: 'Run sentiment analysis on source_defs',
     },
     analyse_entities: {
-      requires: ['raw-sources'],
+      requires: ['raw-asources'],
       provides: ['entity-result'],
-      description: 'Extract named entities from sources',
+      description: 'Extract named entities from source_defs',
     },
     merge_analysis: {
       requires: ['sentiment-result', 'entity-result'],

package/examples/npm-libs/graph-of-graphs/multi-stage-etl.ts

@@ -13,7 +13,7 @@
  *  - Fan-out / fan-in in the outer graph
  *
  * Outer graph:
- *   discover-sources → extract-batch → transform-batch → [load ∥ validate] → finalize
+ *   discover-bsources → extract-batch → transform-batch → [load ∥ validate] → finalize
  *
  * Inner extract graph (event-graph, per source):
  *   connect → [fetch-metadata ∥ fetch-schema] → snapshot-data
@@ -129,11 +129,11 @@ const outerGraph: GraphConfig = {
   id: 'etl-pipeline',
   settings: { completion: 'all-tasks-complete' },
   tasks: {
-    'discover-sources': {
-      provides: ['sources-discovered'],
+    'discover-bsources': {
+      provides: ['source_defs-discovered'],
     },
     'extract-batch': {
-      requires: ['sources-discovered'],
+      requires: ['source_defs-discovered'],
       provides: ['extraction-complete'],
     },
     'transform-batch': {
@@ -191,7 +191,7 @@ async function runTransformFlow(record: { id: string; raw_data: string }) {
 // 4. Sample data
 // ============================================================================
 
-const sources = [
+const source_defs = [
   { id: 'src-orders', db: 'orders_db' },
   { id: 'src-users', db: 'users_db' },
   { id: 'src-products', db: 'products_db' },
@@ -211,13 +211,13 @@ const records = [
 // ============================================================================
 
 const outerHandlers: Record<string, () => Promise<void>> = {
-  'discover-sources': async () => {
-    console.log(`  Found ${sources.length} data sources`);
+  'discover-bsources': async () => {
+    console.log(`  Found ${source_defs.length} data source_defs`);
   },
 
   'extract-batch': async () => {
-    console.log(`  Extracting from ${sources.length} sources (concurrency: 2, mode: event-graph)`);
-    const result = await batch(sources, {
+    console.log(`  Extracting from ${source_defs.length} source_defs (concurrency: 2, mode: event-graph)`);
+    const result = await batch(source_defs, {
       concurrency: 2,
       processor: runExtractGraph,
       onItemComplete: (src, res) => {

package/examples/step-machine-cli/portfolio-tracker/cards/holdings-table.json

@@ -2,7 +2,7 @@
   "id": "holdings-table",
   "meta": { "title": "Holdings Table" },
   "requires": ["holdings", "prices"],
-  "provides": [{ "bindTo": "table", "src": "computed_values.table" }],
+  "provides": [{ "bindTo": "table", "ref": "computed_values.table" }],
   "card_data": {},
   "compute": [
     {

package/examples/step-machine-cli/portfolio-tracker/cards/portfolio-form.json

@@ -6,7 +6,7 @@
   "provides": [
     {
       "bindTo": "holdings",
-      "src": "card_data.holdings"
+      "ref": "card_data.holdings"
     }
   ],
   "card_data": {

package/examples/step-machine-cli/portfolio-tracker/cards/portfolio-value.json

@@ -2,7 +2,7 @@
   "id": "portfolio-value",
   "meta": { "title": "Portfolio Total Value" },
   "requires": ["table"],
-  "provides": [{ "bindTo": "totalValue", "src": "computed_values.totalValue" }],
+  "provides": [{ "bindTo": "totalValue", "ref": "computed_values.totalValue" }],
   "card_data": {},
   "compute": [
     { "bindTo": "totalValue", "expr": "$sum(requires.table.rows.value)" }

package/examples/step-machine-cli/portfolio-tracker/cards/price-fetch.json

@@ -2,14 +2,14 @@
   "id": "price-fetch",
   "meta": { "title": "Fetch Market Prices" },
   "requires": ["holdings"],
-  "provides": [{ "bindTo": "prices", "src": "sources.prices" }],
+  "provides": [{ "bindTo": "prices", "ref": "fetched_sources.prices" }],
   "card_data": {},
-  "sources": [
+  "source_defs": [
     { "cli": "node ../fetch-prices.js --tmp-file-name tmp_file1", "bindTo": "prices", "outputFile": "prices.json" }
   ],
   "view": {
     "elements": [
-      { "kind": "table", "label": "Market Prices", "data": { "bind": "sources.prices" } }
+      { "kind": "table", "label": "Market Prices", "data": { "bind": "fetched_sources.prices" } }
     ]
   }
 }

package/examples/step-machine-cli/portfolio-tracker/handlers/add-cards-cli.js

@@ -12,7 +12,7 @@ try {
     process.exit(0);
   }
 
-  runBoardCli(['add-cards', '--rg', boardDir, '--card-glob', cardsGlob]);
+  runBoardCli(['upsert-card', '--rg', boardDir, '--card-glob', cardsGlob]);
 
   writeResult({
     result: 'success',

package/examples/step-machine-cli/portfolio-tracker/handlers/update-holdings-cli.js

@@ -22,7 +22,7 @@ try {
   card.state.holdings = holdings;
   fs.writeFileSync(cardPath, `${JSON.stringify(card, null, 2)}\n`, 'utf-8');
 
-  runBoardCli(['update-card', '--rg', boardDir, '--card-id', 'portfolio-form', '--restart']);
+  runBoardCli(['upsert-card', '--rg', boardDir, '--card', cardPath, '--restart']);
 
   writeResult({
     result: 'success',

package/examples/step-machine-cli/portfolio-tracker/portfolio-tracker.flow.yaml

@@ -44,7 +44,7 @@ steps:
     expects_data: [board_dir, cards_template_dir]
     produces_data: [cards_dir]
     handler:
-      cli: node %%BOARDCLI_CMD%% add-cards --rg "%%BOARD_DIR%%" --card-glob "%%CARDS_GLOB%%"
+      cli: node %%BOARDCLI_CMD%% upsert-card --rg "%%BOARD_DIR%%" --card-glob "%%CARDS_GLOB%%"
       result-mode: exit-code
       input-transforms:
         BOARD_DIR: board_dir

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "yaml-flow",
-  "version": "5.2.6",
+  "version": "5.3.0",
   "description": "Unified workflow engine: step-machine (sequential) + event-graph (stateless DAG) with pluggable storage",
   "author": "",
   "license": "MIT",
@@ -101,7 +101,7 @@
   "scripts": {
     "build": "tsup",
     "build:browser": "tsup --config tsup.browser.config.ts",
-    "build:example": "node example-board-src/build.js",
+    "build:example": "node demo-src/example-board/build.js",
     "dev": "tsup --watch",
     "test": "vitest",
     "test:run": "vitest run",

package/schema/live-cards.schema.json

@@ -80,10 +80,10 @@
       "type": "array",
       "items": {
         "type": "object",
-        "required": ["bindTo", "src"],
+        "required": ["bindTo", "ref"],
         "properties": {
           "bindTo": { "type": "string", "description": "Token name published downstream" },
-          "src":    { "type": "string", "description": "Path to read value from (card_data.*, requires.*, fetched_sources.*, computed_values.*)" }
+          "ref":    { "type": "string", "description": "Path to read value from (card_data.*, requires.*, fetched_sources.*, computed_values.*)" }
         }
       },
       "description": "Explicit bindings exposing computed or card_data values downstream as named tokens"
@@ -100,14 +100,19 @@
     },
 
     "source_def": {
-      "description": "One source entry. The engine requires 'bindTo' (compute namespace key) and 'outputFile' (delivery signal path). bindTo and outputFile must be unique across all sources in a card. Every other property is yours — add whatever your task-executor needs: kind, url, headers, mailbox, channel, model, query, etc. The full object is passed verbatim as the --in JSON to the executor.",
+      "description": "One source entry. The engine requires 'bindTo' (compute namespace key) and 'outputFile' (delivery signal path). bindTo and outputFile must be unique across all source_defs in a card. Every other property is yours — add whatever your task-executor needs: kind, url, headers, mailbox, channel, model, query, etc. The full object is passed verbatim as the --in JSON to the executor.",
       "type": "object",
       "required": ["bindTo", "outputFile"],
       "additionalProperties": true,
       "properties": {
         "bindTo":     { "type": "string",  "description": "Key under fetched_sources.* available in compute expressions" },
         "outputFile": { "type": "string",  "description": "Board-relative path the executor writes its JSON result to. Presence of this file signals delivery." },
-        "optionalForCompletionGating":   { "type": "boolean", "default": false, "description": "When true this source does not gate card completion. Default false when absent, so sources are completion-gating by default." },
+        "projections": {
+          "type": "object",
+          "description": "Named data projections from card_data or requires, evaluated before the executor is called. Each key is a ref name; each value is a JSONata expression rooted at card_data or requires. The resolved values are passed to the executor as _projections. fetched_sources, computed_values, and source_defs are not accessible here — sources run before those exist.",
+          "additionalProperties": { "type": "string" }
+        },
+        "optionalForCompletionGating":   { "type": "boolean", "default": false, "description": "When true this source does not gate card completion. Default false when absent, so source_defs are completion-gating by default." },
         "timeout":    { "type": "integer", "minimum": 0, "default": 120000, "description": "Executor/script timeout in ms. Default: 120 000 (2 min)." },
         "script":     { "type": "string",  "description": "Legacy direct-run: shell command executed when no .task-executor is registered. stdout is captured as the result." }
       }
@@ -121,7 +126,7 @@
         "kind":      {
           "enum": ["metric", "table", "editable-table", "chart", "form", "filter", "list",
                    "notes", "todo", "alert", "narrative", "badge", "text",
-                   "markdown", "custom", "actions"]
+                   "markdown", "ref", "custom", "actions"]
         },
         "label":     { "type": "string", "description": "Heading above this element" },
         "className": { "type": "string", "description": "Bootstrap grid class, e.g. 'col-12 col-md-6'" },
@@ -215,7 +220,7 @@
   },
 
   "title": "LiveCard",
-  "description": "A unified card node. Behavior depends on which sections are present (sources, compute, view, etc.)",
+  "description": "A unified card node. Behavior depends on which sections are present (source_defs, compute, view, etc.)",
   "type": "object",
   "required": ["id"],
   "additionalProperties": false,
@@ -258,22 +263,22 @@
         },
         "llm_task_completion_inference": {
           "type": "object",
-          "description": "Runtime state written by the inference adapter (advanced/undocumented). Prefer the standard sources → compute → provides pattern for LLM-based signals.",
+          "description": "Runtime state written by the inference adapter (advanced/undocumented). Prefer the standard source_defs → compute → provides pattern for LLM-based signals.",
           "properties": {
             "inferenceRequested": { "type": "string", "format": "date-time", "description": "Timestamp when the latest inference request was initiated" },
             "inferenceCompletedAt": { "type": "string", "format": "date-time", "description": "Timestamp when the latest inference request completed" },
             "isTaskCompleted": { "type": "boolean", "description": "Whether the task is considered complete by the adapter" },
             "reasoning": { "type": "string", "description": "Explanation of completion decision" },
-            "evidence": { "type": "array", "description": "Supporting evidence from sources/compute" }
+            "evidence": { "type": "array", "description": "Supporting evidence from source_defs/compute" }
           },
           "additionalProperties": true
         }
       },
       "additionalProperties": true
     },
-    "sources": {
+    "source_defs": {
       "type": "array",
-      "description": "Source entries. Each entry is passed verbatim to the board's .task-executor (registered via init --task-executor) as the --in JSON file. The executor fetches/generates the data and writes JSON to --out. If no executor is registered, the built-in executor runs the entry's 'cli' command directly. Sources gate completion by default. Set optionalForCompletionGating: true for enrichment-only sources that should not block task-completed.",
+      "description": "Source entries. Each entry is passed verbatim to the board's .task-executor (registered via init --task-executor) as the --in JSON file. The executor fetches/generates the data and writes JSON to --out. If no executor is registered, the built-in executor runs the entry's 'cli' command directly. Sources gate completion by default. Set optionalForCompletionGating: true for enrichment-only source_defs that should not block task-completed.",
       "items": { "$ref": "#/definitions/source_def" }
     },
     "compute": {