yaml-flow 7.0.0 → 8.0.0

package/examples/board/gandalf-cards/card-source-kinds.json

@@ -0,0 +1,36 @@
+{
+  "id": "card-source-kinds",
+  "meta": {
+    "title": "Source Connection Types",
+    "tags": ["admin"],
+    "desc": "Reference card listing the source def kinds supported by the demo task executor.",
+    "_gandalfCard": true
+  },
+  "provides": [],
+  "compute": [],
+  "view": {
+    "elements": [
+      {
+        "kind": "table",
+        "label": "Supported connection types",
+        "data": {
+          "bind": "card_data.kinds",
+          "columns": ["kind", "description"]
+        }
+      }
+    ],
+    "layout": {
+      "board": { "col": 4, "order": 99 },
+      "canvas": { "x": 900, "y": 50, "w": 480, "h": 240 }
+    }
+  },
+  "card_data": {
+    "kinds": [
+      { "kind": "mock", "description": "Look up a key in the built-in MOCK_DB dictionary and return static JSON." },
+      { "kind": "copilot", "description": "Invoke GitHub Copilot CLI using copilot.prompt_template (or top-level prompt_template shorthand) with {{key}} interpolation from _projections and args." },
+      { "kind": "workiq", "description": "Invoke WorkIQ (Microsoft 365 Copilot) using workiq.query_template with {{key}} interpolation. Requires workiq CLI + az login." },
+      { "kind": "url", "description": "Single URL fetch via curl with {{key}} interpolation from _projections. Supports method, headers, args, tickersFrom, and cacheTimeout (seconds)." },
+      { "kind": "url-list", "description": "Fan-out fetch over _projections.url_list (string[]), returning an array of raw JSON responses. Supports method, headers, and cacheTimeout." }
+    ]
+  }
+}

package/examples/board/gandalf-cards/cards/_index.json

@@ -0,0 +1,7 @@
+{
+  "card-source-kinds": {
+    "key": "card-source-kinds",
+    "checksum": "",
+    "updatedAt": "2026-05-07T19:32:58.245Z"
+  }
+}

package/examples/board/gandalf-cards/cards/card-source-kinds.json

@@ -0,0 +1,64 @@
+{
+  "id": "card-source-kinds",
+  "meta": {
+    "title": "Source Connection Types",
+    "tags": [
+      "admin"
+    ],
+    "desc": "Reference card listing the source def kinds supported by the demo task executor.",
+    "_gandalfCard": true
+  },
+  "provides": [],
+  "compute": [],
+  "view": {
+    "elements": [
+      {
+        "kind": "table",
+        "label": "Supported connection types",
+        "data": {
+          "bind": "card_data.kinds",
+          "columns": [
+            "kind",
+            "description"
+          ]
+        }
+      }
+    ],
+    "layout": {
+      "board": {
+        "col": 4,
+        "order": 99
+      },
+      "canvas": {
+        "x": 900,
+        "y": 50,
+        "w": 480,
+        "h": 240
+      }
+    }
+  },
+  "card_data": {
+    "kinds": [
+      {
+        "kind": "mock",
+        "description": "Look up a key in the built-in MOCK_DB dictionary and return static JSON."
+      },
+      {
+        "kind": "copilot",
+        "description": "Invoke GitHub Copilot CLI using copilot.prompt_template (or top-level prompt_template shorthand) with {{key}} interpolation from _projections and args."
+      },
+      {
+        "kind": "workiq",
+        "description": "Invoke WorkIQ (Microsoft 365 Copilot) using workiq.query_template with {{key}} interpolation. Requires workiq CLI + az login."
+      },
+      {
+        "kind": "url",
+        "description": "Single URL fetch via curl with {{key}} interpolation from _projections. Supports method, headers, args, tickersFrom, and cacheTimeout (seconds)."
+      },
+      {
+        "kind": "url-list",
+        "description": "Fan-out fetch over _projections.url_list (string[]), returning an array of raw JSON responses. Supports method, headers, and cacheTimeout."
+      }
+    ]
+  }
+}

package/examples/board/source-def-flows/copilot.flow.json

@@ -0,0 +1,33 @@
+{
+  "id": "demo-source-copilot",
+  "settings": {
+    "start_step": "execute",
+    "max_total_steps": 8,
+    "timeout_ms": 180000
+  },
+  "steps": {
+    "execute": {
+      "description": "Execute copilot source def",
+      "handler": {
+        "type": "ref",
+        "howToRun": "demo-local-module",
+        "whatToRun": { "kind": "fs-path", "value": "./source-def-handlers/copilot-source-handler.js" }
+      },
+      "transitions": {
+        "success": "completed",
+        "failure": "failed"
+      }
+    }
+  },
+  "terminal_states": {
+    "completed": {
+      "description": "Copilot source succeeded",
+      "return_intent": "success",
+      "return_artifacts": ["resultValue", "wroteOutputDirectly"]
+    },
+    "failed": {
+      "description": "Copilot source failed",
+      "return_intent": "failure"
+    }
+  }
+}

package/examples/board/source-def-flows/mock.flow.json

@@ -0,0 +1,35 @@
+{
+  "id": "demo-source-mock",
+  "settings": {
+    "start_step": "execute",
+    "max_total_steps": 5,
+    "timeout_ms": 30000
+  },
+  "steps": {
+    "execute": {
+      "description": "Execute mock source def",
+      "handler": {
+        "type": "compute-jsonata",
+        "expr": [
+          "data.resultValue = $lookup(expects_data.mockDb, expects_data.sourceDef.mock)",
+          "result = $exists(data.resultValue) ? \"success\" : \"failure\""
+        ]
+      },
+      "transitions": {
+        "success": "completed",
+        "failure": "failed"
+      }
+    }
+  },
+  "terminal_states": {
+    "completed": {
+      "description": "Mock lookup succeeded",
+      "return_intent": "success",
+      "return_artifacts": ["resultValue", "wroteOutputDirectly"]
+    },
+    "failed": {
+      "description": "Mock lookup failed",
+      "return_intent": "failure"
+    }
+  }
+}

package/examples/board/source-def-flows/url-list.flow.json

@@ -0,0 +1,33 @@
+{
+  "id": "demo-source-url-list",
+  "settings": {
+    "start_step": "execute",
+    "max_total_steps": 10,
+    "timeout_ms": 60000
+  },
+  "steps": {
+    "execute": {
+      "description": "Execute URL list source def",
+      "handler": {
+        "type": "ref",
+        "howToRun": "demo-local-module",
+        "whatToRun": { "kind": "fs-path", "value": "./source-def-handlers/http-source-handler.js" }
+      },
+      "transitions": {
+        "success": "completed",
+        "failure": "failed"
+      }
+    }
+  },
+  "terminal_states": {
+    "completed": {
+      "description": "URL list fetch succeeded",
+      "return_intent": "success",
+      "return_artifacts": ["resultValue", "wroteOutputDirectly"]
+    },
+    "failed": {
+      "description": "URL list fetch failed",
+      "return_intent": "failure"
+    }
+  }
+}

package/examples/board/source-def-flows/url.flow.json

@@ -0,0 +1,33 @@
+{
+  "id": "demo-source-url",
+  "settings": {
+    "start_step": "execute",
+    "max_total_steps": 10,
+    "timeout_ms": 60000
+  },
+  "steps": {
+    "execute": {
+      "description": "Execute single URL source def",
+      "handler": {
+        "type": "ref",
+        "howToRun": "demo-local-module",
+        "whatToRun": { "kind": "fs-path", "value": "./source-def-handlers/http-source-handler.js" }
+      },
+      "transitions": {
+        "success": "completed",
+        "failure": "failed"
+      }
+    }
+  },
+  "terminal_states": {
+    "completed": {
+      "description": "URL fetch succeeded",
+      "return_intent": "success",
+      "return_artifacts": ["resultValue", "wroteOutputDirectly"]
+    },
+    "failed": {
+      "description": "URL fetch failed",
+      "return_intent": "failure"
+    }
+  }
+}

package/examples/board/source-def-flows/workiq.flow.json

@@ -0,0 +1,34 @@
+{
+  "id": "demo-source-workiq",
+  "settings": {
+    "start_step": "execute",
+    "max_total_steps": 8,
+    "timeout_ms": 120000
+  },
+  "steps": {
+    "execute": {
+      "description": "Execute workiq source def via demo-server HTTP proxy",
+      "handler": {
+        "type": "ref",
+        "howToRun": "http:post",
+        "whatToRun": { "kind": "http-url", "value": "/api/workiq/ask" }
+      },
+      "transitions": {
+        "success": "completed",
+        "failure": "failed"
+      }
+    }
+  },
+  "terminal_states": {
+    "completed": {
+      "description": "WorkIQ source succeeded",
+      "return_intent": "success",
+      "return_artifacts": ["resultValue", "wroteOutputDirectly"]
+    },
+    "failed": {
+      "description": "WorkIQ source failed",
+      "return_intent": "failure",
+      "return_artifacts": ["error"]
+    }
+  }
+}

package/examples/board/source-def-handlers/copilot-source-handler.js

@@ -0,0 +1,141 @@
+#!/usr/bin/env node
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { execFileSync } from 'node:child_process';
+
+function interpolate(template, args) {
+  return String(template).replace(/\{\{\s*([a-zA-Z0-9_.-]+)\s*\}\}/g, (_m, key) => {
+    const v = args?.[key];
+    if (v === undefined) return '';
+    return typeof v === 'string' ? v : JSON.stringify(v);
+  });
+}
+
+const DEFAULT_PROMPT_CONTEXT = {
+  view_kind_guidance: [
+    'VIEW KIND GUIDANCE (for dynamic ref rendering):',
+    '- Return a _view object whenever your output data is meant for a ref element.',
+    '- Allowed _view.kind values only: table, editable-table, chart, metric, list, badge, text, narrative, markdown, form, filter, todo, alert.',
+    '- If uncertain, use "table".',
+    '- For array rows that users should edit, prefer "editable-table" and set _view.data.writeTo to a card_data path.',
+    '- For chart, set _view.data.chartType and _view.data.columns with [labelField, valueField].',
+    '- Keep _view.data minimal and valid JSON (no comments, no trailing text).',
+  ].join('\n'),
+  card_layout_guidance: [
+    'CARD LAYOUT GUIDANCE:',
+    '- Prefer compact outputs that fit a card: one primary structure plus concise rationale text.',
+    '- Avoid repeating values already present in upstream inputs.',
+    '- If you produce both machine-readable and human-readable content, keep machine-readable fields top-level and concise prose in a separate field.',
+  ].join('\n'),
+};
+
+function resolvePrompt(sourceDef, promptContext) {
+  const cfg = sourceDef?.copilot && typeof sourceDef.copilot === 'object' ? sourceDef.copilot : {};
+  const template = cfg.prompt_template ?? sourceDef.prompt_template;
+  if (!template || typeof template !== 'string') return null;
+  const args = {
+    ...(promptContext || DEFAULT_PROMPT_CONTEXT),
+    ...(sourceDef?._projections || {}),
+    ...(cfg.args || sourceDef.args || {}),
+  };
+  return interpolate(template, args);
+}
+
+function runCopilot(prompt, sourceDef, executorDir, extra) {
+  const wrapperPath = path.join(executorDir, 'scripts', 'copilot_wrapper.bat');
+  if (process.platform === 'win32' && fs.existsSync(wrapperPath)) {
+    const tmpBase = path.join(process.env.TEMP || process.cwd(), `copilot-handler-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`);
+    const outFile = `${tmpBase}.out.json`;
+    const promptFile = `${tmpBase}.prompt.txt`;
+    const sessionDir = path.join(
+      extra?.boardSetupRoot || (process.env.TEMP || process.cwd()),
+      'copilot-sessions',
+      String(sourceDef?.bindTo || 'default').replace(/[^a-zA-Z0-9_-]/g, '_'),
+    );
+    fs.mkdirSync(sessionDir, { recursive: true });
+    fs.writeFileSync(promptFile, prompt, 'utf-8');
+
+    let shapeFile = '';
+    const shape = sourceDef?.copilot?.result_shape ?? sourceDef?.result_shape;
+    if (shape && typeof shape === 'object') {
+      shapeFile = `${tmpBase}.shape.json`;
+      fs.writeFileSync(shapeFile, JSON.stringify(shape), 'utf-8');
+    }
+
+    try {
+      execFileSync('cmd.exe', [
+        '/d', '/c',
+        wrapperPath,
+        outFile,
+        sessionDir,
+        extra?.boardSetupRoot || process.cwd(),
+        `@${promptFile}`,
+        'json',
+        String(sourceDef?.bindTo || 'executor'),
+        '',
+        shapeFile,
+      ], {
+        encoding: 'utf-8',
+        stdio: ['ignore', 'pipe', 'pipe'],
+        windowsHide: true,
+        maxBuffer: 10 * 1024 * 1024,
+      });
+      return JSON.parse(fs.readFileSync(outFile, 'utf-8').replace(/^\uFEFF/, ''));
+    } finally {
+      try { fs.unlinkSync(promptFile); } catch {}
+      try { fs.unlinkSync(outFile); } catch {}
+      if (shapeFile) { try { fs.unlinkSync(shapeFile); } catch {} }
+    }
+  }
+
+  const stdout = execFileSync('copilot', ['--allow-all'], {
+    input: prompt,
+    encoding: 'utf-8',
+    stdio: ['pipe', 'pipe', 'pipe'],
+    timeout: 120000,
+    cwd: extra?.boardSetupRoot || process.cwd(),
+    windowsHide: true,
+    maxBuffer: 10 * 1024 * 1024,
+  });
+
+  const firstBrace = stdout.indexOf('{');
+  const firstBracket = stdout.indexOf('[');
+  let jsonStart = -1;
+  if (firstBrace === -1) jsonStart = firstBracket;
+  else if (firstBracket === -1) jsonStart = firstBrace;
+  else jsonStart = Math.min(firstBrace, firstBracket);
+
+  if (jsonStart !== -1) {
+    try {
+      return JSON.parse(stdout.slice(jsonStart));
+    } catch {
+      return stdout;
+    }
+  }
+  return stdout;
+}
+
+export async function execute(context) {
+  const sourceDef = context?.sourceDef || {};
+  const extra = context?.extra || {};
+  const executorDir = context?.executorDir || process.cwd();
+  const promptContext = context?.promptContext || DEFAULT_PROMPT_CONTEXT;
+
+  const prompt = resolvePrompt(sourceDef, promptContext);
+  if (!prompt) {
+    return {
+      result: 'failure',
+      data: { error: 'Source definition missing copilot.prompt_template (or prompt_template)' },
+      error: 'missing prompt_template',
+    };
+  }
+
+  try {
+    const resultValue = runCopilot(prompt, sourceDef, executorDir, extra);
+    return { result: 'success', data: { resultValue } };
+  } catch (err) {
+    const msg = String(err?.message || err);
+    return { result: 'failure', data: { error: `copilot invocation failed: ${msg}` }, error: msg };
+  }
+}

package/examples/board/source-def-handlers/http-source-handler.js

@@ -0,0 +1,145 @@
+#!/usr/bin/env node
+
+import fs from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+import crypto from 'node:crypto';
+
+const CACHE_DIR = path.join(os.tmpdir(), 'demo-executor-cache');
+const DEFAULT_CACHE_TTL_MS = 60 * 60 * 1000;
+
+function interpolate(template, args) {
+  return String(template).replace(/\{\{\s*([a-zA-Z0-9_.-]+)\s*\}\}/g, (_m, key) => {
+    const v = args?.[key];
+    if (v === undefined) return '';
+    return typeof v === 'string' ? v : JSON.stringify(v);
+  });
+}
+
+function cacheKey(seed) {
+  return crypto.createHash('sha1').update(seed).digest('hex');
+}
+
+function readCache(key, ttlMs) {
+  const p = path.join(CACHE_DIR, `${key}.json`);
+  try {
+    const stat = fs.statSync(p);
+    const ageMs = Date.now() - stat.mtimeMs;
+    if (ageMs > ttlMs) return null;
+    return JSON.parse(fs.readFileSync(p, 'utf-8'));
+  } catch {
+    return null;
+  }
+}
+
+function writeCache(key, value) {
+  fs.mkdirSync(CACHE_DIR, { recursive: true });
+  fs.writeFileSync(path.join(CACHE_DIR, `${key}.json`), JSON.stringify(value));
+}
+
+async function fetchJson(url, method, headers) {
+  const response = await fetch(url, { method, headers });
+  const text = await response.text();
+  let parsed;
+  try {
+    parsed = text ? JSON.parse(text) : {};
+  } catch {
+    throw new Error(`Invalid JSON from ${url}`);
+  }
+  if (!response.ok) {
+    const msg = typeof parsed?.error === 'string' ? parsed.error : `HTTP ${response.status}`;
+    throw new Error(msg);
+  }
+  return parsed;
+}
+
+async function doFetchApi(url, method, headers, ttlMs) {
+  const key = cacheKey(`url:${method}:${url}`);
+  const cached = readCache(key, ttlMs);
+  if (cached !== null) {
+    console.error(`[http-source-handler] cache hit for ${url}`);
+    return cached;
+  }
+  const data = await fetchJson(url, method, headers);
+  writeCache(key, data);
+  return data;
+}
+
+function resolveTickersArg(sourceDef, fetchArgs) {
+  const tickersFrom = sourceDef?.tickersFrom;
+  if (typeof tickersFrom !== 'string' || !tickersFrom.includes('.')) return;
+  const [refKey, fieldName] = tickersFrom.split('.', 2);
+  const arr = sourceDef?._projections?.[refKey];
+  if (!Array.isArray(arr)) return;
+  const vals = arr
+    .filter((row) => row && typeof row === 'object' && row[fieldName])
+    .map((row) => String(row[fieldName]));
+  if (vals.length > 0) fetchArgs.tickers = vals.join(',');
+}
+
+async function executeUrl(sourceDef) {
+  const cfg = sourceDef?.url;
+  if (!cfg || typeof cfg !== 'object') {
+    throw new Error('url source requires object config');
+  }
+  const method = String(cfg.method || 'GET').toUpperCase();
+  const headers = cfg.headers && typeof cfg.headers === 'object' ? cfg.headers : {};
+  const ttlMs = typeof cfg.cacheTimeout === 'number' ? cfg.cacheTimeout * 1000 : DEFAULT_CACHE_TTL_MS;
+  const fetchArgs = cfg.args && typeof cfg.args === 'object' ? { ...cfg.args } : {};
+
+  resolveTickersArg(sourceDef, fetchArgs);
+  if (sourceDef?.tickersFrom && !fetchArgs.tickers) {
+    throw new Error('url: tickersFrom resolved to empty list - skipping fetch');
+  }
+
+  const ctx = {
+    ...(sourceDef?._projections || {}),
+    ...fetchArgs,
+  };
+
+  if (typeof cfg.url !== 'string' || !cfg.url) {
+    throw new Error('url source missing url template');
+  }
+  const url = interpolate(cfg.url, ctx);
+  return doFetchApi(url, method, headers, ttlMs);
+}
+
+async function executeUrlList(sourceDef) {
+  const cfg = sourceDef?.['url-list'];
+  if (!cfg || typeof cfg !== 'object') {
+    throw new Error('url-list source requires object config');
+  }
+  const method = String(cfg.method || 'GET').toUpperCase();
+  const headers = cfg.headers && typeof cfg.headers === 'object' ? cfg.headers : {};
+  const ttlMs = typeof cfg.cacheTimeout === 'number' ? cfg.cacheTimeout * 1000 : DEFAULT_CACHE_TTL_MS;
+
+  const urlList = sourceDef?._projections?.url_list;
+  if (!Array.isArray(urlList) || urlList.length === 0) {
+    throw new Error('url-list source requires _projections.url_list as non-empty array');
+  }
+
+  const results = [];
+  for (const u of urlList) {
+    results.push(await doFetchApi(String(u), method, headers, ttlMs));
+  }
+  return results;
+}
+
+export async function execute(context) {
+  const kind = context?.kind;
+  const sourceDef = context?.sourceDef || {};
+  try {
+    let resultValue;
+    if (kind === 'url') {
+      resultValue = await executeUrl(sourceDef);
+    } else if (kind === 'url-list') {
+      resultValue = await executeUrlList(sourceDef);
+    } else {
+      throw new Error(`http-source-handler does not support kind: ${kind}`);
+    }
+    return { result: 'success', data: { resultValue } };
+  } catch (err) {
+    const msg = String(err?.message || err);
+    return { result: 'failure', data: { error: msg }, error: msg };
+  }
+}