yaml-flow 6.0.0 → 7.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (166) hide show
  1. package/board-live-cards-cli.js +4 -4
  2. package/browser/asset-integrity.json +3 -3
  3. package/browser/board-livecards-client.js +2 -0
  4. package/browser/board-livecards-client.js.map +1 -0
  5. package/browser/board-livecards-localstorage.js +10 -0
  6. package/browser/board-livecards-localstorage.js.map +1 -0
  7. package/browser/board-livegraph-engine.js +2 -2
  8. package/browser/board-livegraph-engine.js.map +1 -1
  9. package/browser/card-compute.js +28 -28
  10. package/browser/compute-jsonata.js +5 -0
  11. package/browser/compute-jsonata.js.map +1 -0
  12. package/browser/live-cards.js +264 -151
  13. package/card-store.js +4 -4
  14. package/dist/{board-live-cards-public-CltXYgaY.d.cts → board-live-cards-public-5n1-syA3.d.cts} +8 -5
  15. package/dist/{board-live-cards-public-f-E-FAyp.d.ts → board-live-cards-public-CK_J8uv0.d.ts} +8 -5
  16. package/dist/board-livegraph-runtime/index.cjs +2 -2
  17. package/dist/board-livegraph-runtime/index.cjs.map +1 -1
  18. package/dist/board-livegraph-runtime/index.d.cts +11 -9
  19. package/dist/board-livegraph-runtime/index.d.ts +11 -9
  20. package/dist/board-livegraph-runtime/index.js +2 -2
  21. package/dist/board-livegraph-runtime/index.js.map +1 -1
  22. package/dist/board-livegraph-runtime/jsonata-sync.cjs +37 -1
  23. package/dist/card-compute/index.cjs +4 -4
  24. package/dist/card-compute/index.cjs.map +1 -1
  25. package/dist/card-compute/index.d.cts +5 -1
  26. package/dist/card-compute/index.d.ts +5 -1
  27. package/dist/card-compute/index.js +4 -4
  28. package/dist/card-compute/index.js.map +1 -1
  29. package/dist/card-compute/jsonata-sync.cjs +37 -1
  30. package/dist/cli/browser-api/board-live-cards-browser-adapter.cjs +2 -1
  31. package/dist/cli/browser-api/board-live-cards-browser-adapter.cjs.map +1 -1
  32. package/dist/cli/browser-api/board-live-cards-browser-adapter.d.cts +27 -14
  33. package/dist/cli/browser-api/board-live-cards-browser-adapter.d.ts +27 -14
  34. package/dist/cli/browser-api/board-live-cards-browser-adapter.js +2 -1
  35. package/dist/cli/browser-api/board-live-cards-browser-adapter.js.map +1 -1
  36. package/dist/cli/browser-api/card-store-browser-api.cjs +1 -1
  37. package/dist/cli/browser-api/card-store-browser-api.cjs.map +1 -1
  38. package/dist/cli/browser-api/card-store-browser-api.js +1 -1
  39. package/dist/cli/browser-api/card-store-browser-api.js.map +1 -1
  40. package/dist/cli/browser-api/jsonata-sync.cjs +37 -1
  41. package/dist/cli/node/artifacts-store-cli.cjs +8 -8
  42. package/dist/cli/node/artifacts-store-cli.cjs.map +1 -1
  43. package/dist/cli/node/artifacts-store-cli.js +8 -8
  44. package/dist/cli/node/artifacts-store-cli.js.map +1 -1
  45. package/dist/cli/node/board-live-cards-cli.cjs +7 -7
  46. package/dist/cli/node/board-live-cards-cli.cjs.map +1 -1
  47. package/dist/cli/node/board-live-cards-cli.js +7 -7
  48. package/dist/cli/node/board-live-cards-cli.js.map +1 -1
  49. package/dist/cli/node/card-store-cli.cjs +5 -5
  50. package/dist/cli/node/card-store-cli.cjs.map +1 -1
  51. package/dist/cli/node/card-store-cli.js +5 -5
  52. package/dist/cli/node/card-store-cli.js.map +1 -1
  53. package/dist/cli/node/execution-adapter.cjs +3 -0
  54. package/dist/cli/node/execution-adapter.cjs.map +1 -0
  55. package/dist/cli/node/execution-adapter.d.cts +174 -0
  56. package/dist/cli/node/execution-adapter.d.ts +174 -0
  57. package/dist/cli/node/execution-adapter.js +3 -0
  58. package/dist/cli/node/execution-adapter.js.map +1 -0
  59. package/dist/cli/node/fs-board-adapter.cjs +7 -7
  60. package/dist/cli/node/fs-board-adapter.cjs.map +1 -1
  61. package/dist/cli/node/fs-board-adapter.d.cts +2 -2
  62. package/dist/cli/node/fs-board-adapter.d.ts +2 -2
  63. package/dist/cli/node/fs-board-adapter.js +7 -7
  64. package/dist/cli/node/fs-board-adapter.js.map +1 -1
  65. package/dist/cli/node/jsonata-sync.cjs +37 -1
  66. package/dist/cli/node/source-cli-task-executor.cjs +4 -4
  67. package/dist/cli/node/source-cli-task-executor.cjs.map +1 -1
  68. package/dist/cli/node/source-cli-task-executor.js +4 -4
  69. package/dist/cli/node/source-cli-task-executor.js.map +1 -1
  70. package/dist/continuous-event-graph/index.cjs +2 -2
  71. package/dist/continuous-event-graph/index.cjs.map +1 -1
  72. package/dist/continuous-event-graph/index.js +2 -2
  73. package/dist/continuous-event-graph/index.js.map +1 -1
  74. package/dist/continuous-event-graph/jsonata-sync.cjs +37 -1
  75. package/dist/execution-refs.cjs +2 -1
  76. package/dist/execution-refs.cjs.map +1 -1
  77. package/dist/execution-refs.d.cts +55 -12
  78. package/dist/execution-refs.d.ts +55 -12
  79. package/dist/execution-refs.js +2 -1
  80. package/dist/execution-refs.js.map +1 -1
  81. package/dist/index.cjs +10 -10
  82. package/dist/index.cjs.map +1 -1
  83. package/dist/index.js +10 -10
  84. package/dist/index.js.map +1 -1
  85. package/dist/jsonata-sync.cjs +37 -1
  86. package/dist/server-runtime/index.cjs +9 -0
  87. package/dist/server-runtime/index.cjs.map +1 -0
  88. package/dist/server-runtime/index.d.cts +31 -0
  89. package/dist/server-runtime/index.d.ts +31 -0
  90. package/dist/server-runtime/index.js +9 -0
  91. package/dist/server-runtime/index.js.map +1 -0
  92. package/dist/server-runtime/jsonata-sync.cjs +7623 -0
  93. package/dist/step-machine-public/index.cjs +3 -0
  94. package/dist/step-machine-public/index.cjs.map +1 -0
  95. package/dist/step-machine-public/index.d.cts +166 -0
  96. package/dist/step-machine-public/index.d.ts +166 -0
  97. package/dist/step-machine-public/index.js +3 -0
  98. package/dist/step-machine-public/index.js.map +1 -0
  99. package/dist/step-machine-public/jsonata-sync.cjs +7623 -0
  100. package/dist/storage-refs.cjs +2 -2
  101. package/dist/storage-refs.cjs.map +1 -1
  102. package/dist/storage-refs.d.cts +6 -6
  103. package/dist/storage-refs.d.ts +6 -6
  104. package/dist/storage-refs.js +2 -2
  105. package/dist/storage-refs.js.map +1 -1
  106. package/dist/types-CU3DjTKL.d.cts +147 -0
  107. package/dist/types-HGDTWIun.d.ts +147 -0
  108. package/examples/browser/boards/portfolio-tracker/portfolio-t4.js +9 -10
  109. package/examples/browser/boards/portfolio-tracker/portfolio-tracker-http-test.js +370 -0
  110. package/examples/browser/boards/portfolio-tracker/portfolio-tracker-http-test.py +398 -0
  111. package/examples/browser/boards/portfolio-tracker/portfolio-tracker-public.js +9 -10
  112. package/examples/browser/boards/portfolio-tracker/portfolio-tracker-server.js +300 -0
  113. package/examples/browser/boards/portfolio-tracker/portfolio-tracker-server.py +617 -0
  114. package/examples/browser/boards/portfolio-tracker/portfolio-tracker-sse-worker.js +48 -0
  115. package/examples/browser/boards/portfolio-tracker/portfolio-tracker.py +11 -10
  116. package/examples/cli/step-machine-cli/portfolio-tracker/handlers/_board-cli.js +19 -4
  117. package/examples/cli/step-machine-cli/portfolio-tracker/handlers/add-cards-cli.js +4 -8
  118. package/examples/cli/step-machine-cli/portfolio-tracker/handlers/init-board-cli.js +6 -10
  119. package/examples/cli/step-machine-cli/portfolio-tracker/handlers/poll-status-cli.js +8 -16
  120. package/examples/cli/step-machine-cli/portfolio-tracker/handlers/reset-board-dir-cli.js +2 -6
  121. package/examples/cli/step-machine-cli/portfolio-tracker/handlers/retrigger-cli.js +4 -8
  122. package/examples/cli/step-machine-cli/portfolio-tracker/handlers/status-cli.js +3 -7
  123. package/examples/cli/step-machine-cli/portfolio-tracker/handlers/update-holdings-cli.js +4 -8
  124. package/examples/cli/step-machine-cli/portfolio-tracker/handlers/wait-completed-cli.js +7 -16
  125. package/examples/cli/step-machine-cli/portfolio-tracker/handlers/write-prices-cli.js +2 -6
  126. package/examples/cli/step-machine-cli/portfolio-tracker/handlers-py/_board_pycli.py +13 -3
  127. package/examples/cli/step-machine-cli/portfolio-tracker/handlers-py/add-cards.py +2 -1
  128. package/examples/cli/step-machine-cli/portfolio-tracker/handlers-py/init-board.py +2 -1
  129. package/examples/cli/step-machine-cli/portfolio-tracker/handlers-py/poll-status.py +2 -1
  130. package/examples/cli/step-machine-cli/portfolio-tracker/portfolio-tracker.flow.yaml +20 -24
  131. package/examples/cli/step-machine-cli/portfolio-tracker/run-inline-python-demo-pycli.py +0 -3
  132. package/examples/cli/step-machine-demo/jsonata-init-board-cli.js +8 -13
  133. package/examples/cli/step-machine-demo/jsonata-init-board.flow.yaml +33 -9
  134. package/examples/cli/step-machine-demo/one-step-cli-only.flow.yaml +3 -1
  135. package/examples/cli/step-machine-demo/step2-double-cli.js +6 -12
  136. package/examples/cli/step-machine-demo/two-step-math.flow.yaml +66 -4
  137. package/examples/cli/step-machine-demo/two-step-mixed.flow.yaml +13 -5
  138. package/examples/example-board/agent-instructions.md +1 -1
  139. package/examples/example-board/cards/card-my-identity.json +30 -6
  140. package/examples/example-board/cards/card-portfolio-action.json +24 -6
  141. package/examples/example-board/cards/card-portfolio-intelligence.json +97 -0
  142. package/examples/example-board/cards/card-portfolio-risks.json +24 -6
  143. package/examples/example-board/cards/card-rebalance-impact.json +22 -6
  144. package/examples/example-board/cards/card-rebalance-sim.json +66 -15
  145. package/examples/example-board/cards/cardT-market-prices.json +80 -0
  146. package/examples/example-board/cards/{card-portfolio-value.json → cardT-portfolio-value.json} +38 -10
  147. package/examples/example-board/cards/cardT-portfolio.json +78 -0
  148. package/examples/example-board/demo-server-config.json +1 -1
  149. package/examples/example-board/demo-server.js +383 -69
  150. package/examples/example-board/demo-shell-localstorage.html +774 -0
  151. package/examples/example-board/demo-shell-with-server.html +18 -36
  152. package/examples/example-board/demo-shell.html +5 -4
  153. package/examples/example-board/demo-task-executor.js +213 -265
  154. package/package.json +15 -13
  155. package/step-machine-cli.js +43 -310
  156. package/board-livecards-server-runtime.js +0 -1513
  157. package/browser/board-livecards-runtime-client.js +0 -263
  158. package/dist/pycli/quickjs-board-runtime.global.js +0 -9
  159. package/dist/pycli/quickjs-board-runtime.global.js.map +0 -1
  160. package/dist/pycli/quickjs-step-machine-runtime.global.js +0 -5
  161. package/dist/pycli/quickjs-step-machine-runtime.global.js.map +0 -1
  162. package/examples/cli/step-machine-demo/two-step-math-handlers.js +0 -32
  163. package/examples/cli/step-machine-demo/two-step-mixed-handlers.js +0 -24
  164. package/examples/example-board/cards/card-market-prices.json +0 -56
  165. package/examples/example-board/cards/card-portfolio.json +0 -44
  166. package/examples/example-board/demo-shell-browser.html +0 -675
@@ -58,12 +58,15 @@
58
58
  import fs from 'node:fs';
59
59
  import path from 'node:path';
60
60
  import os from 'node:os';
61
- import crypto from 'node:crypto';
62
61
  import { execFileSync } from 'node:child_process';
63
- import { fileURLToPath } from 'node:url';
62
+ import { fileURLToPath, pathToFileURL } from 'node:url';
64
63
  import { parseRef, blobStorageForRef, reportComplete, reportFailed } from 'yaml-flow/storage-refs';
64
+ import { loadStepFlow, createStepMachine, MemoryStore } from '../../dist/index.js';
65
+ import { buildStepHandlersForFlow } from '../../dist/step-machine-public/index.js';
66
+ import { invokeRefSync } from '../../dist/cli/node/execution-adapter.js';
65
67
 
66
68
  const __dirname = path.dirname(fileURLToPath(import.meta.url));
69
+ const SOURCE_DEF_FLOWS_FILE = path.join(__dirname, 'source_def_flows.json');
67
70
 
68
71
  // ---------------------------------------------------------------------------
69
72
  // Mock data — used when a source has { mock: "key" }.
@@ -83,50 +86,6 @@ const MOCK_DB = {
83
86
  },
84
87
  };
85
88
 
86
- // ---------------------------------------------------------------------------
87
- // Simple file cache for url / url-list results.
88
- // Stored in os.tmpdir()/demo-executor-cache/<hash>.json
89
- // ---------------------------------------------------------------------------
90
- const CACHE_DIR = path.join(os.tmpdir(), 'demo-executor-cache');
91
- const CACHE_TTL_MS = 60 * 60 * 1000; // 1 hour
92
-
93
- function cacheKey(str) {
94
- return crypto.createHash('sha1').update(str).digest('hex');
95
- }
96
-
97
- function readCache(key, ttlMs = CACHE_TTL_MS) {
98
- const file = path.join(CACHE_DIR, `${key}.json`);
99
- try {
100
- const stat = fs.statSync(file);
101
- if (Date.now() - stat.mtimeMs < ttlMs) {
102
- return JSON.parse(fs.readFileSync(file, 'utf-8'));
103
- }
104
- } catch {}
105
- return null;
106
- }
107
-
108
- // Shared single-URL fetch helper used by both url and url-list.
109
- // cacheTimeoutSec: override TTL in seconds (null → use CACHE_TTL_MS default).
110
- function doFetchApi(url, method, headers, cacheTimeoutSec) {
111
- const ttlMs = cacheTimeoutSec != null ? cacheTimeoutSec * 1000 : CACHE_TTL_MS;
112
- const k = cacheKey(`url:${method}:${url}`);
113
- const cached = readCache(k, ttlMs);
114
- if (cached) {
115
- console.warn(`[demo-task-executor] url: cache hit for ${url}`);
116
- return cached;
117
- }
118
- const data = curlFetchJson(url, method, headers);
119
- writeCache(k, data);
120
- return data;
121
- }
122
-
123
- function writeCache(key, value) {
124
- try {
125
- fs.mkdirSync(CACHE_DIR, { recursive: true });
126
- fs.writeFileSync(path.join(CACHE_DIR, `${key}.json`), JSON.stringify(value));
127
- } catch {}
128
- }
129
-
130
89
  function readJson(filePath) {
131
90
  return JSON.parse(fs.readFileSync(filePath, 'utf-8'));
132
91
  }
@@ -160,26 +119,6 @@ const COPILOT_PROMPT_CONTEXT = {
160
119
  ].join('\n'),
161
120
  };
162
121
 
163
- /**
164
- * Fetch a URL using the system curl binary (synchronous, no Node event-loop handles).
165
- * Throws if curl exits non-zero (e.g. HTTP 4xx/5xx with -f, or network error).
166
- */
167
- function curlFetchJson(url, method, headers) {
168
- const bin = process.platform === 'win32' ? 'curl.exe' : 'curl';
169
- // -s : silent (no progress bar)
170
- // -S : show errors despite -s
171
- // -f : fail (non-zero exit) on HTTP 4xx/5xx
172
- // -L : follow redirects
173
- // --max-time 10 : hard timeout
174
- const args = ['-s', '-S', '-f', '-L', '--max-time', '10', '-X', method];
175
- for (const [k, v] of Object.entries(headers)) {
176
- args.push('-H', `${k}: ${v}`);
177
- }
178
- args.push(url);
179
- const raw = execFileSync(bin, args, { encoding: 'utf-8', maxBuffer: 10 * 1024 * 1024 });
180
- return JSON.parse(raw);
181
- }
182
-
183
122
  function resolveCopilotPrompt(sourceDef) {
184
123
  const cfg = sourceDef?.copilot && typeof sourceDef.copilot === 'object' ? sourceDef.copilot : {};
185
124
  const template = cfg.prompt_template ?? sourceDef.prompt_template;
@@ -249,6 +188,7 @@ function runCopilotViaWrapper(prompt, sourceDef, wrapperOutFile, sessionDir, cwd
249
188
  encoding: 'utf-8',
250
189
  stdio: ['ignore', 'pipe', 'pipe'],
251
190
  maxBuffer: 10 * 1024 * 1024,
191
+ windowsHide: true,
252
192
  });
253
193
  } finally {
254
194
  try { fs.unlinkSync(promptFile); } catch {}
@@ -268,7 +208,155 @@ function fail(msg, errFile) {
268
208
  process.exit(1);
269
209
  }
270
210
 
271
- function runSourceFetchSubcommand(argv) {
211
+ function loadSourceDefFlowsConfig() {
212
+ try {
213
+ return readJson(SOURCE_DEF_FLOWS_FILE);
214
+ } catch (err) {
215
+ fail(`Cannot read source flow registry at ${SOURCE_DEF_FLOWS_FILE}: ${String(err && err.message || err)}`);
216
+ }
217
+ }
218
+
219
+ function matchesDetectRule(sourceDef, detect) {
220
+ if (!detect || typeof detect !== 'object') return false;
221
+ if (typeof detect.field === 'string') {
222
+ return sourceDef[detect.field] !== undefined;
223
+ }
224
+ if (Array.isArray(detect.anyOfFields)) {
225
+ return detect.anyOfFields.some((field) => sourceDef[field] !== undefined);
226
+ }
227
+ return false;
228
+ }
229
+
230
+ function resolveSourceKind(sourceDef, registry) {
231
+ const kinds = registry?.kinds && typeof registry.kinds === 'object' ? registry.kinds : {};
232
+ const order = Array.isArray(registry?.resolveOrder) ? registry.resolveOrder : Object.keys(kinds);
233
+ const matched = [];
234
+ for (const kind of order) {
235
+ const spec = kinds[kind];
236
+ if (!spec) continue;
237
+ if (matchesDetectRule(sourceDef, spec.detect)) {
238
+ matched.push(kind);
239
+ }
240
+ }
241
+
242
+ if (matched.length === 0) {
243
+ const knownKinds = Object.keys(kinds);
244
+ throw new Error(`No recognised source kind. Known kinds: ${knownKinds.join(', ')}`);
245
+ }
246
+ if (matched.length > 1) {
247
+ throw new Error(`Multiple source kinds specified: [${matched.join(', ')}]. Use exactly one.`);
248
+ }
249
+ return matched[0];
250
+ }
251
+
252
+ async function executeStepMachineSourceFlow(context) {
253
+ const { kind, registry } = context;
254
+ const spec = registry?.kinds?.[kind];
255
+ if (!spec) {
256
+ throw new Error(`Missing flow registration for kind: ${kind}`);
257
+ }
258
+
259
+ const flowRef = spec.flow;
260
+ if (typeof flowRef !== 'string' || flowRef.length === 0) {
261
+ throw new Error(`Invalid or missing flow for kind: ${kind}`);
262
+ }
263
+
264
+ const flowPath = path.resolve(__dirname, flowRef);
265
+ const flow = await loadStepFlow(flowPath);
266
+
267
+ const invokeHttpRef = async (ref, args) => {
268
+ const rawUrl = typeof ref.whatToRun === 'object' ? ref.whatToRun.value : parseRef(ref.whatToRun).value;
269
+
270
+ const base = String(args?.extra?.serverUrl || 'http://127.0.0.1:7799').replace(/\/$/, '');
271
+ const resolvedUrl = /^https?:\/\//i.test(rawUrl)
272
+ ? rawUrl
273
+ : `${base}${rawUrl.startsWith('/') ? '' : '/'}${rawUrl}`;
274
+
275
+ let body = args;
276
+ const workiqCfg = args?.sourceDef?.workiq;
277
+ if (workiqCfg && typeof workiqCfg === 'object' && typeof workiqCfg.query_template === 'string') {
278
+ const interpolationContext = {
279
+ ...(args?.sourceDef?._projections || {}),
280
+ ...(workiqCfg.args || {}),
281
+ };
282
+ body = {
283
+ query: interpolatePrompt(workiqCfg.query_template, interpolationContext),
284
+ };
285
+ }
286
+
287
+ const method = ref.howToRun === 'http:get' ? 'GET' : 'POST';
288
+ const response = await fetch(resolvedUrl, {
289
+ method,
290
+ headers: { 'Content-Type': 'application/json' },
291
+ ...(method === 'POST' ? { body: JSON.stringify(body) } : {}),
292
+ });
293
+
294
+ const text = await response.text();
295
+ let parsed;
296
+ try {
297
+ parsed = text ? JSON.parse(text) : {};
298
+ } catch {
299
+ parsed = { response: text };
300
+ }
301
+
302
+ if (!response.ok) {
303
+ const msg = typeof parsed?.error === 'string' ? parsed.error : `HTTP ${response.status}`;
304
+ return { result: 'failure', data: { error: msg }, error: msg };
305
+ }
306
+
307
+ if (typeof parsed?.error === 'string') {
308
+ return { result: 'failure', data: { error: parsed.error }, error: parsed.error };
309
+ }
310
+
311
+ return {
312
+ result: 'success',
313
+ data: {
314
+ resultValue: Object.prototype.hasOwnProperty.call(parsed, 'response') ? parsed.response : parsed,
315
+ },
316
+ };
317
+ };
318
+
319
+ const invoke = async (ref, args) => {
320
+ if (ref.howToRun === 'http:post' || ref.howToRun === 'http:get') {
321
+ return invokeHttpRef(ref, args);
322
+ }
323
+ if (ref.howToRun === 'demo-local-module') {
324
+ const whatValue = typeof ref.whatToRun === 'object' ? ref.whatToRun.value : parseRef(ref.whatToRun).value;
325
+ const modulePath = path.resolve(__dirname, whatValue);
326
+ const mod = await import(pathToFileURL(modulePath).href);
327
+ if (typeof mod.execute !== 'function') {
328
+ throw new Error(`Flow module ${JSON.stringify(ref.whatToRun)} must export execute(context)`);
329
+ }
330
+ return mod.execute(args);
331
+ }
332
+ return invokeRefSync(ref, args, { cliDir: __dirname, cwd: process.cwd() });
333
+ };
334
+
335
+ const handlers = buildStepHandlersForFlow(flow, { invoke });
336
+ const machine = createStepMachine(flow, handlers, { store: new MemoryStore() });
337
+ const run = await machine.run({
338
+ ...context,
339
+ promptContext: COPILOT_PROMPT_CONTEXT,
340
+ executorDir: __dirname,
341
+ });
342
+
343
+ if (run.status !== 'completed') {
344
+ const reason = run.error?.message ?? run.intent ?? run.status;
345
+ throw new Error(`flow execution failed: ${reason}`);
346
+ }
347
+
348
+ if (run.intent !== 'success') {
349
+ const reason = typeof run.data?.error === 'string' ? run.data.error : `flow returned intent: ${run.intent}`;
350
+ throw new Error(reason);
351
+ }
352
+
353
+ return {
354
+ resultValue: run.data?.resultValue,
355
+ wroteOutputDirectly: !!run.data?.wroteOutputDirectly,
356
+ };
357
+ }
358
+
359
+ async function runSourceFetchSubcommand(argv) {
272
360
  const inIdx = argv.indexOf('--in-ref');
273
361
  const outIdx = argv.indexOf('--out-ref');
274
362
  const errIdx = argv.indexOf('--err-ref');
@@ -285,7 +373,7 @@ function runSourceFetchSubcommand(argv) {
285
373
  }
286
374
 
287
375
  if (!inRefStr || !outRefStr) {
288
- fail('Usage: run-source-fetch --in-ref <::kind::value> --out-ref <::kind::value> [--err-ref <::kind::value>]');
376
+ fail('Usage: run-source-fetch --in-ref <b64:<base64url(json)>> --out-ref <b64:<base64url(json)>> [--err-ref <b64:<base64url(json)>>]');
289
377
  }
290
378
 
291
379
  let inRef, outRef, errRef;
@@ -330,174 +418,37 @@ function runSourceFetchSubcommand(argv) {
330
418
  failRef(`Cannot resolve source_def: ${String(err && err.message || err)}`, callback);
331
419
  }
332
420
 
333
- let resultValue;
334
-
335
- if (sourceDef['url']) {
336
- // ---------------------------------------------------------------------------
337
- // url single URL fetch via curl
338
- // {{key}} interpolation applied to url from _projections and optional args.
339
- // cacheTimeout: seconds to cache the response (default: CACHE_TTL_MS / 1000).
340
- // ---------------------------------------------------------------------------
341
- const cfg = sourceDef['url'];
342
- const method = (cfg.method || 'GET').toUpperCase();
343
- const headers = { ...(cfg.headers || {}) };
344
- const cacheTimeoutSec = cfg.cacheTimeout != null ? Number(cfg.cacheTimeout) : null;
345
-
346
- const fetchArgs = { ...(cfg.args || {}) };
347
- if (sourceDef.tickersFrom) {
348
- const dotIdx = sourceDef.tickersFrom.indexOf('.');
349
- if (dotIdx > 0) {
350
- const refKey = sourceDef.tickersFrom.slice(0, dotIdx);
351
- const fieldName = sourceDef.tickersFrom.slice(dotIdx + 1);
352
- const arr = sourceDef._projections?.[refKey];
353
- if (Array.isArray(arr)) {
354
- fetchArgs.tickers = arr.map(h => h[fieldName]).filter(Boolean).join(',');
355
- }
356
- }
357
- }
358
- if (sourceDef.tickersFrom && !fetchArgs.tickers) {
359
- failRef('url: tickersFrom resolved to empty list — skipping fetch', callback);
360
- }
361
- const urlContext = { ...(sourceDef._projections || {}), ...fetchArgs };
362
- const url = interpolatePrompt(cfg.url, urlContext);
363
- try {
364
- resultValue = doFetchApi(url, method, headers, cacheTimeoutSec);
365
- } catch (err) {
366
- failRef(`url failed: ${err.message}`, callback);
367
- }
368
-
369
- } else if (sourceDef['url-list']) {
370
- // ---------------------------------------------------------------------------
371
- // url-list — fan-out over a URL list, calling url logic per URL.
372
- // url_list must be a string[] pre-resolved in _projections.url_list.
373
- // cacheTimeout: seconds to cache each individual response.
374
- // ---------------------------------------------------------------------------
375
- const cfg = sourceDef['url-list'];
376
- const method = (cfg.method || 'GET').toUpperCase();
377
- const headers = { ...(cfg.headers || {}) };
378
- const cacheTimeoutSec = cfg.cacheTimeout != null ? Number(cfg.cacheTimeout) : null;
379
-
380
- const urlList = Array.isArray(sourceDef._projections?.url_list)
381
- ? sourceDef._projections.url_list : null;
382
-
383
- if (!urlList || urlList.length === 0) {
384
- failRef('url-list: _projections.url_list must be a non-empty string array', callback);
385
- }
386
-
387
- const results = [];
388
- for (const u of urlList) {
389
- try {
390
- results.push(doFetchApi(u, method, headers, cacheTimeoutSec));
391
- } catch (err) {
392
- failRef(`url-list fetch failed for ${u}: ${err.message}`, callback);
393
- }
394
- }
395
- resultValue = results;
396
-
397
- } else if (sourceDef.copilot || sourceDef.prompt_template) {
398
- const prompt = resolveCopilotPrompt(sourceDef);
399
- if (!prompt) {
400
- failRef('Source definition missing copilot.prompt_template (or prompt_template)', callback);
401
- }
421
+ const registry = loadSourceDefFlowsConfig();
422
+ let kind;
423
+ try {
424
+ kind = resolveSourceKind(sourceDef, registry);
425
+ } catch (err) {
426
+ failRef(String(err && err.message || err), callback);
427
+ }
402
428
 
403
- // Use boardSetupRoot (from --extra) as copilot working directory
404
- const copilotCwd = extra.boardSetupRoot || undefined;
405
-
406
- // On Windows, delegate entirely to copilot_wrapper.bat which handles:
407
- // - session management (--resume UUID for multi-turn continuity)
408
- // - noise/footer stripping, JSON extraction, agentic retry on bad shape
409
- // On non-Windows, fall back to a basic direct invocation (no retry).
410
- const wrapperPath = path.join(__dirname, 'scripts', 'copilot_wrapper.bat');
411
- const useWrapper = process.platform === 'win32' && fs.existsSync(wrapperPath);
412
-
413
- if (useWrapper) {
414
- // Session dir is stable across refreshes so --resume continues the conversation.
415
- const sessionDir = path.join(
416
- extra.boardSetupRoot || os.tmpdir(),
417
- 'copilot-sessions',
418
- String(sourceDef.bindTo || 'default').replace(/[^a-zA-Z0-9_-]/g, '_'),
419
- );
420
- const wrapperOutFile = outRef.value + '.wrapper-out.json';
421
- try {
422
- resultValue = runCopilotViaWrapper(prompt, sourceDef, wrapperOutFile, sessionDir, copilotCwd);
423
- } catch (err) {
424
- failRef(`copilot invocation failed: ${String(err && err.message || err)}`, callback);
425
- } finally {
426
- try { fs.unlinkSync(wrapperOutFile); } catch {}
427
- }
428
- } else {
429
- // Non-Windows fallback: call copilot directly via cmd.exe and do basic JSON extraction.
430
- let rawOutput = '';
431
- try {
432
- rawOutput = execFileSync('cmd.exe', ['/d', '/c', 'copilot --allow-all'], {
433
- input: String(prompt),
434
- encoding: 'utf-8',
435
- stdio: ['pipe', 'pipe', 'pipe'],
436
- maxBuffer: 10 * 1024 * 1024,
437
- ...(copilotCwd ? { cwd: copilotCwd } : {}),
438
- });
439
- } catch (err) {
440
- failRef(`copilot invocation failed: ${String(err && err.message || err)}`, callback);
441
- }
442
- // Basic JSON extraction: find first { or [ in output
443
- const firstBrace = rawOutput.indexOf('{');
444
- const firstBracket = rawOutput.indexOf('[');
445
- const jsonStart = (firstBrace === -1) ? firstBracket
446
- : (firstBracket === -1) ? firstBrace
447
- : Math.min(firstBrace, firstBracket);
448
- if (jsonStart !== -1) {
449
- try {
450
- const parsed = JSON.parse(rawOutput.slice(jsonStart));
451
- resultValue = (parsed && typeof parsed === 'object') ? parsed : rawOutput;
452
- } catch {
453
- resultValue = rawOutput;
454
- }
455
- } else {
456
- resultValue = rawOutput;
457
- }
458
- }
459
- } else if (sourceDef.workiq) {
460
- const cfg = typeof sourceDef.workiq === 'object' ? sourceDef.workiq : {};
461
- if (!cfg.query_template || typeof cfg.query_template !== 'string') {
462
- failRef('Source definition missing workiq.query_template', callback);
463
- }
464
- const interpolationContext = { ...sourceDef._projections, ...(cfg.args ?? {}) };
465
- const query = interpolatePrompt(cfg.query_template, interpolationContext);
429
+ let flowResult;
430
+ try {
431
+ flowResult = await executeStepMachineSourceFlow({
432
+ kind,
433
+ registry,
434
+ sourceDef,
435
+ extra,
436
+ inRef,
437
+ outRef,
438
+ errRef,
439
+ mockDb: MOCK_DB,
440
+ });
441
+ } catch (err) {
442
+ const detail = (err && (err.stderr || err.stdout)) ? `\n${err.stderr || err.stdout}`.trimEnd() : '';
443
+ failRef(`${kind} invocation failed: ${String(err && err.message || err)}${detail}`, callback);
444
+ }
466
445
 
467
- const wrapperPath = path.join(__dirname, 'scripts', 'workiq_wrapper.mjs');
468
- if (!fs.existsSync(wrapperPath)) {
469
- failRef('workiq source kind requires workiq_wrapper.js in scripts/', callback);
470
- }
446
+ if (!flowResult?.wroteOutputDirectly) {
471
447
  try {
472
- execFileSync(process.execPath, [wrapperPath, outRef.value], {
473
- encoding: 'utf-8',
474
- stdio: ['inherit', 'pipe', 'pipe'],
475
- maxBuffer: 10 * 1024 * 1024,
476
- env: {
477
- ...process.env,
478
- WORKIQ_QUERY: query,
479
- ...(extra.serverUrl ? { WORKIQ_SERVER_URL: extra.serverUrl } : {}),
480
- },
481
- });
482
- return; // wrapper wrote directly to out file
448
+ outStorage.write(outRef.value, JSON.stringify(flowResult?.resultValue, null, 2));
483
449
  } catch (err) {
484
- failRef(`workiq invocation failed: ${String(err && err.message || err)}`, callback);
485
- }
486
- } else if (sourceDef.mock) {
487
- // MOCK_DB lookup — data hardcoded at the top of this file
488
- resultValue = MOCK_DB[sourceDef.mock];
489
- if (resultValue === undefined) {
490
- failRef(`Key "${sourceDef.mock}" not found in MOCK_DB`, callback);
450
+ failRef(`Cannot write output: ${String(err && err.message || err)}`, callback);
491
451
  }
492
- } else {
493
- failRef('Source definition has no recognised kind (url, url-list, copilot, workiq, mock)', callback);
494
- }
495
-
496
- // Write result to --out via storage abstraction, then report back to board.
497
- try {
498
- outStorage.write(outRef.value, JSON.stringify(resultValue, null, 2));
499
- } catch (err) {
500
- failRef(`Cannot write output: ${String(err && err.message || err)}`, callback);
501
452
  }
502
453
 
503
454
  if (callback) {
@@ -537,30 +488,16 @@ function validateSourceDefSubcommand(argv) {
537
488
  }
538
489
 
539
490
  const errors = [];
491
+ const registry = loadSourceDefFlowsConfig();
540
492
 
541
- // Determine source kind and validate required fields
542
- const hasUrl = !!sourceDef['url'];
543
- const hasUrlList = !!sourceDef['url-list'];
544
- const hasCopilot = !!sourceDef.copilot;
545
- const hasPromptTemplate = typeof sourceDef.prompt_template === 'string';
546
- const hasWorkiq = !!sourceDef.workiq;
547
- const hasMock = sourceDef.mock !== undefined;
548
-
549
- const kindCount = [hasUrl, hasUrlList, hasCopilot || hasPromptTemplate, hasWorkiq, hasMock].filter(Boolean).length;
550
-
551
- if (kindCount === 0) {
552
- errors.push('No recognised source kind (url, url-list, copilot, workiq, mock). Add one of these fields.');
553
- } else if (kindCount > 1) {
554
- const kinds = [];
555
- if (hasUrl) kinds.push('url');
556
- if (hasUrlList) kinds.push('url-list');
557
- if (hasCopilot || hasPromptTemplate) kinds.push('copilot');
558
- if (hasWorkiq) kinds.push('workiq');
559
- if (hasMock) kinds.push('mock');
560
- errors.push(`Multiple source kinds specified: [${kinds.join(', ')}]. Use exactly one.`);
561
- }
562
-
563
- if (hasUrl) {
493
+ let kind = '';
494
+ try {
495
+ kind = resolveSourceKind(sourceDef, registry);
496
+ } catch (err) {
497
+ errors.push(String(err && err.message || err));
498
+ }
499
+
500
+ if (kind === 'url') {
564
501
  if (typeof sourceDef['url'] !== 'object') {
565
502
  errors.push('url must be an object.');
566
503
  } else if (!sourceDef['url'].url || typeof sourceDef['url'].url !== 'string') {
@@ -568,24 +505,24 @@ function validateSourceDefSubcommand(argv) {
568
505
  }
569
506
  }
570
507
 
571
- if (hasUrlList) {
508
+ if (kind === 'url-list') {
572
509
  if (typeof sourceDef['url-list'] !== 'object') {
573
510
  errors.push('url-list must be an object.');
574
511
  }
575
512
  // url_list is supplied via _projections at runtime — no static validation needed.
576
513
  }
577
514
 
578
- if (hasCopilot) {
515
+ if (kind === 'copilot') {
579
516
  if (typeof sourceDef.copilot !== 'object') {
580
- errors.push('copilot must be an object.');
581
- } else {
582
- if (!sourceDef.copilot.prompt_template && !hasPromptTemplate) {
583
- errors.push('copilot.prompt_template is required (or use top-level prompt_template).');
517
+ if (typeof sourceDef.prompt_template !== 'string') {
518
+ errors.push('copilot must be an object when prompt_template is not provided at top level.');
584
519
  }
520
+ } else if (!sourceDef.copilot.prompt_template && typeof sourceDef.prompt_template !== 'string') {
521
+ errors.push('copilot.prompt_template is required (or use top-level prompt_template).');
585
522
  }
586
523
  }
587
524
 
588
- if (hasWorkiq) {
525
+ if (kind === 'workiq') {
589
526
  if (typeof sourceDef.workiq !== 'object') {
590
527
  errors.push('workiq must be an object.');
591
528
  } else if (!sourceDef.workiq.query_template || typeof sourceDef.workiq.query_template !== 'string') {
@@ -593,7 +530,7 @@ function validateSourceDefSubcommand(argv) {
593
530
  }
594
531
  }
595
532
 
596
- if (hasMock) {
533
+ if (kind === 'mock') {
597
534
  if (typeof sourceDef.mock !== 'string') {
598
535
  errors.push('mock must be a string key.');
599
536
  }
@@ -701,16 +638,27 @@ const CAPABILITIES = {
701
638
  };
702
639
 
703
640
  function describeCapabilities() {
704
- console.log(JSON.stringify(CAPABILITIES, null, 2));
641
+ const registry = loadSourceDefFlowsConfig();
642
+ const merged = {
643
+ ...CAPABILITIES,
644
+ sourceKinds: Object.fromEntries(
645
+ Object.entries(registry?.kinds || {}).map(([kind, spec]) => {
646
+ const existing = CAPABILITIES.sourceKinds[kind] || {};
647
+ const manifest = spec?.manifest && typeof spec.manifest === 'object' ? spec.manifest : {};
648
+ return [kind, { ...existing, ...manifest }];
649
+ }),
650
+ ),
651
+ };
652
+ console.log(JSON.stringify(merged, null, 2));
705
653
  }
706
654
 
707
655
  async function main() {
708
656
  const sub = process.argv[2];
709
657
  if (sub === 'run-source-fetch') {
710
- runSourceFetchSubcommand(process.argv.slice(3));
658
+ await runSourceFetchSubcommand(process.argv.slice(3));
711
659
  return;
712
660
  }
713
- if (sub === 'describe-capabilities') {
661
+ if (sub === 'describe' || sub === 'describe-capabilities') {
714
662
  describeCapabilities();
715
663
  return;
716
664
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "yaml-flow",
3
- "version": "6.0.0",
3
+ "version": "7.1.0",
4
4
  "description": "Unified workflow engine: step-machine (sequential) + event-graph (stateless DAG) with pluggable storage",
5
5
  "author": "",
6
6
  "license": "MIT",
@@ -106,7 +106,10 @@
106
106
  "types": "./dist/cli/browser-api/card-store-browser-api.d.ts",
107
107
  "import": "./dist/cli/browser-api/card-store-browser-api.js"
108
108
  },
109
- "./board-livecards-server-runtime": "./board-livecards-server-runtime.js",
109
+ "./server-runtime": {
110
+ "types": "./dist/server-runtime/index.d.ts",
111
+ "import": "./dist/server-runtime/index.js"
112
+ },
110
113
  "./package.json": "./package.json"
111
114
  },
112
115
  "browser": {
@@ -116,7 +119,6 @@
116
119
  "board-live-cards-cli.js",
117
120
  "storage-refs.js",
118
121
  "step-machine-cli.js",
119
- "board-livecards-server-runtime.js",
120
122
  "dist",
121
123
  "schema",
122
124
  "browser",
@@ -125,32 +127,33 @@
125
127
  "scripts": {
126
128
  "build": "tsup",
127
129
  "build:browser": "tsup --config tsup.browser.config.ts && node scripts/generate-browser-integrity.mjs",
128
- "build:quickjs": "tsup --config tsup.quickjs.config.ts",
129
130
  "check:bundle-budget": "node scripts/check-bundle-budgets.mjs",
130
131
  "check:browser-bundle": "node scripts/check-browser-bundle-safety.mjs",
131
- "release:gate": "npm run -s build && npm run -s build:browser && npm run -s build:quickjs && npm run -s test:run && npm run -s check:bundle-budget && npm run -s check:browser-bundle",
132
- "standalone": "npm run -s test:cli-parity && npm run -s build && npm run -s build:quickjs && node scripts/build-standalone.mjs",
132
+ "release:gate": "npm run -s build && npm run -s build:browser && npm run -s test:run && npm run -s check:bundle-budget && npm run -s check:browser-bundle",
133
+ "standalone": "npm run -s test:cli-parity && npm run -s build && node scripts/build-standalone.mjs",
133
134
  "pycli:install": "python -m pip install -r pycli/requirements.txt",
134
135
  "pycli:install:venv": "c:/Users/sreenaga/ADO/ai-tool-evolver/.venv/Scripts/python.exe -m pip install -r pycli/requirements.txt",
135
136
  "pycli:install:py312": "py -3.12 -m pip install -r pycli/requirements.txt",
136
137
  "pycli:install:venv312": "./.venv312/Scripts/python.exe -m pip install -r pycli/requirements.txt",
137
138
  "build:example": "node demo-src/example-board/build.js",
138
139
  "dev": "tsup --watch",
139
- "test": "vitest",
140
- "test:run": "vitest run",
141
- "test:cli-parity": "npm run -s build:quickjs && python scripts/cli_parity_check.py",
142
- "test:cli-parity:full": "npm run -s build:quickjs && python scripts/cli_parity_check.py --include-portfolio",
143
- "test:e2e": "npm run -s build:quickjs && .\\.venv312\\Scripts\\python.exe examples/browser/boards/portfolio-tracker/portfolio-tracker.py --run-pycli",
140
+ "test": "npm run build:example && vitest",
141
+ "test:run": "npm run build && npm run build:example && vitest run",
142
+ "test:cli-parity": "python scripts/cli_parity_check.py",
143
+ "test:cli-parity:full": "python scripts/cli_parity_check.py --include-portfolio",
144
+ "test:e2e": "python examples/browser/boards/portfolio-tracker/portfolio-tracker.py --run-pycli",
145
+ "test:portfolio-tracker-server": "node examples/browser/boards/portfolio-tracker/portfolio-tracker-http-test.js",
144
146
  "test:example-board": "vitest run tests/examples/example-board.test.ts",
145
147
  "check:example-board": "npm run test:example-board && npm run validate:cards -- \"examples/example-board/cards/*.json\"",
146
148
  "validate:cards": "tsx scripts/validate-live-cards.ts",
147
149
  "lint": "eslint src/",
148
150
  "typecheck": "tsc --noEmit",
149
- "prepublishOnly": "npm run build && npm run build:browser && npm run build:quickjs && npm run check:bundle-budget && npm run check:browser-bundle"
151
+ "prepublishOnly": "npm run build && npm run build:browser && npm run check:bundle-budget && npm run check:browser-bundle"
150
152
  },
151
153
  "devDependencies": {
152
154
  "@types/node": "^20.10.0",
153
155
  "@types/proper-lockfile": "^4.1.4",
156
+ "jsdom": "^24.0.0",
154
157
  "tsup": "^8.0.0",
155
158
  "tsx": "^4.21.0",
156
159
  "typescript": "^5.3.0",
@@ -159,7 +162,6 @@
159
162
  "dependencies": {
160
163
  "ajv-formats": "^3.0.1",
161
164
  "fast-glob": "^3.3.3",
162
- "jsonata": "^2.1.0",
163
165
  "proper-lockfile": "^4.1.2",
164
166
  "yaml": "^2.3.4"
165
167
  },