yaml-flow 6.0.0 → 7.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (162) hide show
  1. package/board-live-cards-cli.js +4 -4
  2. package/browser/asset-integrity.json +3 -3
  3. package/browser/board-livecards-client.js +2 -0
  4. package/browser/board-livecards-client.js.map +1 -0
  5. package/browser/board-livecards-localstorage.js +10 -0
  6. package/browser/board-livecards-localstorage.js.map +1 -0
  7. package/browser/board-livegraph-engine.js +2 -2
  8. package/browser/board-livegraph-engine.js.map +1 -1
  9. package/browser/card-compute.js +28 -28
  10. package/browser/compute-jsonata.js +5 -0
  11. package/browser/compute-jsonata.js.map +1 -0
  12. package/browser/live-cards.js +261 -150
  13. package/card-store.js +4 -4
  14. package/dist/{board-live-cards-public-CltXYgaY.d.cts → board-live-cards-public-CW5074xr.d.cts} +9 -5
  15. package/dist/{board-live-cards-public-f-E-FAyp.d.ts → board-live-cards-public-hnZo0mAf.d.ts} +9 -5
  16. package/dist/board-livegraph-runtime/index.cjs +2 -2
  17. package/dist/board-livegraph-runtime/index.cjs.map +1 -1
  18. package/dist/board-livegraph-runtime/index.d.cts +11 -9
  19. package/dist/board-livegraph-runtime/index.d.ts +11 -9
  20. package/dist/board-livegraph-runtime/index.js +2 -2
  21. package/dist/board-livegraph-runtime/index.js.map +1 -1
  22. package/dist/board-livegraph-runtime/jsonata-sync.cjs +37 -1
  23. package/dist/card-compute/index.cjs +4 -4
  24. package/dist/card-compute/index.cjs.map +1 -1
  25. package/dist/card-compute/index.d.cts +5 -1
  26. package/dist/card-compute/index.d.ts +5 -1
  27. package/dist/card-compute/index.js +4 -4
  28. package/dist/card-compute/index.js.map +1 -1
  29. package/dist/card-compute/jsonata-sync.cjs +37 -1
  30. package/dist/cli/browser-api/board-live-cards-browser-adapter.cjs +2 -1
  31. package/dist/cli/browser-api/board-live-cards-browser-adapter.cjs.map +1 -1
  32. package/dist/cli/browser-api/board-live-cards-browser-adapter.d.cts +27 -14
  33. package/dist/cli/browser-api/board-live-cards-browser-adapter.d.ts +27 -14
  34. package/dist/cli/browser-api/board-live-cards-browser-adapter.js +2 -1
  35. package/dist/cli/browser-api/board-live-cards-browser-adapter.js.map +1 -1
  36. package/dist/cli/browser-api/card-store-browser-api.cjs +1 -1
  37. package/dist/cli/browser-api/card-store-browser-api.cjs.map +1 -1
  38. package/dist/cli/browser-api/card-store-browser-api.js +1 -1
  39. package/dist/cli/browser-api/card-store-browser-api.js.map +1 -1
  40. package/dist/cli/browser-api/jsonata-sync.cjs +37 -1
  41. package/dist/cli/node/artifacts-store-cli.cjs +8 -8
  42. package/dist/cli/node/artifacts-store-cli.cjs.map +1 -1
  43. package/dist/cli/node/artifacts-store-cli.js +8 -8
  44. package/dist/cli/node/artifacts-store-cli.js.map +1 -1
  45. package/dist/cli/node/board-live-cards-cli.cjs +7 -7
  46. package/dist/cli/node/board-live-cards-cli.cjs.map +1 -1
  47. package/dist/cli/node/board-live-cards-cli.js +7 -7
  48. package/dist/cli/node/board-live-cards-cli.js.map +1 -1
  49. package/dist/cli/node/card-store-cli.cjs +5 -5
  50. package/dist/cli/node/card-store-cli.cjs.map +1 -1
  51. package/dist/cli/node/card-store-cli.js +5 -5
  52. package/dist/cli/node/card-store-cli.js.map +1 -1
  53. package/dist/cli/node/execution-adapter.cjs +3 -0
  54. package/dist/cli/node/execution-adapter.cjs.map +1 -0
  55. package/dist/cli/node/execution-adapter.d.cts +174 -0
  56. package/dist/cli/node/execution-adapter.d.ts +174 -0
  57. package/dist/cli/node/execution-adapter.js +3 -0
  58. package/dist/cli/node/execution-adapter.js.map +1 -0
  59. package/dist/cli/node/fs-board-adapter.cjs +7 -7
  60. package/dist/cli/node/fs-board-adapter.cjs.map +1 -1
  61. package/dist/cli/node/fs-board-adapter.d.cts +2 -2
  62. package/dist/cli/node/fs-board-adapter.d.ts +2 -2
  63. package/dist/cli/node/fs-board-adapter.js +7 -7
  64. package/dist/cli/node/fs-board-adapter.js.map +1 -1
  65. package/dist/cli/node/jsonata-sync.cjs +37 -1
  66. package/dist/cli/node/source-cli-task-executor.cjs +4 -4
  67. package/dist/cli/node/source-cli-task-executor.cjs.map +1 -1
  68. package/dist/cli/node/source-cli-task-executor.js +4 -4
  69. package/dist/cli/node/source-cli-task-executor.js.map +1 -1
  70. package/dist/continuous-event-graph/index.cjs +2 -2
  71. package/dist/continuous-event-graph/index.cjs.map +1 -1
  72. package/dist/continuous-event-graph/index.js +2 -2
  73. package/dist/continuous-event-graph/index.js.map +1 -1
  74. package/dist/continuous-event-graph/jsonata-sync.cjs +37 -1
  75. package/dist/execution-refs.cjs +2 -1
  76. package/dist/execution-refs.cjs.map +1 -1
  77. package/dist/execution-refs.d.cts +49 -11
  78. package/dist/execution-refs.d.ts +49 -11
  79. package/dist/execution-refs.js +2 -1
  80. package/dist/execution-refs.js.map +1 -1
  81. package/dist/index.cjs +10 -10
  82. package/dist/index.cjs.map +1 -1
  83. package/dist/index.js +10 -10
  84. package/dist/index.js.map +1 -1
  85. package/dist/jsonata-sync.cjs +37 -1
  86. package/dist/server-runtime/index.cjs +9 -0
  87. package/dist/server-runtime/index.cjs.map +1 -0
  88. package/dist/server-runtime/index.d.cts +31 -0
  89. package/dist/server-runtime/index.d.ts +31 -0
  90. package/dist/server-runtime/index.js +9 -0
  91. package/dist/server-runtime/index.js.map +1 -0
  92. package/dist/server-runtime/jsonata-sync.cjs +7623 -0
  93. package/dist/step-machine-public/index.cjs +2 -0
  94. package/dist/step-machine-public/index.cjs.map +1 -0
  95. package/dist/step-machine-public/index.d.cts +159 -0
  96. package/dist/step-machine-public/index.d.ts +159 -0
  97. package/dist/step-machine-public/index.js +2 -0
  98. package/dist/step-machine-public/index.js.map +1 -0
  99. package/dist/step-machine-public/jsonata-sync.cjs +7623 -0
  100. package/dist/storage-refs.cjs +2 -2
  101. package/dist/storage-refs.cjs.map +1 -1
  102. package/dist/storage-refs.d.cts +7 -6
  103. package/dist/storage-refs.d.ts +7 -6
  104. package/dist/storage-refs.js +2 -2
  105. package/dist/storage-refs.js.map +1 -1
  106. package/dist/types-B1ZRa4aI.d.ts +147 -0
  107. package/dist/types-BxEFcVK9.d.cts +147 -0
  108. package/examples/browser/boards/portfolio-tracker/portfolio-t4.js +9 -10
  109. package/examples/browser/boards/portfolio-tracker/portfolio-tracker-http-test.js +357 -0
  110. package/examples/browser/boards/portfolio-tracker/portfolio-tracker-public.js +9 -10
  111. package/examples/browser/boards/portfolio-tracker/portfolio-tracker-server.js +300 -0
  112. package/examples/browser/boards/portfolio-tracker/portfolio-tracker-server.py +617 -0
  113. package/examples/browser/boards/portfolio-tracker/portfolio-tracker-sse-worker.js +48 -0
  114. package/examples/browser/boards/portfolio-tracker/portfolio-tracker.py +11 -10
  115. package/examples/cli/step-machine-cli/portfolio-tracker/handlers/_board-cli.js +19 -4
  116. package/examples/cli/step-machine-cli/portfolio-tracker/handlers/add-cards-cli.js +4 -8
  117. package/examples/cli/step-machine-cli/portfolio-tracker/handlers/init-board-cli.js +6 -10
  118. package/examples/cli/step-machine-cli/portfolio-tracker/handlers/poll-status-cli.js +8 -16
  119. package/examples/cli/step-machine-cli/portfolio-tracker/handlers/reset-board-dir-cli.js +2 -6
  120. package/examples/cli/step-machine-cli/portfolio-tracker/handlers/retrigger-cli.js +4 -8
  121. package/examples/cli/step-machine-cli/portfolio-tracker/handlers/status-cli.js +3 -7
  122. package/examples/cli/step-machine-cli/portfolio-tracker/handlers/update-holdings-cli.js +4 -8
  123. package/examples/cli/step-machine-cli/portfolio-tracker/handlers/wait-completed-cli.js +7 -16
  124. package/examples/cli/step-machine-cli/portfolio-tracker/handlers/write-prices-cli.js +2 -6
  125. package/examples/cli/step-machine-cli/portfolio-tracker/handlers-py/_board_pycli.py +13 -3
  126. package/examples/cli/step-machine-cli/portfolio-tracker/handlers-py/add-cards.py +2 -1
  127. package/examples/cli/step-machine-cli/portfolio-tracker/handlers-py/init-board.py +2 -1
  128. package/examples/cli/step-machine-cli/portfolio-tracker/handlers-py/poll-status.py +2 -1
  129. package/examples/cli/step-machine-cli/portfolio-tracker/portfolio-tracker.flow.yaml +20 -24
  130. package/examples/cli/step-machine-cli/portfolio-tracker/run-inline-python-demo-pycli.py +0 -3
  131. package/examples/cli/step-machine-demo/jsonata-init-board-cli.js +8 -13
  132. package/examples/cli/step-machine-demo/jsonata-init-board.flow.yaml +33 -9
  133. package/examples/cli/step-machine-demo/one-step-cli-only.flow.yaml +3 -1
  134. package/examples/cli/step-machine-demo/step2-double-cli.js +6 -12
  135. package/examples/cli/step-machine-demo/two-step-math.flow.yaml +66 -4
  136. package/examples/cli/step-machine-demo/two-step-mixed.flow.yaml +13 -5
  137. package/examples/example-board/cards/_index.json +47 -0
  138. package/examples/example-board/cards/card-market-prices.json +33 -9
  139. package/examples/example-board/cards/card-my-identity.json +30 -6
  140. package/examples/example-board/cards/card-portfolio-action.json +24 -6
  141. package/examples/example-board/cards/card-portfolio-intelligence.json +97 -0
  142. package/examples/example-board/cards/card-portfolio-risks.json +24 -6
  143. package/examples/example-board/cards/card-portfolio-value.json +38 -10
  144. package/examples/example-board/cards/card-portfolio.json +57 -13
  145. package/examples/example-board/cards/card-rebalance-impact.json +22 -6
  146. package/examples/example-board/cards/card-rebalance-sim.json +66 -15
  147. package/examples/example-board/demo-server.js +360 -69
  148. package/examples/example-board/demo-shell-localstorage.html +774 -0
  149. package/examples/example-board/demo-shell-with-server.html +18 -36
  150. package/examples/example-board/demo-shell.html +5 -4
  151. package/examples/example-board/demo-task-executor.js +217 -265
  152. package/package.json +15 -13
  153. package/step-machine-cli.js +43 -310
  154. package/board-livecards-server-runtime.js +0 -1513
  155. package/browser/board-livecards-runtime-client.js +0 -263
  156. package/dist/pycli/quickjs-board-runtime.global.js +0 -9
  157. package/dist/pycli/quickjs-board-runtime.global.js.map +0 -1
  158. package/dist/pycli/quickjs-step-machine-runtime.global.js +0 -5
  159. package/dist/pycli/quickjs-step-machine-runtime.global.js.map +0 -1
  160. package/examples/cli/step-machine-demo/two-step-math-handlers.js +0 -32
  161. package/examples/cli/step-machine-demo/two-step-mixed-handlers.js +0 -24
  162. package/examples/example-board/demo-shell-browser.html +0 -675
@@ -58,12 +58,15 @@
58
58
  import fs from 'node:fs';
59
59
  import path from 'node:path';
60
60
  import os from 'node:os';
61
- import crypto from 'node:crypto';
62
61
  import { execFileSync } from 'node:child_process';
63
- import { fileURLToPath } from 'node:url';
62
+ import { fileURLToPath, pathToFileURL } from 'node:url';
64
63
  import { parseRef, blobStorageForRef, reportComplete, reportFailed } from 'yaml-flow/storage-refs';
64
+ import { loadStepFlow, createStepMachine, MemoryStore } from '../../dist/index.js';
65
+ import { buildStepHandlersForFlow } from '../../dist/step-machine-public/index.js';
66
+ import { invokeRefSync } from '../../dist/cli/node/execution-adapter.js';
65
67
 
66
68
  const __dirname = path.dirname(fileURLToPath(import.meta.url));
69
+ const SOURCE_DEF_FLOWS_FILE = path.join(__dirname, 'source_def_flows.json');
67
70
 
68
71
  // ---------------------------------------------------------------------------
69
72
  // Mock data — used when a source has { mock: "key" }.
@@ -83,50 +86,6 @@ const MOCK_DB = {
83
86
  },
84
87
  };
85
88
 
86
- // ---------------------------------------------------------------------------
87
- // Simple file cache for url / url-list results.
88
- // Stored in os.tmpdir()/demo-executor-cache/<hash>.json
89
- // ---------------------------------------------------------------------------
90
- const CACHE_DIR = path.join(os.tmpdir(), 'demo-executor-cache');
91
- const CACHE_TTL_MS = 60 * 60 * 1000; // 1 hour
92
-
93
- function cacheKey(str) {
94
- return crypto.createHash('sha1').update(str).digest('hex');
95
- }
96
-
97
- function readCache(key, ttlMs = CACHE_TTL_MS) {
98
- const file = path.join(CACHE_DIR, `${key}.json`);
99
- try {
100
- const stat = fs.statSync(file);
101
- if (Date.now() - stat.mtimeMs < ttlMs) {
102
- return JSON.parse(fs.readFileSync(file, 'utf-8'));
103
- }
104
- } catch {}
105
- return null;
106
- }
107
-
108
- // Shared single-URL fetch helper used by both url and url-list.
109
- // cacheTimeoutSec: override TTL in seconds (null → use CACHE_TTL_MS default).
110
- function doFetchApi(url, method, headers, cacheTimeoutSec) {
111
- const ttlMs = cacheTimeoutSec != null ? cacheTimeoutSec * 1000 : CACHE_TTL_MS;
112
- const k = cacheKey(`url:${method}:${url}`);
113
- const cached = readCache(k, ttlMs);
114
- if (cached) {
115
- console.warn(`[demo-task-executor] url: cache hit for ${url}`);
116
- return cached;
117
- }
118
- const data = curlFetchJson(url, method, headers);
119
- writeCache(k, data);
120
- return data;
121
- }
122
-
123
- function writeCache(key, value) {
124
- try {
125
- fs.mkdirSync(CACHE_DIR, { recursive: true });
126
- fs.writeFileSync(path.join(CACHE_DIR, `${key}.json`), JSON.stringify(value));
127
- } catch {}
128
- }
129
-
130
89
  function readJson(filePath) {
131
90
  return JSON.parse(fs.readFileSync(filePath, 'utf-8'));
132
91
  }
@@ -160,26 +119,6 @@ const COPILOT_PROMPT_CONTEXT = {
160
119
  ].join('\n'),
161
120
  };
162
121
 
163
- /**
164
- * Fetch a URL using the system curl binary (synchronous, no Node event-loop handles).
165
- * Throws if curl exits non-zero (e.g. HTTP 4xx/5xx with -f, or network error).
166
- */
167
- function curlFetchJson(url, method, headers) {
168
- const bin = process.platform === 'win32' ? 'curl.exe' : 'curl';
169
- // -s : silent (no progress bar)
170
- // -S : show errors despite -s
171
- // -f : fail (non-zero exit) on HTTP 4xx/5xx
172
- // -L : follow redirects
173
- // --max-time 10 : hard timeout
174
- const args = ['-s', '-S', '-f', '-L', '--max-time', '10', '-X', method];
175
- for (const [k, v] of Object.entries(headers)) {
176
- args.push('-H', `${k}: ${v}`);
177
- }
178
- args.push(url);
179
- const raw = execFileSync(bin, args, { encoding: 'utf-8', maxBuffer: 10 * 1024 * 1024 });
180
- return JSON.parse(raw);
181
- }
182
-
183
122
  function resolveCopilotPrompt(sourceDef) {
184
123
  const cfg = sourceDef?.copilot && typeof sourceDef.copilot === 'object' ? sourceDef.copilot : {};
185
124
  const template = cfg.prompt_template ?? sourceDef.prompt_template;
@@ -249,6 +188,7 @@ function runCopilotViaWrapper(prompt, sourceDef, wrapperOutFile, sessionDir, cwd
249
188
  encoding: 'utf-8',
250
189
  stdio: ['ignore', 'pipe', 'pipe'],
251
190
  maxBuffer: 10 * 1024 * 1024,
191
+ windowsHide: true,
252
192
  });
253
193
  } finally {
254
194
  try { fs.unlinkSync(promptFile); } catch {}
@@ -268,7 +208,159 @@ function fail(msg, errFile) {
268
208
  process.exit(1);
269
209
  }
270
210
 
271
- function runSourceFetchSubcommand(argv) {
211
+ function loadSourceDefFlowsConfig() {
212
+ try {
213
+ return readJson(SOURCE_DEF_FLOWS_FILE);
214
+ } catch (err) {
215
+ fail(`Cannot read source flow registry at ${SOURCE_DEF_FLOWS_FILE}: ${String(err && err.message || err)}`);
216
+ }
217
+ }
218
+
219
+ function matchesDetectRule(sourceDef, detect) {
220
+ if (!detect || typeof detect !== 'object') return false;
221
+ if (typeof detect.field === 'string') {
222
+ return sourceDef[detect.field] !== undefined;
223
+ }
224
+ if (Array.isArray(detect.anyOfFields)) {
225
+ return detect.anyOfFields.some((field) => sourceDef[field] !== undefined);
226
+ }
227
+ return false;
228
+ }
229
+
230
+ function resolveSourceKind(sourceDef, registry) {
231
+ const kinds = registry?.kinds && typeof registry.kinds === 'object' ? registry.kinds : {};
232
+ const order = Array.isArray(registry?.resolveOrder) ? registry.resolveOrder : Object.keys(kinds);
233
+ const matched = [];
234
+ for (const kind of order) {
235
+ const spec = kinds[kind];
236
+ if (!spec) continue;
237
+ if (matchesDetectRule(sourceDef, spec.detect)) {
238
+ matched.push(kind);
239
+ }
240
+ }
241
+
242
+ if (matched.length === 0) {
243
+ const knownKinds = Object.keys(kinds);
244
+ throw new Error(`No recognised source kind. Known kinds: ${knownKinds.join(', ')}`);
245
+ }
246
+ if (matched.length > 1) {
247
+ throw new Error(`Multiple source kinds specified: [${matched.join(', ')}]. Use exactly one.`);
248
+ }
249
+ return matched[0];
250
+ }
251
+
252
+ async function executeStepMachineSourceFlow(context) {
253
+ const { kind, registry } = context;
254
+ const spec = registry?.kinds?.[kind];
255
+ if (!spec) {
256
+ throw new Error(`Missing flow registration for kind: ${kind}`);
257
+ }
258
+
259
+ const flowRef = spec.flow;
260
+ if (typeof flowRef !== 'string' || flowRef.length === 0) {
261
+ throw new Error(`Invalid or missing flow for kind: ${kind}`);
262
+ }
263
+
264
+ const flowPath = path.resolve(__dirname, flowRef);
265
+ const flow = await loadStepFlow(flowPath);
266
+
267
+ const invokeHttpRef = async (ref, args) => {
268
+ let rawUrl = ref.whatToRun;
269
+ try {
270
+ rawUrl = parseRef(ref.whatToRun).value;
271
+ } catch {
272
+ // Keep raw value when whatToRun is already a URL.
273
+ }
274
+
275
+ const base = String(args?.extra?.serverUrl || 'http://127.0.0.1:7799').replace(/\/$/, '');
276
+ const resolvedUrl = /^https?:\/\//i.test(rawUrl)
277
+ ? rawUrl
278
+ : `${base}${rawUrl.startsWith('/') ? '' : '/'}${rawUrl}`;
279
+
280
+ let body = args;
281
+ const workiqCfg = args?.sourceDef?.workiq;
282
+ if (workiqCfg && typeof workiqCfg === 'object' && typeof workiqCfg.query_template === 'string') {
283
+ const interpolationContext = {
284
+ ...(args?.sourceDef?._projections || {}),
285
+ ...(workiqCfg.args || {}),
286
+ };
287
+ body = {
288
+ query: interpolatePrompt(workiqCfg.query_template, interpolationContext),
289
+ };
290
+ }
291
+
292
+ const method = ref.howToRun === 'http:get' ? 'GET' : 'POST';
293
+ const response = await fetch(resolvedUrl, {
294
+ method,
295
+ headers: { 'Content-Type': 'application/json' },
296
+ ...(method === 'POST' ? { body: JSON.stringify(body) } : {}),
297
+ });
298
+
299
+ const text = await response.text();
300
+ let parsed;
301
+ try {
302
+ parsed = text ? JSON.parse(text) : {};
303
+ } catch {
304
+ parsed = { response: text };
305
+ }
306
+
307
+ if (!response.ok) {
308
+ const msg = typeof parsed?.error === 'string' ? parsed.error : `HTTP ${response.status}`;
309
+ return { result: 'failure', data: { error: msg }, error: msg };
310
+ }
311
+
312
+ if (typeof parsed?.error === 'string') {
313
+ return { result: 'failure', data: { error: parsed.error }, error: parsed.error };
314
+ }
315
+
316
+ return {
317
+ result: 'success',
318
+ data: {
319
+ resultValue: Object.prototype.hasOwnProperty.call(parsed, 'response') ? parsed.response : parsed,
320
+ },
321
+ };
322
+ };
323
+
324
+ const invoke = async (ref, args) => {
325
+ if (ref.howToRun === 'http:post' || ref.howToRun === 'http:get') {
326
+ return invokeHttpRef(ref, args);
327
+ }
328
+ if (ref.howToRun === 'demo-local-module') {
329
+ const modulePath = path.resolve(__dirname, ref.whatToRun);
330
+ const mod = await import(pathToFileURL(modulePath).href);
331
+ if (typeof mod.execute !== 'function') {
332
+ throw new Error(`Flow module ${ref.whatToRun} must export execute(context)`);
333
+ }
334
+ return mod.execute(args);
335
+ }
336
+ return invokeRefSync(ref, args, { cliDir: __dirname, cwd: process.cwd() });
337
+ };
338
+
339
+ const handlers = buildStepHandlersForFlow(flow, { invoke });
340
+ const machine = createStepMachine(flow, handlers, { store: new MemoryStore() });
341
+ const run = await machine.run({
342
+ ...context,
343
+ promptContext: COPILOT_PROMPT_CONTEXT,
344
+ executorDir: __dirname,
345
+ });
346
+
347
+ if (run.status !== 'completed') {
348
+ const reason = run.error?.message ?? run.intent ?? run.status;
349
+ throw new Error(`flow execution failed: ${reason}`);
350
+ }
351
+
352
+ if (run.intent !== 'success') {
353
+ const reason = typeof run.data?.error === 'string' ? run.data.error : `flow returned intent: ${run.intent}`;
354
+ throw new Error(reason);
355
+ }
356
+
357
+ return {
358
+ resultValue: run.data?.resultValue,
359
+ wroteOutputDirectly: !!run.data?.wroteOutputDirectly,
360
+ };
361
+ }
362
+
363
+ async function runSourceFetchSubcommand(argv) {
272
364
  const inIdx = argv.indexOf('--in-ref');
273
365
  const outIdx = argv.indexOf('--out-ref');
274
366
  const errIdx = argv.indexOf('--err-ref');
@@ -285,7 +377,7 @@ function runSourceFetchSubcommand(argv) {
285
377
  }
286
378
 
287
379
  if (!inRefStr || !outRefStr) {
288
- fail('Usage: run-source-fetch --in-ref <::kind::value> --out-ref <::kind::value> [--err-ref <::kind::value>]');
380
+ fail('Usage: run-source-fetch --in-ref <b64:<base64url(json)>> --out-ref <b64:<base64url(json)>> [--err-ref <b64:<base64url(json)>>]');
289
381
  }
290
382
 
291
383
  let inRef, outRef, errRef;
@@ -330,174 +422,37 @@ function runSourceFetchSubcommand(argv) {
330
422
  failRef(`Cannot resolve source_def: ${String(err && err.message || err)}`, callback);
331
423
  }
332
424
 
333
- let resultValue;
334
-
335
- if (sourceDef['url']) {
336
- // ---------------------------------------------------------------------------
337
- // url single URL fetch via curl
338
- // {{key}} interpolation applied to url from _projections and optional args.
339
- // cacheTimeout: seconds to cache the response (default: CACHE_TTL_MS / 1000).
340
- // ---------------------------------------------------------------------------
341
- const cfg = sourceDef['url'];
342
- const method = (cfg.method || 'GET').toUpperCase();
343
- const headers = { ...(cfg.headers || {}) };
344
- const cacheTimeoutSec = cfg.cacheTimeout != null ? Number(cfg.cacheTimeout) : null;
345
-
346
- const fetchArgs = { ...(cfg.args || {}) };
347
- if (sourceDef.tickersFrom) {
348
- const dotIdx = sourceDef.tickersFrom.indexOf('.');
349
- if (dotIdx > 0) {
350
- const refKey = sourceDef.tickersFrom.slice(0, dotIdx);
351
- const fieldName = sourceDef.tickersFrom.slice(dotIdx + 1);
352
- const arr = sourceDef._projections?.[refKey];
353
- if (Array.isArray(arr)) {
354
- fetchArgs.tickers = arr.map(h => h[fieldName]).filter(Boolean).join(',');
355
- }
356
- }
357
- }
358
- if (sourceDef.tickersFrom && !fetchArgs.tickers) {
359
- failRef('url: tickersFrom resolved to empty list — skipping fetch', callback);
360
- }
361
- const urlContext = { ...(sourceDef._projections || {}), ...fetchArgs };
362
- const url = interpolatePrompt(cfg.url, urlContext);
363
- try {
364
- resultValue = doFetchApi(url, method, headers, cacheTimeoutSec);
365
- } catch (err) {
366
- failRef(`url failed: ${err.message}`, callback);
367
- }
368
-
369
- } else if (sourceDef['url-list']) {
370
- // ---------------------------------------------------------------------------
371
- // url-list — fan-out over a URL list, calling url logic per URL.
372
- // url_list must be a string[] pre-resolved in _projections.url_list.
373
- // cacheTimeout: seconds to cache each individual response.
374
- // ---------------------------------------------------------------------------
375
- const cfg = sourceDef['url-list'];
376
- const method = (cfg.method || 'GET').toUpperCase();
377
- const headers = { ...(cfg.headers || {}) };
378
- const cacheTimeoutSec = cfg.cacheTimeout != null ? Number(cfg.cacheTimeout) : null;
379
-
380
- const urlList = Array.isArray(sourceDef._projections?.url_list)
381
- ? sourceDef._projections.url_list : null;
382
-
383
- if (!urlList || urlList.length === 0) {
384
- failRef('url-list: _projections.url_list must be a non-empty string array', callback);
385
- }
386
-
387
- const results = [];
388
- for (const u of urlList) {
389
- try {
390
- results.push(doFetchApi(u, method, headers, cacheTimeoutSec));
391
- } catch (err) {
392
- failRef(`url-list fetch failed for ${u}: ${err.message}`, callback);
393
- }
394
- }
395
- resultValue = results;
396
-
397
- } else if (sourceDef.copilot || sourceDef.prompt_template) {
398
- const prompt = resolveCopilotPrompt(sourceDef);
399
- if (!prompt) {
400
- failRef('Source definition missing copilot.prompt_template (or prompt_template)', callback);
401
- }
425
+ const registry = loadSourceDefFlowsConfig();
426
+ let kind;
427
+ try {
428
+ kind = resolveSourceKind(sourceDef, registry);
429
+ } catch (err) {
430
+ failRef(String(err && err.message || err), callback);
431
+ }
402
432
 
403
- // Use boardSetupRoot (from --extra) as copilot working directory
404
- const copilotCwd = extra.boardSetupRoot || undefined;
405
-
406
- // On Windows, delegate entirely to copilot_wrapper.bat which handles:
407
- // - session management (--resume UUID for multi-turn continuity)
408
- // - noise/footer stripping, JSON extraction, agentic retry on bad shape
409
- // On non-Windows, fall back to a basic direct invocation (no retry).
410
- const wrapperPath = path.join(__dirname, 'scripts', 'copilot_wrapper.bat');
411
- const useWrapper = process.platform === 'win32' && fs.existsSync(wrapperPath);
412
-
413
- if (useWrapper) {
414
- // Session dir is stable across refreshes so --resume continues the conversation.
415
- const sessionDir = path.join(
416
- extra.boardSetupRoot || os.tmpdir(),
417
- 'copilot-sessions',
418
- String(sourceDef.bindTo || 'default').replace(/[^a-zA-Z0-9_-]/g, '_'),
419
- );
420
- const wrapperOutFile = outRef.value + '.wrapper-out.json';
421
- try {
422
- resultValue = runCopilotViaWrapper(prompt, sourceDef, wrapperOutFile, sessionDir, copilotCwd);
423
- } catch (err) {
424
- failRef(`copilot invocation failed: ${String(err && err.message || err)}`, callback);
425
- } finally {
426
- try { fs.unlinkSync(wrapperOutFile); } catch {}
427
- }
428
- } else {
429
- // Non-Windows fallback: call copilot directly via cmd.exe and do basic JSON extraction.
430
- let rawOutput = '';
431
- try {
432
- rawOutput = execFileSync('cmd.exe', ['/d', '/c', 'copilot --allow-all'], {
433
- input: String(prompt),
434
- encoding: 'utf-8',
435
- stdio: ['pipe', 'pipe', 'pipe'],
436
- maxBuffer: 10 * 1024 * 1024,
437
- ...(copilotCwd ? { cwd: copilotCwd } : {}),
438
- });
439
- } catch (err) {
440
- failRef(`copilot invocation failed: ${String(err && err.message || err)}`, callback);
441
- }
442
- // Basic JSON extraction: find first { or [ in output
443
- const firstBrace = rawOutput.indexOf('{');
444
- const firstBracket = rawOutput.indexOf('[');
445
- const jsonStart = (firstBrace === -1) ? firstBracket
446
- : (firstBracket === -1) ? firstBrace
447
- : Math.min(firstBrace, firstBracket);
448
- if (jsonStart !== -1) {
449
- try {
450
- const parsed = JSON.parse(rawOutput.slice(jsonStart));
451
- resultValue = (parsed && typeof parsed === 'object') ? parsed : rawOutput;
452
- } catch {
453
- resultValue = rawOutput;
454
- }
455
- } else {
456
- resultValue = rawOutput;
457
- }
458
- }
459
- } else if (sourceDef.workiq) {
460
- const cfg = typeof sourceDef.workiq === 'object' ? sourceDef.workiq : {};
461
- if (!cfg.query_template || typeof cfg.query_template !== 'string') {
462
- failRef('Source definition missing workiq.query_template', callback);
463
- }
464
- const interpolationContext = { ...sourceDef._projections, ...(cfg.args ?? {}) };
465
- const query = interpolatePrompt(cfg.query_template, interpolationContext);
433
+ let flowResult;
434
+ try {
435
+ flowResult = await executeStepMachineSourceFlow({
436
+ kind,
437
+ registry,
438
+ sourceDef,
439
+ extra,
440
+ inRef,
441
+ outRef,
442
+ errRef,
443
+ mockDb: MOCK_DB,
444
+ });
445
+ } catch (err) {
446
+ const detail = (err && (err.stderr || err.stdout)) ? `\n${err.stderr || err.stdout}`.trimEnd() : '';
447
+ failRef(`${kind} invocation failed: ${String(err && err.message || err)}${detail}`, callback);
448
+ }
466
449
 
467
- const wrapperPath = path.join(__dirname, 'scripts', 'workiq_wrapper.mjs');
468
- if (!fs.existsSync(wrapperPath)) {
469
- failRef('workiq source kind requires workiq_wrapper.js in scripts/', callback);
470
- }
450
+ if (!flowResult?.wroteOutputDirectly) {
471
451
  try {
472
- execFileSync(process.execPath, [wrapperPath, outRef.value], {
473
- encoding: 'utf-8',
474
- stdio: ['inherit', 'pipe', 'pipe'],
475
- maxBuffer: 10 * 1024 * 1024,
476
- env: {
477
- ...process.env,
478
- WORKIQ_QUERY: query,
479
- ...(extra.serverUrl ? { WORKIQ_SERVER_URL: extra.serverUrl } : {}),
480
- },
481
- });
482
- return; // wrapper wrote directly to out file
452
+ outStorage.write(outRef.value, JSON.stringify(flowResult?.resultValue, null, 2));
483
453
  } catch (err) {
484
- failRef(`workiq invocation failed: ${String(err && err.message || err)}`, callback);
485
- }
486
- } else if (sourceDef.mock) {
487
- // MOCK_DB lookup — data hardcoded at the top of this file
488
- resultValue = MOCK_DB[sourceDef.mock];
489
- if (resultValue === undefined) {
490
- failRef(`Key "${sourceDef.mock}" not found in MOCK_DB`, callback);
454
+ failRef(`Cannot write output: ${String(err && err.message || err)}`, callback);
491
455
  }
492
- } else {
493
- failRef('Source definition has no recognised kind (url, url-list, copilot, workiq, mock)', callback);
494
- }
495
-
496
- // Write result to --out via storage abstraction, then report back to board.
497
- try {
498
- outStorage.write(outRef.value, JSON.stringify(resultValue, null, 2));
499
- } catch (err) {
500
- failRef(`Cannot write output: ${String(err && err.message || err)}`, callback);
501
456
  }
502
457
 
503
458
  if (callback) {
@@ -537,30 +492,16 @@ function validateSourceDefSubcommand(argv) {
537
492
  }
538
493
 
539
494
  const errors = [];
495
+ const registry = loadSourceDefFlowsConfig();
540
496
 
541
- // Determine source kind and validate required fields
542
- const hasUrl = !!sourceDef['url'];
543
- const hasUrlList = !!sourceDef['url-list'];
544
- const hasCopilot = !!sourceDef.copilot;
545
- const hasPromptTemplate = typeof sourceDef.prompt_template === 'string';
546
- const hasWorkiq = !!sourceDef.workiq;
547
- const hasMock = sourceDef.mock !== undefined;
548
-
549
- const kindCount = [hasUrl, hasUrlList, hasCopilot || hasPromptTemplate, hasWorkiq, hasMock].filter(Boolean).length;
550
-
551
- if (kindCount === 0) {
552
- errors.push('No recognised source kind (url, url-list, copilot, workiq, mock). Add one of these fields.');
553
- } else if (kindCount > 1) {
554
- const kinds = [];
555
- if (hasUrl) kinds.push('url');
556
- if (hasUrlList) kinds.push('url-list');
557
- if (hasCopilot || hasPromptTemplate) kinds.push('copilot');
558
- if (hasWorkiq) kinds.push('workiq');
559
- if (hasMock) kinds.push('mock');
560
- errors.push(`Multiple source kinds specified: [${kinds.join(', ')}]. Use exactly one.`);
561
- }
562
-
563
- if (hasUrl) {
497
+ let kind = '';
498
+ try {
499
+ kind = resolveSourceKind(sourceDef, registry);
500
+ } catch (err) {
501
+ errors.push(String(err && err.message || err));
502
+ }
503
+
504
+ if (kind === 'url') {
564
505
  if (typeof sourceDef['url'] !== 'object') {
565
506
  errors.push('url must be an object.');
566
507
  } else if (!sourceDef['url'].url || typeof sourceDef['url'].url !== 'string') {
@@ -568,24 +509,24 @@ function validateSourceDefSubcommand(argv) {
568
509
  }
569
510
  }
570
511
 
571
- if (hasUrlList) {
512
+ if (kind === 'url-list') {
572
513
  if (typeof sourceDef['url-list'] !== 'object') {
573
514
  errors.push('url-list must be an object.');
574
515
  }
575
516
  // url_list is supplied via _projections at runtime — no static validation needed.
576
517
  }
577
518
 
578
- if (hasCopilot) {
519
+ if (kind === 'copilot') {
579
520
  if (typeof sourceDef.copilot !== 'object') {
580
- errors.push('copilot must be an object.');
581
- } else {
582
- if (!sourceDef.copilot.prompt_template && !hasPromptTemplate) {
583
- errors.push('copilot.prompt_template is required (or use top-level prompt_template).');
521
+ if (typeof sourceDef.prompt_template !== 'string') {
522
+ errors.push('copilot must be an object when prompt_template is not provided at top level.');
584
523
  }
524
+ } else if (!sourceDef.copilot.prompt_template && typeof sourceDef.prompt_template !== 'string') {
525
+ errors.push('copilot.prompt_template is required (or use top-level prompt_template).');
585
526
  }
586
527
  }
587
528
 
588
- if (hasWorkiq) {
529
+ if (kind === 'workiq') {
589
530
  if (typeof sourceDef.workiq !== 'object') {
590
531
  errors.push('workiq must be an object.');
591
532
  } else if (!sourceDef.workiq.query_template || typeof sourceDef.workiq.query_template !== 'string') {
@@ -593,7 +534,7 @@ function validateSourceDefSubcommand(argv) {
593
534
  }
594
535
  }
595
536
 
596
- if (hasMock) {
537
+ if (kind === 'mock') {
597
538
  if (typeof sourceDef.mock !== 'string') {
598
539
  errors.push('mock must be a string key.');
599
540
  }
@@ -701,16 +642,27 @@ const CAPABILITIES = {
701
642
  };
702
643
 
703
644
  function describeCapabilities() {
704
- console.log(JSON.stringify(CAPABILITIES, null, 2));
645
+ const registry = loadSourceDefFlowsConfig();
646
+ const merged = {
647
+ ...CAPABILITIES,
648
+ sourceKinds: Object.fromEntries(
649
+ Object.entries(registry?.kinds || {}).map(([kind, spec]) => {
650
+ const existing = CAPABILITIES.sourceKinds[kind] || {};
651
+ const manifest = spec?.manifest && typeof spec.manifest === 'object' ? spec.manifest : {};
652
+ return [kind, { ...existing, ...manifest }];
653
+ }),
654
+ ),
655
+ };
656
+ console.log(JSON.stringify(merged, null, 2));
705
657
  }
706
658
 
707
659
  async function main() {
708
660
  const sub = process.argv[2];
709
661
  if (sub === 'run-source-fetch') {
710
- runSourceFetchSubcommand(process.argv.slice(3));
662
+ await runSourceFetchSubcommand(process.argv.slice(3));
711
663
  return;
712
664
  }
713
- if (sub === 'describe-capabilities') {
665
+ if (sub === 'describe' || sub === 'describe-capabilities') {
714
666
  describeCapabilities();
715
667
  return;
716
668
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "yaml-flow",
3
- "version": "6.0.0",
3
+ "version": "7.0.0",
4
4
  "description": "Unified workflow engine: step-machine (sequential) + event-graph (stateless DAG) with pluggable storage",
5
5
  "author": "",
6
6
  "license": "MIT",
@@ -106,7 +106,10 @@
106
106
  "types": "./dist/cli/browser-api/card-store-browser-api.d.ts",
107
107
  "import": "./dist/cli/browser-api/card-store-browser-api.js"
108
108
  },
109
- "./board-livecards-server-runtime": "./board-livecards-server-runtime.js",
109
+ "./server-runtime": {
110
+ "types": "./dist/server-runtime/index.d.ts",
111
+ "import": "./dist/server-runtime/index.js"
112
+ },
110
113
  "./package.json": "./package.json"
111
114
  },
112
115
  "browser": {
@@ -116,7 +119,6 @@
116
119
  "board-live-cards-cli.js",
117
120
  "storage-refs.js",
118
121
  "step-machine-cli.js",
119
- "board-livecards-server-runtime.js",
120
122
  "dist",
121
123
  "schema",
122
124
  "browser",
@@ -125,32 +127,33 @@
125
127
  "scripts": {
126
128
  "build": "tsup",
127
129
  "build:browser": "tsup --config tsup.browser.config.ts && node scripts/generate-browser-integrity.mjs",
128
- "build:quickjs": "tsup --config tsup.quickjs.config.ts",
129
130
  "check:bundle-budget": "node scripts/check-bundle-budgets.mjs",
130
131
  "check:browser-bundle": "node scripts/check-browser-bundle-safety.mjs",
131
- "release:gate": "npm run -s build && npm run -s build:browser && npm run -s build:quickjs && npm run -s test:run && npm run -s check:bundle-budget && npm run -s check:browser-bundle",
132
- "standalone": "npm run -s test:cli-parity && npm run -s build && npm run -s build:quickjs && node scripts/build-standalone.mjs",
132
+ "release:gate": "npm run -s build && npm run -s build:browser && npm run -s test:run && npm run -s check:bundle-budget && npm run -s check:browser-bundle",
133
+ "standalone": "npm run -s test:cli-parity && npm run -s build && node scripts/build-standalone.mjs",
133
134
  "pycli:install": "python -m pip install -r pycli/requirements.txt",
134
135
  "pycli:install:venv": "c:/Users/sreenaga/ADO/ai-tool-evolver/.venv/Scripts/python.exe -m pip install -r pycli/requirements.txt",
135
136
  "pycli:install:py312": "py -3.12 -m pip install -r pycli/requirements.txt",
136
137
  "pycli:install:venv312": "./.venv312/Scripts/python.exe -m pip install -r pycli/requirements.txt",
137
138
  "build:example": "node demo-src/example-board/build.js",
138
139
  "dev": "tsup --watch",
139
- "test": "vitest",
140
- "test:run": "vitest run",
141
- "test:cli-parity": "npm run -s build:quickjs && python scripts/cli_parity_check.py",
142
- "test:cli-parity:full": "npm run -s build:quickjs && python scripts/cli_parity_check.py --include-portfolio",
143
- "test:e2e": "npm run -s build:quickjs && .\\.venv312\\Scripts\\python.exe examples/browser/boards/portfolio-tracker/portfolio-tracker.py --run-pycli",
140
+ "test": "npm run build:example && vitest",
141
+ "test:run": "npm run build:example && vitest run",
142
+ "test:cli-parity": "python scripts/cli_parity_check.py",
143
+ "test:cli-parity:full": "python scripts/cli_parity_check.py --include-portfolio",
144
+ "test:e2e": "python examples/browser/boards/portfolio-tracker/portfolio-tracker.py --run-pycli",
145
+ "test:portfolio-tracker-server": "node examples/browser/boards/portfolio-tracker/portfolio-tracker-http-test.js",
144
146
  "test:example-board": "vitest run tests/examples/example-board.test.ts",
145
147
  "check:example-board": "npm run test:example-board && npm run validate:cards -- \"examples/example-board/cards/*.json\"",
146
148
  "validate:cards": "tsx scripts/validate-live-cards.ts",
147
149
  "lint": "eslint src/",
148
150
  "typecheck": "tsc --noEmit",
149
- "prepublishOnly": "npm run build && npm run build:browser && npm run build:quickjs && npm run check:bundle-budget && npm run check:browser-bundle"
151
+ "prepublishOnly": "npm run build && npm run build:browser && npm run check:bundle-budget && npm run check:browser-bundle"
150
152
  },
151
153
  "devDependencies": {
152
154
  "@types/node": "^20.10.0",
153
155
  "@types/proper-lockfile": "^4.1.4",
156
+ "jsdom": "^24.0.0",
154
157
  "tsup": "^8.0.0",
155
158
  "tsx": "^4.21.0",
156
159
  "typescript": "^5.3.0",
@@ -159,7 +162,6 @@
159
162
  "dependencies": {
160
163
  "ajv-formats": "^3.0.1",
161
164
  "fast-glob": "^3.3.3",
162
- "jsonata": "^2.1.0",
163
165
  "proper-lockfile": "^4.1.2",
164
166
  "yaml": "^2.3.4"
165
167
  },