yaml-flow 3.1.1 → 5.0.0

package/examples/step-machine-cli/portfolio-tracker/run-portfolio-tracker.bat

@@ -0,0 +1,29 @@
+@echo off
+setlocal
+cd /d "%~dp0"
+
+set "BOARD_LIVE_CARDS_NO_SPAWN=1"
+set "EXAMPLE_TEMP_ROOT=%TEMP%\yaml-flow-step-machine-portfolio-tracker"
+set "STORE_DIR=%EXAMPLE_TEMP_ROOT%\store"
+set "RUNTIME_ROOT=%EXAMPLE_TEMP_ROOT%\runtime"
+
+if /I "%~1"=="pause" goto :pause
+if /I "%~1"=="resume" goto :resume
+if /I "%~1"=="status" goto :status
+goto :run
+
+:pause
+node ..\..\..\step-machine-cli.js --store file --store-dir "%STORE_DIR%" --pause
+exit /b %ERRORLEVEL%
+
+:resume
+node ..\..\..\step-machine-cli.js portfolio-tracker.flow.yaml --store file --store-dir "%STORE_DIR%" --resume
+exit /b %ERRORLEVEL%
+
+:status
+node ..\..\..\step-machine-cli.js --store file --store-dir "%STORE_DIR%" --status
+exit /b %ERRORLEVEL%
+
+:run
+	node -e "const fs=require('fs');const cp=require('child_process');const raw=JSON.parse(fs.readFileSync('portfolio-tracker.input.json','utf8'));raw.runtime_root=(process.env.RUNTIME_ROOT||'').replace(/\\\\/g,'/');const input=JSON.stringify(raw);const r=cp.spawnSync(process.execPath,['..\\..\\..\\step-machine-cli.js','portfolio-tracker.flow.yaml','--store','file','--store-dir',process.env.STORE_DIR,'--initial-data',input],{stdio:'inherit',windowsHide:true,env:{...process.env,BOARD_LIVE_CARDS_NO_SPAWN:'1'}});process.exit(r.status??1);"
+exit /b %ERRORLEVEL%

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "yaml-flow",
-  "version": "3.1.1",
+  "version": "5.0.0",
   "description": "Unified workflow engine: step-machine (sequential) + event-graph (stateless DAG) with pluggable storage",
   "author": "",
   "license": "MIT",
@@ -8,6 +8,10 @@
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",
   "types": "./dist/index.d.ts",
+  "bin": {
+    "board-live-cards-cli": "./board-live-cards-cli.js",
+    "step-machine-cli": "./step-machine-cli.js"
+  },
   "exports": {
     ".": {
       "types": "./dist/index.d.ts",
@@ -59,6 +63,11 @@
       "import": "./dist/continuous-event-graph/index.js",
       "require": "./dist/continuous-event-graph/index.cjs"
     },
+    "./board-livegraph-runtime": {
+      "types": "./dist/board-livegraph-runtime/index.d.ts",
+      "import": "./dist/board-livegraph-runtime/index.js",
+      "require": "./dist/board-livegraph-runtime/index.cjs"
+    },
     "./inference": {
       "types": "./dist/inference/index.d.ts",
       "import": "./dist/inference/index.js",
@@ -68,12 +77,19 @@
       "types": "./dist/card-compute/index.d.ts",
       "import": "./dist/card-compute/index.js",
       "require": "./dist/card-compute/index.cjs"
+    },
+    "./runtime-artifacts": {
+      "types": "./dist/runtime-artifacts/index.d.ts",
+      "import": "./dist/runtime-artifacts/index.js",
+      "require": "./dist/runtime-artifacts/index.cjs"
     }
   },
   "browser": {
     "./stores/file": false
   },
   "files": [
+    "board-live-cards-cli.js",
+    "step-machine-cli.js",
     "dist",
     "schema",
     "browser",
@@ -81,21 +97,30 @@
   ],
   "scripts": {
     "build": "tsup",
+    "build:browser": "tsup --config tsup.browser.config.ts",
     "dev": "tsup --watch",
     "test": "vitest",
     "test:run": "vitest run",
+    "test:example-board": "vitest run tests/examples/example-board.test.ts",
+    "check:example-board": "npm run test:example-board && npm run validate:cards -- \"examples/example-board/cards/*.json\"",
+    "validate:cards": "tsx scripts/validate-live-cards.ts",
     "lint": "eslint src/",
     "typecheck": "tsc --noEmit",
-    "prepublishOnly": "npm run build"
+    "prepublishOnly": "npm run build && npm run build:browser"
   },
   "devDependencies": {
     "@types/node": "^20.10.0",
+    "@types/proper-lockfile": "^4.1.4",
     "tsup": "^8.0.0",
+    "tsx": "^4.21.0",
     "typescript": "^5.3.0",
     "vitest": "^1.0.0"
   },
   "dependencies": {
     "ajv-formats": "^3.0.1",
+    "fast-glob": "^3.3.3",
+    "jsonata": "^2.1.0",
+    "proper-lockfile": "^4.1.2",
     "yaml": "^2.3.4"
   },
   "optionalDependencies": {

package/schema/board-status.schema.json

@@ -0,0 +1,118 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://yaml-flow.dev/schema/board-status.schema.json",
+  "title": "Board Status Object",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["schema_version", "meta", "summary", "cards"],
+  "properties": {
+    "schema_version": {
+      "type": "string",
+      "const": "v1"
+    },
+    "meta": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["board"],
+      "properties": {
+        "board": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["path"],
+          "properties": {
+            "path": { "type": "string" }
+          }
+        }
+      }
+    },
+    "summary": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["card_count", "completed", "eligible", "pending", "blocked", "unresolved"],
+      "properties": {
+        "card_count": { "type": "integer", "minimum": 0 },
+        "completed": { "type": "integer", "minimum": 0 },
+        "eligible": { "type": "integer", "minimum": 0 },
+        "pending": { "type": "integer", "minimum": 0 },
+        "blocked": { "type": "integer", "minimum": 0 },
+        "unresolved": { "type": "integer", "minimum": 0 },
+        "failed": { "type": "integer", "minimum": 0 },
+        "in_progress": { "type": "integer", "minimum": 0 },
+        "orphan_cards": { "type": "integer", "minimum": 0 },
+        "topology": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["edge_count", "max_fan_out_card", "max_fan_out"],
+          "properties": {
+            "edge_count": { "type": "integer", "minimum": 0 },
+            "max_fan_out_card": { "type": ["string", "null"] },
+            "max_fan_out": { "type": "integer", "minimum": 0 }
+          }
+        }
+      }
+    },
+    "cards": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": [
+          "name",
+          "status",
+          "requires",
+          "requires_satisfied",
+          "requires_missing",
+          "provides_declared",
+          "provides_runtime",
+          "blocked_by",
+          "unblocks",
+          "runtime"
+        ],
+        "properties": {
+          "name": { "type": "string" },
+          "status": { "type": "string" },
+          "error": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["message"],
+            "properties": {
+              "message": { "type": "string" },
+              "code": { "type": "string" },
+              "at": { "type": ["string", "null"] },
+              "source": { "type": "string" }
+            }
+          },
+          "requires": { "type": "array", "items": { "type": "string" } },
+          "requires_satisfied": { "type": "array", "items": { "type": "string" } },
+          "requires_missing": { "type": "array", "items": { "type": "string" } },
+          "provides_declared": { "type": "array", "items": { "type": "string" } },
+          "provides_runtime": { "type": "array", "items": { "type": "string" } },
+          "blocked_by": { "type": "array", "items": { "type": "string" } },
+          "unblocks": { "type": "array", "items": { "type": "string" } },
+          "runtime": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": [
+              "attempt_count",
+              "restart_count",
+              "in_progress_since",
+              "last_transition_at",
+              "last_completed_at",
+              "last_restarted_at",
+              "status_age_ms"
+            ],
+            "properties": {
+              "attempt_count": { "type": "integer", "minimum": 0 },
+              "restart_count": { "type": "integer", "minimum": 0 },
+              "in_progress_since": { "type": ["string", "null"] },
+              "last_transition_at": { "type": ["string", "null"] },
+              "last_completed_at": { "type": ["string", "null"] },
+              "last_restarted_at": { "type": ["string", "null"] },
+              "status_age_ms": { "type": ["integer", "null"], "minimum": 0 }
+            }
+          }
+        }
+      }
+    }
+  }
+}

package/schema/card-runtime.schema.json

@@ -0,0 +1,25 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://yaml-flow.dev/schema/card-runtime.schema.json",
+  "title": "Card Runtime Computed Artifact",
+  "description": "Published per-card runtime artifact written to runtime-out/cards/<card-id>.computed.json. Contains only values produced by the compute step.",
+  "type": "object",
+  "required": ["schema_version", "card_id", "computed_values"],
+  "additionalProperties": false,
+  "properties": {
+    "schema_version": {
+      "type": "string",
+      "const": "v1",
+      "description": "Artifact schema version."
+    },
+    "card_id": {
+      "type": "string",
+      "description": "The ID of the card this artifact belongs to."
+    },
+    "computed_values": {
+      "type": "object",
+      "description": "Key/value map of outputs produced by the card's JSONata compute steps.",
+      "additionalProperties": true
+    }
+  }
+}

package/schema/flow.schema.json

@@ -77,6 +77,11 @@
           "additionalProperties": { "type": "string" },
           "minProperties": 1
         },
+        "failure_transitions": {
+          "type": "object",
+          "description": "Mapping of failure-like results (e.g. failure, timeout) -> next step name",
+          "additionalProperties": { "type": "string" }
+        },
         "retry": {
           "$ref": "#/definitions/retry_config"
         },

package/schema/live-cards.schema.json

@@ -1,15 +1,13 @@
 {
   "$schema": "http://json-schema.org/draft-07/schema#",
   "$id": "https://nsreehari.github.io/boards/live-cards.schema.json",
-  "title": "LiveCards Node Schema",
-  "description": "Schema for Card and ExternalSource nodes in the LiveCards Board/Canvas engine",
 
   "definitions": {
 
     "bind_ref": {
-      "description": "A state path reference, e.g. 'state.raw_quotes' or 'state.compute_vars.total'",
+      "description": "A card data path reference, e.g. 'card_data.raw_quotes' or 'requires.upstream'",
       "type": "string",
-      "pattern": "^state\\."
+      "pattern": "^(card_data|requires|fetched_sources|computed_values)(\\.|$)"
     },
 
     "bind_or_literal": {
@@ -46,7 +44,7 @@
           ]
         },
         "input": {
-          "description": "state.path, literal, array of inputs, or nested compute_expr",
+          "description": "card_data.path, literal, array of inputs, or nested compute_expr",
           "oneOf": [
             { "type": "string" },
             { "type": "number" },
@@ -72,39 +70,46 @@
       }
     },
 
-    "data": {
+    "requires": {
+      "type": "array",
+      "items": { "type": "string" },
+      "description": "IDs of upstream nodes this node depends on"
+    },
+
+    "provides": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": ["bindTo", "src"],
+        "properties": {
+          "bindTo": { "type": "string", "description": "Token name published downstream" },
+          "src":    { "type": "string", "description": "Path to read value from (card_data.*, requires.*, fetched_sources.*, computed_values.*)" }
+        }
+      },
+      "description": "Explicit bindings exposing computed or card_data values downstream as named tokens"
+    },
+
+    "compute_step": {
+      "description": "A single ordered compute step: reads card_data.*/requires.*/computed_values.*, writes to computed_values[bindTo]",
       "type": "object",
+      "required": ["bindTo", "expr"],
       "properties": {
-        "requires": {
-          "type": "array",
-          "items": { "type": "string" },
-          "description": "IDs of upstream nodes this node depends on"
-        },
-        "provides": {
-          "type": "object",
-          "description": "Subset of state exposed downstream. Keys are published names, values are bind refs.",
-          "additionalProperties": { "$ref": "#/definitions/bind_or_literal" }
-        }
+        "bindTo": { "type": "string", "description": "Key in computed_values to write result" },
+        "expr":   { "type": "string", "description": "JSONata expression evaluated against { card_data, requires, fetched_sources, computed_values }" }
       }
     },
 
     "source_def": {
+      "description": "One source entry. The engine only cares about 'bindTo' (compute namespace key) and 'outputFile' (delivery signal). Every other property is yours — add whatever your task-executor needs: kind, url, headers, mailbox, channel, model, query, etc. The full object is passed verbatim as the --in JSON to the executor.",
       "type": "object",
-      "required": ["kind", "bindTo"],
+      "required": ["bindTo"],
+      "additionalProperties": true,
       "properties": {
-        "kind":           { "enum": ["api", "websocket", "static", "llm"] },
-        "bindTo":         { "$ref": "#/definitions/bind_ref", "description": "state path to write fetched data into" },
-        "method":         { "enum": ["GET", "POST", "PUT", "DELETE"], "default": "GET" },
-        "url_template":   { "type": "string", "description": "URL with {{var}} placeholders" },
-        "headers":        { "type": "object", "additionalProperties": { "$ref": "#/definitions/bind_or_literal" } },
-        "body_template":  { "type": "object", "description": "Request body with {{var}} placeholders or bind refs" },
-        "template_vars":  {
-          "type": "object",
-          "additionalProperties": { "$ref": "#/definitions/bind_or_literal" },
-          "description": "Variables for url/body templates — static values or bind refs"
-        },
-        "poll_interval":  { "type": "integer", "minimum": 0, "description": "Auto-refresh in seconds (0 = manual)" },
-        "transform":      { "type": "string", "description": "Dot-path to extract from response, e.g. 'data.items'" }
+        "bindTo":     { "type": "string",  "description": "Key under fetched_sources.* available in compute expressions" },
+        "outputFile": { "type": "string",  "description": "Board-relative path the executor writes its JSON result to. Presence of this file signals delivery." },
+        "optionalForCompletionGating":   { "type": "boolean", "default": false, "description": "When true this source does not gate card completion. Default false when absent, so sources are completion-gating by default." },
+        "timeout":    { "type": "integer", "minimum": 0, "default": 120000, "description": "Executor/script timeout in ms. Default: 120 000 (2 min)." },
+        "script":     { "type": "string",  "description": "Legacy direct-run: shell command executed when no .task-executor is registered. stdout is captured as the result." }
       }
     },
 
@@ -116,16 +121,16 @@
         "kind":      {
           "enum": ["metric", "table", "chart", "form", "filter", "list",
                    "notes", "todo", "alert", "narrative", "badge", "text",
-                   "markdown", "custom", "file-upload", "chat", "actions"]
+                   "markdown", "custom", "actions"]
         },
         "label":     { "type": "string", "description": "Heading above this element" },
         "className": { "type": "string", "description": "Bootstrap grid class, e.g. 'col-12 col-md-6'" },
-        "visible":   { "type": "string", "description": "state path — element shown only if truthy" },
+        "visible":   { "type": "string", "description": "card_data/requires/fetched_sources/computed_values path — element shown only if truthy" },
         "data": {
           "type": "object",
           "properties": {
-            "bind":        { "$ref": "#/definitions/bind_ref", "description": "state path to read data from" },
-            "writeTo":     { "$ref": "#/definitions/bind_ref", "description": "state path for user input (form, filter, todo, notes)" },
+            "bind":        { "$ref": "#/definitions/bind_ref", "description": "card_data/requires/fetched_sources/computed_values path to read data from" },
+            "writeTo":     { "$ref": "#/definitions/bind_ref", "description": "card_data path for user input (form, filter, todo, notes)" },
             "columns":     { "type": "array", "items": { "type": "string" }, "description": "table: visible columns" },
             "maxRows":     { "type": "integer", "description": "table/list: max rows to display" },
             "sortable":    { "type": "boolean", "default": true, "description": "table: enable click-to-sort" },
@@ -158,7 +163,7 @@
                   "label":    { "type": "string" },
                   "style":    { "type": "string", "description": "Bootstrap button variant, e.g. 'success', 'outline-danger'" },
                   "size":     { "type": "string", "default": "sm" },
-                  "disabled": { "oneOf": [{ "type": "boolean" }, { "type": "string", "description": "state path — truthy = disabled" }] }
+                  "disabled": { "oneOf": [{ "type": "boolean" }, { "type": "string", "description": "card_data/requires/fetched_sources/computed_values path — truthy = disabled" }] }
                 }
               }
             }
@@ -209,58 +214,60 @@
     }
   },
 
-  "oneOf": [
-    {
-      "title": "Card",
-      "description": "A renderable card node with view elements",
+  "title": "LiveCard",
+  "description": "A unified card node. Behavior depends on which sections are present (sources, compute, view, etc.)",
+  "type": "object",
+  "required": ["id"],
+  "additionalProperties": false,
+  "properties": {
+    "id":       { "type": "string" },
+    "requires": { "$ref": "#/definitions/requires" },
+    "provides": { "$ref": "#/definitions/provides" },
+    "meta":     { "$ref": "#/definitions/meta" },
+    "view":     { "$ref": "#/definitions/view" },
+    "card_data": {
       "type": "object",
-      "required": ["id", "type", "view", "state"],
-      "additionalProperties": false,
+      "description": "Authored card data supplied in the card definition",
       "properties": {
-        "id":      { "type": "string" },
-        "type":    { "const": "card" },
-        "meta":    { "$ref": "#/definitions/meta" },
-        "data":    { "$ref": "#/definitions/data" },
-        "view":    { "$ref": "#/definitions/view" },
-        "state":   { "type": "object", "additionalProperties": true,
-          "properties": {
-            "status":  { "enum": ["fresh", "stale", "loading", "error"] },
-            "lastRun": { "type": "string", "format": "date-time" },
-            "error":   { "type": "string" }
+        "files": {
+          "type": "array",
+          "description": "Optional uploaded-file metadata maintained by host handlers. Stored name is normalized and serial-prefixed (for example 001-my_file.pdf).",
+          "items": {
+            "type": "object",
+            "required": ["name", "stored_name"],
+            "properties": {
+              "name": { "type": "string", "minLength": 1 },
+              "stored_name": {
+                "type": "string",
+                "minLength": 5,
+                "maxLength": 32,
+                "pattern": "^[0-9]{3,}-[a-z0-9._-]+$"
+              },
+              "size": {
+                "oneOf": [
+                  { "type": "integer", "minimum": 0 },
+                  { "type": "null" }
+                ]
+              },
+              "mime_type": { "type": "string" },
+              "path": { "type": "string", "pattern": "^[^\\s]+/files/[0-9]{3,}-[a-z0-9._-]+$" },
+              "uploaded_at": { "type": "string", "format": "date-time" }
+            },
+            "additionalProperties": false
           }
-        },
-        "compute": {
-          "type": "object",
-          "description": "Derived state: key = state path to write, value = compute_expr",
-          "additionalProperties": { "$ref": "#/definitions/compute_expr" }
         }
-      }
+      },
+      "additionalProperties": true
     },
-    {
-      "title": "ExternalSource",
-      "description": "A data-only node that fetches from external systems (no view)",
-      "type": "object",
-      "required": ["id", "type", "source", "state"],
-      "additionalProperties": false,
-      "properties": {
-        "id":      { "type": "string" },
-        "type":    { "const": "source" },
-        "meta":    { "$ref": "#/definitions/meta" },
-        "data":    { "$ref": "#/definitions/data" },
-        "source":  { "$ref": "#/definitions/source_def" },
-        "state":   { "type": "object", "additionalProperties": true,
-          "properties": {
-            "status":  { "enum": ["fresh", "stale", "loading", "error"] },
-            "lastRun": { "type": "string", "format": "date-time" },
-            "error":   { "type": "string" }
-          }
-        },
-        "compute": {
-          "type": "object",
-          "description": "Derived state: key = state path to write, value = compute_expr",
-          "additionalProperties": { "$ref": "#/definitions/compute_expr" }
-        }
-      }
+    "sources": {
+      "type": "array",
+      "description": "Source entries. Each entry is passed verbatim to the board's .task-executor (registered via init --task-executor) as the --in JSON file. The executor fetches/generates the data and writes JSON to --out. If no executor is registered, the built-in executor runs the entry's 'cli' command directly. Sources gate completion by default. Set optionalForCompletionGating: true for enrichment-only sources that should not block task-completed.",
+      "items": { "$ref": "#/definitions/source_def" }
+    },
+    "compute": {
+      "type": "array",
+      "description": "Ordered array of compute steps. Each reads card_data.*/requires.*/fetched_sources.*/computed_values.* and writes to ephemeral computed_values[bindTo].",
+      "items": { "$ref": "#/definitions/compute_step" }
     }
-  ]
+  }
 }