yaml-flow 2.7.0 → 3.0.0

package/dist/continuous-event-graph/index.js

@@ -1,3 +1,5 @@
+import { existsSync, writeFileSync, appendFileSync, readFileSync } from 'fs';
+
 // src/event-graph/constants.ts
 var TASK_STATUS = {
   NOT_STARTED: "not-started",
@@ -25,15 +27,11 @@ function isNonActiveTask(taskState) {
   if (!taskState) return false;
   return taskState.status === TASK_STATUS.FAILED || taskState.status === TASK_STATUS.INACTIVATED;
 }
-function isRepeatableTask(taskConfig) {
-  return taskConfig.repeatable === true || typeof taskConfig.repeatable === "object" && taskConfig.repeatable !== null;
+function getRefreshStrategy(taskConfig, graphSettings) {
+  return taskConfig.refreshStrategy ?? graphSettings?.refreshStrategy ?? "data-changed";
 }
-function getRepeatableMax(taskConfig) {
-  if (taskConfig.repeatable === true) return void 0;
-  if (typeof taskConfig.repeatable === "object" && taskConfig.repeatable !== null) {
-    return taskConfig.repeatable.max;
-  }
-  return void 0;
+function getMaxExecutions(taskConfig) {
+  return taskConfig.maxExecutions;
 }
 function computeAvailableOutputs(graph, taskStates) {
   const outputs = /* @__PURE__ */ new Set();
@@ -81,7 +79,7 @@ function applyTaskStart(state, taskName) {
     lastUpdated: (/* @__PURE__ */ new Date()).toISOString()
   };
 }
-function applyTaskCompletion(state, graph, taskName, result) {
+function applyTaskCompletion(state, graph, taskName, result, dataHash) {
   const existingTask = state.tasks[taskName] ?? createDefaultTaskState();
   const taskConfig = graph.tasks[taskName];
   if (!taskConfig) {
@@ -93,6 +91,19 @@ function applyTaskCompletion(state, graph, taskName, result) {
   } else {
     outputTokens = getProvides(taskConfig);
   }
+  const lastConsumedHashes = { ...existingTask.lastConsumedHashes };
+  const requires = taskConfig.requires ?? [];
+  for (const token of requires) {
+    for (const [otherName, otherConfig] of Object.entries(graph.tasks)) {
+      if (getProvides(otherConfig).includes(token)) {
+        const otherState = state.tasks[otherName];
+        if (otherState?.lastDataHash) {
+          lastConsumedHashes[token] = otherState.lastDataHash;
+        }
+        break;
+      }
+    }
+  }
   const updatedTask = {
     ...existingTask,
     status: "completed",
@@ -100,11 +111,10 @@ function applyTaskCompletion(state, graph, taskName, result) {
     lastUpdated: (/* @__PURE__ */ new Date()).toISOString(),
     executionCount: existingTask.executionCount + 1,
     lastEpoch: existingTask.executionCount + 1,
+    lastDataHash: dataHash,
+    lastConsumedHashes,
     error: void 0
   };
-  if (isRepeatableTask(taskConfig)) {
-    updatedTask.status = "not-started";
-  }
   const newOutputs = [.../* @__PURE__ */ new Set([...state.availableOutputs, ...outputTokens])];
   return {
     ...state,
@@ -217,7 +227,7 @@ function applyEvent(live, event) {
       newState = applyTaskStart(state, event.taskName);
       break;
     case "task-completed":
-      newState = applyTaskCompletion(state, config, event.taskName, event.result);
+      newState = applyTaskCompletion(state, config, event.taskName, event.result, event.dataHash);
       break;
     case "task-failed":
       newState = applyTaskFailure(state, config, event.taskName, event.error);
@@ -240,6 +250,9 @@ function applyEvent(live, event) {
   }
   return { config, state: newState };
 }
+function applyEvents(live, events) {
+  return events.reduce((current, event) => applyEvent(current, event), live);
+}
 function addNode(live, name, taskConfig) {
   if (live.config.tasks[name]) return live;
   return {
@@ -484,38 +497,84 @@ function schedule(live) {
   const blocked = [];
   for (const [taskName, taskConfig] of Object.entries(graphTasks)) {
     const taskState = state.tasks[taskName];
-    if (!isRepeatableTask(taskConfig)) {
-      if (taskState?.status === TASK_STATUS.COMPLETED || taskState?.status === TASK_STATUS.RUNNING || isNonActiveTask(taskState)) {
-        continue;
-      }
-    } else {
-      if (taskState?.status === TASK_STATUS.RUNNING || isNonActiveTask(taskState)) {
-        continue;
-      }
-      const maxExec = getRepeatableMax(taskConfig);
-      if (maxExec !== void 0 && taskState && taskState.executionCount >= maxExec) {
-        continue;
-      }
-      if (taskConfig.circuit_breaker && taskState && taskState.executionCount >= taskConfig.circuit_breaker.max_executions) {
-        continue;
-      }
-      if (taskState?.status === TASK_STATUS.COMPLETED) {
-        const requires2 = getRequires(taskConfig);
-        if (requires2.length > 0) {
-          const hasRefreshed = requires2.some((req) => {
-            for (const [otherName, otherConfig] of Object.entries(graphTasks)) {
-              if (getProvides(otherConfig).includes(req)) {
-                const otherState = state.tasks[otherName];
-                if (otherState && otherState.executionCount > taskState.lastEpoch) return true;
+    const strategy = getRefreshStrategy(taskConfig, config.settings);
+    const rerunnable = strategy !== "once";
+    if (taskState?.status === TASK_STATUS.RUNNING || isNonActiveTask(taskState)) {
+      continue;
+    }
+    const maxExec = getMaxExecutions(taskConfig);
+    if (maxExec !== void 0 && taskState && taskState.executionCount >= maxExec) {
+      continue;
+    }
+    if (taskConfig.circuit_breaker && taskState && taskState.executionCount >= taskConfig.circuit_breaker.max_executions) {
+      continue;
+    }
+    if (!rerunnable && taskState?.status === TASK_STATUS.COMPLETED) {
+      continue;
+    }
+    if (rerunnable && taskState?.status === TASK_STATUS.COMPLETED) {
+      const requires2 = getRequires(taskConfig);
+      let shouldSkip = false;
+      switch (strategy) {
+        case "data-changed": {
+          if (requires2.length > 0) {
+            const hasChangedData = requires2.some((req) => {
+              for (const [otherName, otherConfig] of Object.entries(graphTasks)) {
+                if (getProvides(otherConfig).includes(req)) {
+                  const otherState = state.tasks[otherName];
+                  if (!otherState) continue;
+                  const consumed = taskState.lastConsumedHashes?.[req];
+                  if (otherState.lastDataHash == null) {
+                    return otherState.executionCount > taskState.lastEpoch;
+                  }
+                  return otherState.lastDataHash !== consumed;
+                }
               }
-            }
-            return false;
-          });
-          if (!hasRefreshed) continue;
-        } else {
-          continue;
+              return false;
+            });
+            if (!hasChangedData) shouldSkip = true;
+          } else {
+            shouldSkip = true;
+          }
+          break;
+        }
+        case "epoch-changed": {
+          if (requires2.length > 0) {
+            const hasRefreshed = requires2.some((req) => {
+              for (const [otherName, otherConfig] of Object.entries(graphTasks)) {
+                if (getProvides(otherConfig).includes(req)) {
+                  const otherState = state.tasks[otherName];
+                  if (otherState && otherState.executionCount > taskState.lastEpoch) return true;
+                }
+              }
+              return false;
+            });
+            if (!hasRefreshed) shouldSkip = true;
+          } else {
+            shouldSkip = true;
+          }
+          break;
+        }
+        case "time-based": {
+          const interval = taskConfig.refreshInterval ?? 0;
+          if (interval <= 0) {
+            shouldSkip = true;
+            break;
+          }
+          const completedAt = taskState.completedAt;
+          if (!completedAt) {
+            shouldSkip = true;
+            break;
+          }
+          const elapsedSec = (Date.now() - Date.parse(completedAt)) / 1e3;
+          if (elapsedSec < interval) shouldSkip = true;
+          break;
         }
+        case "manual":
+          shouldSkip = true;
+          break;
       }
+      if (shouldSkip) continue;
     }
     const requires = getRequires(taskConfig);
     if (requires.length === 0) {
@@ -910,7 +969,301 @@ function getDownstream(live, nodeName) {
   }));
   return { nodeName, nodes, tokens: [...tokenSet] };
 }
+var MemoryJournal = class {
+  buffer = [];
+  append(event) {
+    this.buffer.push(event);
+  }
+  drain() {
+    const events = this.buffer;
+    this.buffer = [];
+    return events;
+  }
+  get size() {
+    return this.buffer.length;
+  }
+};
+var FileJournal = class {
+  constructor(path) {
+    this.path = path;
+    if (!existsSync(path)) {
+      writeFileSync(path, "", "utf-8");
+    }
+  }
+  path;
+  pending = 0;
+  append(event) {
+    appendFileSync(this.path, JSON.stringify(event) + "\n", "utf-8");
+    this.pending++;
+  }
+  drain() {
+    const content = readFileSync(this.path, "utf-8").trim();
+    writeFileSync(this.path, "", "utf-8");
+    this.pending = 0;
+    if (!content) return [];
+    return content.split("\n").map((line) => JSON.parse(line));
+  }
+  get size() {
+    try {
+      const content = readFileSync(this.path, "utf-8").trim();
+      if (!content) return 0;
+      return content.split("\n").length;
+    } catch {
+      return this.pending;
+    }
+  }
+};
+
+// src/continuous-event-graph/reactive.ts
+function createReactiveGraph(config, options, executionId) {
+  const {
+    handlers: initialHandlers,
+    maxDispatchRetries = 3,
+    defaultTimeoutMs = 3e4,
+    journal = new MemoryJournal(),
+    onDispatchFailed,
+    onAbandoned,
+    onDrain
+  } = options;
+  let live = createLiveGraph(config, executionId);
+  let disposed = false;
+  const handlers = new Map(Object.entries(initialHandlers));
+  const dispatched = /* @__PURE__ */ new Map();
+  const timeoutTimers = /* @__PURE__ */ new Map();
+  let draining = false;
+  let drainQueued = false;
+  function drain() {
+    if (disposed) return;
+    if (draining) {
+      drainQueued = true;
+      return;
+    }
+    draining = true;
+    try {
+      do {
+        drainQueued = false;
+        drainOnce();
+      } while (drainQueued);
+    } finally {
+      draining = false;
+    }
+  }
+  function drainOnce() {
+    sweepTimeouts();
+    const events = journal.drain();
+    for (const event of events) {
+      if (event.type === "task-completed" || event.type === "task-failed") {
+        const taskName = event.taskName;
+        dispatched.delete(taskName);
+        clearTimeout(timeoutTimers.get(taskName));
+        timeoutTimers.delete(taskName);
+      }
+    }
+    if (events.length > 0) {
+      live = applyEvents(live, events);
+    }
+    const result = schedule(live);
+    if (onDrain && events.length > 0) {
+      onDrain(events, live, result);
+    }
+    for (const taskName of result.eligible) {
+      if (dispatched.has(taskName)) continue;
+      dispatchTask(taskName);
+    }
+    for (const [taskName, entry] of dispatched) {
+      if (entry.status === "retry-queued") {
+        dispatchTask(taskName);
+      }
+    }
+  }
+  function dispatchTask(taskName) {
+    const handler = handlers.get(taskName);
+    if (!handler) {
+      journal.append({
+        type: "task-failed",
+        taskName,
+        error: `No handler registered for task "${taskName}"`,
+        timestamp: (/* @__PURE__ */ new Date()).toISOString()
+      });
+      drainQueued = true;
+      return;
+    }
+    const existing = dispatched.get(taskName);
+    const attempt = existing ? existing.dispatchAttempts + 1 : 1;
+    if (attempt > maxDispatchRetries) {
+      dispatched.set(taskName, {
+        status: "abandoned",
+        dispatchedAt: existing?.dispatchedAt ?? Date.now(),
+        dispatchAttempts: attempt - 1,
+        lastError: existing?.lastError
+      });
+      onAbandoned?.(taskName);
+      journal.append({
+        type: "task-failed",
+        taskName,
+        error: `dispatch-abandoned: handler unreachable after ${attempt - 1} attempts${existing?.lastError ? ` (${existing.lastError})` : ""}`,
+        timestamp: (/* @__PURE__ */ new Date()).toISOString()
+      });
+      drainQueued = true;
+      return;
+    }
+    dispatched.set(taskName, {
+      status: "initiated",
+      dispatchedAt: Date.now(),
+      dispatchAttempts: attempt
+    });
+    journal.append({
+      type: "task-started",
+      taskName,
+      timestamp: (/* @__PURE__ */ new Date()).toISOString()
+    });
+    if (defaultTimeoutMs > 0) {
+      const timer = setTimeout(() => {
+        if (disposed) return;
+        const entry = dispatched.get(taskName);
+        if (entry?.status === "initiated") {
+          dispatched.set(taskName, {
+            ...entry,
+            status: "timed-out"
+          });
+          dispatched.set(taskName, {
+            ...entry,
+            status: entry.dispatchAttempts >= maxDispatchRetries ? "abandoned" : "retry-queued"
+          });
+          if (entry.dispatchAttempts >= maxDispatchRetries) {
+            onAbandoned?.(taskName);
+            journal.append({
+              type: "task-failed",
+              taskName,
+              error: `dispatch-timeout: no callback after ${defaultTimeoutMs}ms (${entry.dispatchAttempts} attempts)`,
+              timestamp: (/* @__PURE__ */ new Date()).toISOString()
+            });
+          }
+          drain();
+        }
+      }, defaultTimeoutMs);
+      timeoutTimers.set(taskName, timer);
+    }
+    const ctx = {
+      taskName,
+      live,
+      config: live.config.tasks[taskName]
+    };
+    try {
+      const promise = handler(ctx);
+      promise.then(
+        (handlerResult) => {
+          if (disposed) return;
+          clearTimeout(timeoutTimers.get(taskName));
+          timeoutTimers.delete(taskName);
+          journal.append({
+            type: "task-completed",
+            taskName,
+            result: handlerResult.result,
+            data: handlerResult.data,
+            dataHash: handlerResult.dataHash,
+            timestamp: (/* @__PURE__ */ new Date()).toISOString()
+          });
+          drain();
+        },
+        (error) => {
+          if (disposed) return;
+          clearTimeout(timeoutTimers.get(taskName));
+          timeoutTimers.delete(taskName);
+          journal.append({
+            type: "task-failed",
+            taskName,
+            error: error.message ?? String(error),
+            timestamp: (/* @__PURE__ */ new Date()).toISOString()
+          });
+          drain();
+        }
+      );
+    } catch (syncError) {
+      const err = syncError instanceof Error ? syncError : new Error(String(syncError));
+      dispatched.set(taskName, {
+        status: "dispatch-failed",
+        dispatchedAt: Date.now(),
+        dispatchAttempts: attempt,
+        lastError: err.message
+      });
+      onDispatchFailed?.(taskName, err, attempt);
+      dispatched.set(taskName, {
+        ...dispatched.get(taskName),
+        status: "retry-queued"
+      });
+      drainQueued = true;
+    }
+  }
+  function sweepTimeouts() {
+    const now = Date.now();
+    for (const [taskName, entry] of dispatched) {
+      if (entry.status !== "initiated") continue;
+      if (defaultTimeoutMs <= 0) continue;
+      if (now - entry.dispatchedAt >= defaultTimeoutMs) {
+        dispatched.set(taskName, {
+          ...entry,
+          status: entry.dispatchAttempts >= maxDispatchRetries ? "abandoned" : "retry-queued"
+        });
+        if (entry.dispatchAttempts >= maxDispatchRetries) {
+          onAbandoned?.(taskName);
+          journal.append({
+            type: "task-failed",
+            taskName,
+            error: `dispatch-timeout: no callback after ${defaultTimeoutMs}ms (${entry.dispatchAttempts} attempts)`,
+            timestamp: (/* @__PURE__ */ new Date()).toISOString()
+          });
+        }
+        clearTimeout(timeoutTimers.get(taskName));
+        timeoutTimers.delete(taskName);
+      }
+    }
+  }
+  return {
+    push(event) {
+      if (disposed) return;
+      live = applyEvent(live, event);
+      drain();
+    },
+    pushAll(events) {
+      if (disposed) return;
+      if (events.length === 0) return;
+      live = applyEvents(live, events);
+      drain();
+    },
+    addNode(name, taskConfig, handler) {
+      if (disposed) return;
+      live = addNode(live, name, taskConfig);
+      handlers.set(name, handler);
+      drain();
+    },
+    removeNode(name) {
+      if (disposed) return;
+      live = removeNode(live, name);
+      handlers.delete(name);
+      dispatched.delete(name);
+      clearTimeout(timeoutTimers.get(name));
+      timeoutTimers.delete(name);
+    },
+    getState() {
+      return live;
+    },
+    getSchedule() {
+      return schedule(live);
+    },
+    getDispatchState() {
+      return dispatched;
+    },
+    dispose() {
+      disposed = true;
+      for (const timer of timeoutTimers.values()) {
+        clearTimeout(timer);
+      }
+      timeoutTimers.clear();
+    }
+  };
+}
 
-export { addNode, addProvides, addRequires, applyEvent, createLiveGraph, disableNode, drainTokens, enableNode, getDownstream, getNode, getUnreachableNodes, getUnreachableTokens, getUpstream, injectTokens, inspect, removeNode, removeProvides, removeRequires, resetNode, restore, schedule, snapshot };
+export { FileJournal, MemoryJournal, addNode, addProvides, addRequires, applyEvent, applyEvents, createLiveGraph, createReactiveGraph, disableNode, drainTokens, enableNode, getDownstream, getNode, getUnreachableNodes, getUnreachableTokens, getUpstream, injectTokens, inspect, removeNode, removeProvides, removeRequires, resetNode, restore, schedule, snapshot };
 //# sourceMappingURL=index.js.map
 //# sourceMappingURL=index.js.map