space-data-module-sdk 0.8.3 → 0.8.5

package/src/flow/jsFlowRuntime.js

@@ -0,0 +1,275 @@
+import { normalizeProgramForSdnFlow } from "./normalize.js";
+
+function cloneFrame(frame, portId) {
+  return Object.assign({}, frame, {
+    portId,
+  });
+}
+
+function normalizeQueueDepth(value, fallback = Infinity) {
+  const numeric = Number(value);
+  if (!Number.isFinite(numeric) || numeric <= 0) {
+    return fallback;
+  }
+  return Math.max(1, Math.trunc(numeric));
+}
+
+function applyBackpressure(
+  queue,
+  frames,
+  policy = "queue",
+  queueDepth = Infinity,
+) {
+  const normalizedDepth = normalizeQueueDepth(queueDepth);
+  const normalizedPolicy = String(policy ?? "queue")
+    .trim()
+    .toLowerCase();
+
+  for (const frame of frames) {
+    if (queue.length < normalizedDepth) {
+      queue.push(frame);
+      continue;
+    }
+
+    if (normalizedPolicy === "drop-oldest") {
+      queue.shift();
+      queue.push(frame);
+    } else if (normalizedPolicy !== "drop-newest") {
+      queue.push(frame);
+      if (queue.length > normalizedDepth) {
+        queue.splice(0, queue.length - normalizedDepth);
+      }
+    }
+  }
+}
+
+function getPortQueues(queueMap, nodeId) {
+  let portQueues = queueMap.get(nodeId);
+  if (!portQueues) {
+    portQueues = new Map();
+    queueMap.set(nodeId, portQueues);
+  }
+  return portQueues;
+}
+
+function getFrameQueue(queueMap, nodeId, portId) {
+  const portQueues = getPortQueues(queueMap, nodeId);
+  let queue = portQueues.get(portId);
+  if (!queue) {
+    queue = [];
+    portQueues.set(portId, queue);
+  }
+  return queue;
+}
+
+function snapshotQueues(queueMap) {
+  const snapshot = {};
+  for (const [nodeId, portQueues] of queueMap.entries()) {
+    snapshot[nodeId] = {};
+    for (const [portId, frames] of portQueues.entries()) {
+      snapshot[nodeId][portId] = frames.length;
+    }
+  }
+  return snapshot;
+}
+
+function drainReadyFrames(queue, maxBatch) {
+  if (!Array.isArray(queue) || queue.length === 0) {
+    return [];
+  }
+  return queue.splice(0, Math.min(queue.length, maxBatch));
+}
+
+export class FlowRuntime {
+  #registry;
+
+  #program = null;
+
+  #queueMap = new Map();
+
+  #maxInvocationsPerDrain;
+
+  #onSinkOutput;
+
+  constructor(options = {}) {
+    this.#registry = options.registry ?? null;
+    this.#maxInvocationsPerDrain = Math.max(
+      1,
+      Number(options.maxInvocationsPerDrain ?? 1024) || 1024,
+    );
+    this.#onSinkOutput =
+      typeof options.onSinkOutput === "function" ? options.onSinkOutput : null;
+  }
+
+  loadProgram(program) {
+    this.#program = normalizeProgramForSdnFlow(program);
+    this.#queueMap.clear();
+    return this.#program;
+  }
+
+  getProgram() {
+    return this.#program;
+  }
+
+  inspectQueues() {
+    return snapshotQueues(this.#queueMap);
+  }
+
+  enqueueTriggerFrames(triggerId, frames) {
+    if (!this.#program) {
+      throw new Error("FlowRuntime requires loadProgram() before enqueue.");
+    }
+    const bindings = this.#program.triggerBindings.filter(
+      (binding) => binding.triggerId === triggerId,
+    );
+    for (const binding of bindings) {
+      const queue = getFrameQueue(
+        this.#queueMap,
+        binding.targetNodeId,
+        binding.targetPortId,
+      );
+      applyBackpressure(
+        queue,
+        frames.map((frame) => cloneFrame(frame, binding.targetPortId)),
+        binding.backpressurePolicy,
+        binding.queueDepth,
+      );
+    }
+  }
+
+  enqueueNodeFrames(nodeId, portId, frames, backpressurePolicy, queueDepth) {
+    if (!this.#program) {
+      throw new Error("FlowRuntime requires loadProgram() before enqueue.");
+    }
+    const queue = getFrameQueue(this.#queueMap, nodeId, portId);
+    applyBackpressure(
+      queue,
+      frames.map((frame) => cloneFrame(frame, portId)),
+      backpressurePolicy,
+      queueDepth,
+    );
+  }
+
+  isIdle() {
+    for (const portQueues of this.#queueMap.values()) {
+      for (const frames of portQueues.values()) {
+        if (frames.length > 0) {
+          return false;
+        }
+      }
+    }
+    return true;
+  }
+
+  async drain(options = {}) {
+    if (!this.#program) {
+      throw new Error("FlowRuntime requires loadProgram() before drain.");
+    }
+    if (!this.#registry || typeof this.#registry.invoke !== "function") {
+      throw new Error(
+        "FlowRuntime requires a MethodRegistry-compatible registry.",
+      );
+    }
+
+    const maxInvocations = Math.max(
+      1,
+      Number(options.maxInvocationsPerDrain ?? this.#maxInvocationsPerDrain) ||
+        this.#maxInvocationsPerDrain,
+    );
+    let invocations = 0;
+
+    while (invocations < maxInvocations) {
+      let readyNode = null;
+      let descriptor = null;
+      let inputs = [];
+
+      for (const node of this.#program.nodes ?? []) {
+        descriptor = this.#registry.getMethod(node.pluginId, node.methodId);
+        if (!descriptor) {
+          continue;
+        }
+
+        const maxBatch = Math.max(
+          1,
+          Number(descriptor.method?.maxBatch ?? 1) || 1,
+        );
+        const portQueues = getPortQueues(this.#queueMap, node.nodeId);
+        const candidateInputs = [];
+        let missingRequiredInput = false;
+
+        for (const port of descriptor.method?.inputPorts ?? []) {
+          const queue = portQueues.get(port.portId) ?? [];
+          if (port.required !== false && queue.length === 0) {
+            missingRequiredInput = true;
+            break;
+          }
+          if (queue.length > 0) {
+            candidateInputs.push(...drainReadyFrames(queue, maxBatch));
+          }
+        }
+
+        if (missingRequiredInput || candidateInputs.length === 0) {
+          continue;
+        }
+
+        readyNode = node;
+        inputs = candidateInputs;
+        break;
+      }
+
+      if (!readyNode || !descriptor || inputs.length === 0) {
+        break;
+      }
+
+      const response = await this.#registry.invoke({
+        pluginId: readyNode.pluginId,
+        methodId: readyNode.methodId,
+        inputs,
+        outputStreamCap: Number(options.outputStreamCap ?? 0) || 0,
+      });
+      invocations += 1;
+
+      for (const outputFrame of response.outputs ?? []) {
+        const edges = (this.#program.edges ?? []).filter(
+          (edge) =>
+            edge.fromNodeId === readyNode.nodeId &&
+            edge.fromPortId === outputFrame.portId,
+        );
+        if (edges.length === 0 && this.#onSinkOutput) {
+          await this.#onSinkOutput({
+            frame: outputFrame,
+            node: readyNode,
+            descriptor,
+            runtime: this,
+          });
+          continue;
+        }
+        for (const edge of edges) {
+          const queue = getFrameQueue(
+            this.#queueMap,
+            edge.toNodeId,
+            edge.toPortId,
+          );
+          applyBackpressure(
+            queue,
+            [cloneFrame(outputFrame, edge.toPortId)],
+            edge.backpressurePolicy,
+            edge.queueDepth,
+          );
+        }
+      }
+
+      if (response.yielded === true) {
+        break;
+      }
+    }
+
+    return {
+      invocations,
+      idle: this.isIdle(),
+      queues: this.inspectQueues(),
+    };
+  }
+}
+
+export default FlowRuntime;

package/src/flow/normalize.js

@@ -0,0 +1,294 @@
+import {
+  BackpressurePolicy,
+  FlowProgramT,
+  NodeKind,
+  TriggerKind,
+} from "../generated/orbpro/flow.js";
+import {
+  DrainPolicy,
+  PluginFamily,
+  PluginManifestT,
+} from "../generated/orbpro/manifest.js";
+import { decodeFlowProgram, decodePluginManifestPman } from "./flowCodec.js";
+import {
+  normalizePayloadSchemaHash,
+  normalizePayloadWireFormatName,
+} from "../manifest/typeRefs.js";
+
+function enumSymbol(enumType, value, fallback = null) {
+  if (typeof value === "number" && enumType[value]) {
+    return enumType[value];
+  }
+  if (typeof value === "string" && value.trim() !== "") {
+    return value.trim();
+  }
+  return fallback;
+}
+
+function toKebabEnum(enumType, value, fallback) {
+  const symbol = enumSymbol(enumType, value, null);
+  if (!symbol) {
+    return fallback;
+  }
+  return symbol.toLowerCase().replace(/_/g, "-");
+}
+
+function toPlainTypeRef(typeRef) {
+  const rawSchemaHash = typeRef?.schemaHash ?? typeRef?.SCHEMA_HASH;
+  const schemaHash = normalizePayloadSchemaHash(rawSchemaHash) ?? [];
+  return {
+    schemaName: typeRef?.schemaName ?? typeRef?.SCHEMA_NAME ?? null,
+    fileIdentifier:
+      typeRef?.fileIdentifier ?? typeRef?.FILE_IDENTIFIER ?? null,
+    schemaVersion:
+      typeRef?.schemaVersion ?? typeRef?.SCHEMA_VERSION ?? null,
+    schemaHash,
+    acceptsAnyFlatbuffer: typeRef?.acceptsAnyFlatbuffer === true,
+    // WIRE FORMAT IS A DECLARATION, NEVER A DEFAULT-BY-ACCIDENT.
+    //
+    // This used to read `typeRef?.wireFormat === PayloadWireFormat.AlignedBinary`.
+    // The generated enum member is `ALIGNED_BINARY`; `AlignedBinary` does not
+    // exist, so that expression was `undefined === undefined` — TRUE — for every
+    // type ref that omitted `wireFormat`. The result: every CANONICAL FlatBuffer
+    // port type in every manifest normalized to "aligned-binary", so
+    // concreteTypesByWireFormat(types, "flatbuffer") found ZERO canonical types
+    // on both sides of an edge and resolveEdgeTypeContract fell through to its
+    // null return — reported as the self-contradictory
+    // "CAQ.fbs/$CAQ does not satisfy CAQ.fbs/$CAQ". Five shipped flow bundles
+    // could not be recompiled from main.
+    //
+    // The single canonical parser lives in ../manifest/typeRefs.js and already
+    // handles both the numeric FlatBuffer enum (1) and the string spellings;
+    // absent/empty means the default, "flatbuffer".
+    wireFormat:
+      normalizePayloadWireFormatName(
+        typeRef?.wireFormat ?? typeRef?.wire_format ?? typeRef?.WIRE_FORMAT,
+      ) ?? "flatbuffer",
+    rootTypeName:
+      typeRef?.rootTypeName ?? typeRef?.rootType ?? typeRef?.ROOT_TYPE ?? null,
+    fixedStringLength: Number(typeRef?.fixedStringLength ?? 0),
+    byteLength: Number(typeRef?.byteLength ?? 0),
+    requiredAlignment: Number(typeRef?.requiredAlignment ?? 0),
+  };
+}
+
+function toPlainAcceptedTypeSet(typeSet) {
+  return {
+    setId: typeSet?.setId ?? null,
+    allowedTypes: Array.isArray(typeSet?.allowedTypes)
+      ? typeSet.allowedTypes.map(toPlainTypeRef)
+      : [],
+    description: typeSet?.description ?? null,
+  };
+}
+
+function toPlainPort(port) {
+  return {
+    portId: port?.portId ?? "",
+    displayName: port?.displayName ?? null,
+    acceptedTypeSets: Array.isArray(port?.acceptedTypeSets)
+      ? port.acceptedTypeSets.map(toPlainAcceptedTypeSet)
+      : [],
+    minStreams: Number(port?.minStreams ?? 1),
+    maxStreams: Number(port?.maxStreams ?? 1),
+    required: port?.required !== false,
+    description: port?.description ?? null,
+  };
+}
+
+function toPlainMethod(method) {
+  return {
+    methodId: method?.methodId ?? "",
+    displayName: method?.displayName ?? null,
+    inputPorts: Array.isArray(method?.inputPorts)
+      ? method.inputPorts.map(toPlainPort)
+      : [],
+    outputPorts: Array.isArray(method?.outputPorts)
+      ? method.outputPorts.map(toPlainPort)
+      : [],
+    maxBatch: Number(method?.maxBatch ?? 1),
+    drainPolicy: toKebabEnum(
+      DrainPolicy,
+      method?.drainPolicy,
+      "drain-until-yield",
+    ),
+    description: method?.description ?? null,
+  };
+}
+
+function toPlainBuildArtifact(artifact) {
+  return {
+    artifactId: artifact?.artifactId ?? null,
+    kind: artifact?.kind ?? null,
+    path: artifact?.path ?? null,
+    target: artifact?.target ?? null,
+    entrySymbol: artifact?.entrySymbol ?? null,
+  };
+}
+
+function toPlainCapability(capability) {
+  return capability
+    ? {
+        capability: capability.capability ?? null,
+        scope: capability.scope ?? null,
+        required: capability.required !== false,
+        description: capability.description ?? null,
+      }
+    : null;
+}
+
+function toPlainTimer(timer) {
+  return timer
+    ? {
+        timerId: timer.timerId ?? null,
+        methodId: timer.methodId ?? null,
+        inputPortId: timer.inputPortId ?? null,
+        defaultIntervalMs: timer.defaultIntervalMs ?? 0,
+        description: timer.description ?? null,
+      }
+    : null;
+}
+
+function toPlainProtocol(protocol) {
+  return protocol
+    ? {
+        protocolId: protocol.protocolId ?? null,
+        methodId: protocol.methodId ?? null,
+        inputPortId: protocol.inputPortId ?? null,
+        outputPortId: protocol.outputPortId ?? null,
+        description: protocol.description ?? null,
+      }
+    : null;
+}
+
+function asManifest(manifest) {
+  if (manifest instanceof PluginManifestT) {
+    return manifest;
+  }
+  if (
+    manifest instanceof Uint8Array ||
+    manifest instanceof ArrayBuffer ||
+    ArrayBuffer.isView(manifest)
+  ) {
+    return decodePluginManifestPman(manifest);
+  }
+  return Object.assign(new PluginManifestT(), manifest);
+}
+
+function asProgram(program) {
+  if (program instanceof FlowProgramT) {
+    return program;
+  }
+  if (
+    program instanceof Uint8Array ||
+    program instanceof ArrayBuffer ||
+    ArrayBuffer.isView(program)
+  ) {
+    return decodeFlowProgram(program);
+  }
+  return Object.assign(new FlowProgramT(), program);
+}
+
+export function normalizeManifestForSdnFlow(manifest) {
+  const normalized = asManifest(manifest);
+  return {
+    pluginId: normalized.pluginId ?? "",
+    name: normalized.name ?? null,
+    version: normalized.version ?? null,
+    pluginFamily: enumSymbol(
+      PluginFamily,
+      normalized.pluginFamily,
+      String(normalized.pluginFamily ?? ""),
+    ),
+    methods: Array.isArray(normalized.methods)
+      ? normalized.methods.map(toPlainMethod)
+      : [],
+    capabilities: Array.isArray(normalized.capabilities)
+      ? normalized.capabilities.map(toPlainCapability).filter(Boolean)
+      : [],
+    timers: Array.isArray(normalized.timers)
+      ? normalized.timers.map(toPlainTimer).filter(Boolean)
+      : [],
+    protocols: Array.isArray(normalized.protocols)
+      ? normalized.protocols.map(toPlainProtocol).filter(Boolean)
+      : [],
+    schemasUsed: Array.isArray(normalized.schemasUsed)
+      ? normalized.schemasUsed.map(toPlainTypeRef)
+      : [],
+    buildArtifacts: Array.isArray(normalized.buildArtifacts)
+      ? normalized.buildArtifacts.map(toPlainBuildArtifact)
+      : [],
+    abiVersion: Number(normalized.abiVersion ?? 1),
+  };
+}
+
+export function normalizeProgramForSdnFlow(program) {
+  const normalized = asProgram(program);
+  return {
+    programId: normalized.programId ?? "",
+    name: normalized.name ?? null,
+    version: normalized.version ?? null,
+    nodes: Array.isArray(normalized.nodes)
+      ? normalized.nodes.map((node) => ({
+          nodeId: node?.nodeId ?? "",
+          pluginId: node?.pluginId ?? "",
+          methodId: node?.methodId ?? "",
+          kind: toKebabEnum(NodeKind, node?.kind, "transform"),
+          drainPolicy: toKebabEnum(
+            DrainPolicy,
+            node?.drainPolicy,
+            "drain-until-yield",
+          ),
+          timeSliceMicros: Number(node?.timeSliceMicros ?? 0),
+        }))
+      : [],
+    edges: Array.isArray(normalized.edges)
+      ? normalized.edges.map((edge) => ({
+          edgeId: edge?.edgeId ?? "",
+          fromNodeId: edge?.fromNodeId ?? "",
+          fromPortId: edge?.fromPortId ?? "",
+          toNodeId: edge?.toNodeId ?? "",
+          toPortId: edge?.toPortId ?? "",
+          acceptedTypes: Array.isArray(edge?.acceptedTypes)
+            ? edge.acceptedTypes.map(toPlainTypeRef)
+            : [],
+          backpressurePolicy: toKebabEnum(
+            BackpressurePolicy,
+            edge?.backpressurePolicy,
+            "queue",
+          ),
+          queueDepth: Number(edge?.queueDepth ?? 1),
+        }))
+      : [],
+    triggers: Array.isArray(normalized.triggers)
+      ? normalized.triggers.map((trigger) => ({
+          triggerId: trigger?.triggerId ?? "",
+          kind: toKebabEnum(TriggerKind, trigger?.kind, "manual"),
+          source: trigger?.source ?? null,
+          protocolId: trigger?.protocolId ?? null,
+          defaultIntervalMs: trigger?.defaultIntervalMs ?? 0,
+          acceptedTypes: Array.isArray(trigger?.acceptedTypes)
+            ? trigger.acceptedTypes.map(toPlainTypeRef)
+            : [],
+          description: trigger?.description ?? null,
+        }))
+      : [],
+    triggerBindings: Array.isArray(normalized.triggerBindings)
+      ? normalized.triggerBindings.map((binding) => ({
+          triggerId: binding?.triggerId ?? "",
+          targetNodeId: binding?.targetNodeId ?? "",
+          targetPortId: binding?.targetPortId ?? "",
+          backpressurePolicy: toKebabEnum(
+            BackpressurePolicy,
+            binding?.backpressurePolicy,
+            "queue",
+          ),
+          queueDepth: Number(binding?.queueDepth ?? 1),
+        }))
+      : [],
+    requiredPlugins: Array.isArray(normalized.requiredPlugins)
+      ? [...normalized.requiredPlugins]
+      : [],
+    description: normalized.description ?? null,
+  };
+}