sdn-flow 0.2.0

package/src/index.js

@@ -0,0 +1,8 @@
+export * from "./runtime/index.js";
+export * from "./designer/index.js";
+export * from "./auth/index.js";
+export * from "./host/index.js";
+export * from "./transport/index.js";
+export * from "./deploy/index.js";
+export * from "./compiler/index.js";
+export * from "./compliance/index.js";

package/src/runtime/FlowRuntime.js

@@ -0,0 +1,273 @@
+import { BackpressurePolicy, DrainPolicy } from "./constants.js";
+import { normalizeFrame, normalizeProgram } from "./normalize.js";
+
+// Temporary migration harness used for authoring/tests until the generated C++
+// runtime is the only execution path.
+
+function groupBy(items, keySelector) {
+  const grouped = new Map();
+  for (const item of Array.isArray(items) ? items : []) {
+    const key = keySelector(item);
+    const bucket = grouped.get(key);
+    if (bucket) {
+      bucket.push(item);
+    } else {
+      grouped.set(key, [item]);
+    }
+  }
+  return grouped;
+}
+
+export class FlowRuntime {
+  #registry;
+
+  #program = null;
+
+  #nodes = new Map();
+
+  #triggerBindings = new Map();
+
+  #edgesBySource = new Map();
+
+  #maxInvocationsPerDrain;
+
+  #onSinkOutput;
+
+  constructor(options = {}) {
+    this.#registry = options.registry;
+    if (!this.#registry) {
+      throw new Error("FlowRuntime requires a MethodRegistry instance.");
+    }
+    this.#maxInvocationsPerDrain = options.maxInvocationsPerDrain ?? 1024;
+    this.#onSinkOutput = options.onSinkOutput ?? null;
+  }
+
+  loadProgram(program) {
+    const normalized = normalizeProgram(program);
+    this.#program = normalized;
+    this.#nodes.clear();
+    this.#triggerBindings = groupBy(
+      normalized.triggerBindings,
+      (binding) => binding.triggerId,
+    );
+    this.#edgesBySource = groupBy(
+      normalized.edges,
+      (edge) => `${edge.fromNodeId}:${edge.fromPortId}`,
+    );
+
+    for (const node of normalized.nodes) {
+      this.#nodes.set(node.nodeId, {
+        node,
+        queues: new Map(),
+        invocations: 0,
+      });
+    }
+
+    return normalized;
+  }
+
+  getProgram() {
+    return this.#program;
+  }
+
+  inspectQueues() {
+    const snapshot = {};
+    for (const [nodeId, state] of this.#nodes.entries()) {
+      snapshot[nodeId] = {};
+      for (const [portId, queue] of state.queues.entries()) {
+        snapshot[nodeId][portId] = queue.length;
+      }
+    }
+    return snapshot;
+  }
+
+  enqueueTriggerFrames(triggerId, frames) {
+    const bindings = this.#triggerBindings.get(triggerId) ?? [];
+    if (bindings.length === 0) {
+      throw new Error(`Trigger "${triggerId}" is not bound to any node input.`);
+    }
+    for (const binding of bindings) {
+      this.enqueueNodeFrames(
+        binding.targetNodeId,
+        binding.targetPortId,
+        frames,
+        binding.backpressurePolicy,
+        binding.queueDepth,
+      );
+    }
+  }
+
+  enqueueNodeFrames(
+    nodeId,
+    portId,
+    frames,
+    backpressurePolicy = BackpressurePolicy.QUEUE,
+    queueDepth = 0,
+  ) {
+    const state = this.#nodes.get(nodeId);
+    if (!state) {
+      throw new Error(`Unknown node "${nodeId}".`);
+    }
+    const queue = state.queues.get(portId) ?? [];
+    const normalizedFrames = Array.isArray(frames) ? frames : [frames];
+    for (const frameInput of normalizedFrames) {
+      const frame = Object.assign(normalizeFrame(frameInput), { portId });
+      this.#applyBackpressure(queue, frame, backpressurePolicy, queueDepth);
+    }
+    state.queues.set(portId, queue);
+  }
+
+  async drain(options = {}) {
+    const maxInvocations =
+      options.maxInvocations ?? this.#maxInvocationsPerDrain;
+    let invocations = 0;
+
+    while (invocations < maxInvocations) {
+      const readyState = this.#findReadyNodeState();
+      if (!readyState) {
+        break;
+      }
+      await this.#invokeNode(readyState);
+      invocations += 1;
+    }
+
+    return {
+      invocations,
+      yielded: invocations >= maxInvocations,
+      idle: this.isIdle(),
+    };
+  }
+
+  isIdle() {
+    for (const state of this.#nodes.values()) {
+      for (const queue of state.queues.values()) {
+        if (queue.length > 0) {
+          return false;
+        }
+      }
+    }
+    return true;
+  }
+
+  #findReadyNodeState() {
+    if (!this.#program) {
+      throw new Error("FlowRuntime has no loaded program.");
+    }
+    for (const node of this.#program.nodes) {
+      const state = this.#nodes.get(node.nodeId);
+      const descriptor = this.#registry.getMethod(node.pluginId, node.methodId);
+      if (!state || !descriptor) {
+        continue;
+      }
+      let ready = false;
+      let missingRequiredPort = false;
+      for (const port of descriptor.method.inputPorts) {
+        const queue = state.queues.get(port.portId) ?? [];
+        if (port.required && queue.length === 0) {
+          missingRequiredPort = true;
+          break;
+        }
+        if (queue.length > 0) {
+          ready = true;
+        }
+      }
+      if (!missingRequiredPort && ready) {
+        return state;
+      }
+    }
+    return null;
+  }
+
+  async #invokeNode(state) {
+    const descriptor = this.#registry.getMethod(
+      state.node.pluginId,
+      state.node.methodId,
+    );
+    const maxBatch = Math.max(1, descriptor.method.maxBatch ?? 1);
+    const inputs = [];
+    for (const port of descriptor.method.inputPorts) {
+      const queue = state.queues.get(port.portId) ?? [];
+      const takeCount = Math.min(queue.length, maxBatch);
+      for (let index = 0; index < takeCount; index += 1) {
+        inputs.push(queue.shift());
+      }
+      state.queues.set(port.portId, queue);
+    }
+
+    const drainPolicy =
+      state.node.drainPolicy ??
+      descriptor.method.drainPolicy ??
+      DrainPolicy.DRAIN_UNTIL_YIELD;
+    const response = await this.#registry.invoke({
+      pluginId: state.node.pluginId,
+      methodId: state.node.methodId,
+      inputs,
+      outputStreamCap: 0,
+      drainPolicy,
+      context: {
+        nodeId: state.node.nodeId,
+        programId: this.#program.programId,
+      },
+    });
+
+    state.invocations += 1;
+    this.#routeOutputs(state.node.nodeId, response.outputs ?? []);
+  }
+
+  #routeOutputs(nodeId, outputs) {
+    for (const frame of outputs) {
+      const sourceKey = `${nodeId}:${frame.portId}`;
+      const edges = this.#edgesBySource.get(sourceKey) ?? [];
+      if (edges.length === 0) {
+        if (typeof this.#onSinkOutput === "function") {
+          this.#onSinkOutput({ nodeId, frame });
+        }
+        continue;
+      }
+      for (const edge of edges) {
+        this.enqueueNodeFrames(
+          edge.toNodeId,
+          edge.toPortId,
+          [frame],
+          edge.backpressurePolicy,
+          edge.queueDepth,
+        );
+      }
+    }
+  }
+
+  #applyBackpressure(queue, frame, policy, queueDepth) {
+    const cap = Number(queueDepth ?? 0);
+    const bounded = cap > 0;
+    switch (policy) {
+      case BackpressurePolicy.DROP:
+        if (!bounded || queue.length < cap) {
+          queue.push(frame);
+        }
+        return;
+      case BackpressurePolicy.LATEST:
+      case BackpressurePolicy.COALESCE:
+        if (!bounded || queue.length < cap) {
+          queue.push(frame);
+        } else {
+          queue.splice(0, queue.length, frame);
+        }
+        return;
+      case BackpressurePolicy.BLOCK_REQUEST:
+        if (bounded && queue.length >= cap) {
+          throw new Error("Backpressure queue is full.");
+        }
+        queue.push(frame);
+        return;
+      case BackpressurePolicy.DRAIN_TO_EMPTY:
+      case BackpressurePolicy.QUEUE:
+      default:
+        if (bounded && queue.length >= cap) {
+          queue.shift();
+        }
+        queue.push(frame);
+    }
+  }
+}
+
+export default FlowRuntime;

package/src/runtime/MethodRegistry.js

@@ -0,0 +1,295 @@
+import { DrainPolicy } from "./constants.js";
+import { normalizeFrame, normalizeManifest } from "./normalize.js";
+
+function groupFramesByPort(frames) {
+  const grouped = new Map();
+  for (const frame of frames) {
+    if (!frame.portId) {
+      throw new Error("Flow frame is missing portId.");
+    }
+    const bucket = grouped.get(frame.portId);
+    if (bucket) {
+      bucket.push(frame);
+    } else {
+      grouped.set(frame.portId, [frame]);
+    }
+  }
+  return grouped;
+}
+
+function countDistinctStreams(frames) {
+  const streamIds = new Set();
+  for (const frame of frames) {
+    streamIds.add(Number(frame.streamId ?? 0));
+  }
+  return streamIds.size;
+}
+
+function buildPortMap(ports) {
+  const map = new Map();
+  for (const port of ports) {
+    map.set(port.portId, port);
+  }
+  return map;
+}
+
+function typeMatches(acceptedType, frameType) {
+  if (!acceptedType) {
+    return false;
+  }
+  if (acceptedType.acceptsAnyFlatbuffer === true) {
+    return true;
+  }
+  if (!frameType) {
+    return false;
+  }
+  if (
+    acceptedType.schemaName &&
+    acceptedType.schemaName !== frameType.schemaName
+  ) {
+    return false;
+  }
+  if (
+    acceptedType.fileIdentifier &&
+    acceptedType.fileIdentifier !== frameType.fileIdentifier
+  ) {
+    return false;
+  }
+  if (
+    Array.isArray(acceptedType.schemaHash) &&
+    acceptedType.schemaHash.length > 0
+  ) {
+    if (
+      !Array.isArray(frameType.schemaHash) ||
+      frameType.schemaHash.length !== acceptedType.schemaHash.length
+    ) {
+      return false;
+    }
+    for (let index = 0; index < acceptedType.schemaHash.length; index += 1) {
+      if (acceptedType.schemaHash[index] !== frameType.schemaHash[index]) {
+        return false;
+      }
+    }
+  }
+  return true;
+}
+
+function portAcceptsFrame(port, frame) {
+  const acceptedTypeSets = Array.isArray(port.acceptedTypeSets)
+    ? port.acceptedTypeSets
+    : [];
+  if (acceptedTypeSets.length === 0) {
+    return true;
+  }
+  for (const typeSet of acceptedTypeSets) {
+    const allowedTypes = Array.isArray(typeSet.allowedTypes)
+      ? typeSet.allowedTypes
+      : [];
+    for (const acceptedType of allowedTypes) {
+      if (typeMatches(acceptedType, frame.typeRef)) {
+        return true;
+      }
+    }
+  }
+  return false;
+}
+
+function hydrateOutputPorts(response, method, outputStreamCap) {
+  const outputs = Array.isArray(response.outputs) ? response.outputs : [];
+  if (outputStreamCap > 0 && outputs.length > outputStreamCap) {
+    throw new Error(
+      `Method "${method.methodId}" produced ${outputs.length} output frames, exceeding outputStreamCap ${outputStreamCap}.`,
+    );
+  }
+  if (method.outputPorts.length === 1) {
+    const onlyPortId = method.outputPorts[0].portId;
+    for (const frame of outputs) {
+      if (!frame.portId) {
+        frame.portId = onlyPortId;
+      }
+    }
+  }
+  for (const frame of outputs) {
+    if (!frame.portId) {
+      throw new Error(
+        `Method "${method.methodId}" produced a frame without portId and has multiple output ports.`,
+      );
+    }
+  }
+  return {
+    outputs,
+    backlogRemaining: Number(response.backlogRemaining ?? 0),
+    yielded: response.yielded === true,
+    errorCode: Number(response.errorCode ?? 0),
+    errorMessage: response.errorMessage ?? null,
+  };
+}
+
+export class MethodRegistry {
+  #plugins = new Map();
+
+  #methods = new Map();
+
+  registerPlugin({ manifest, handlers = {}, plugin = null }) {
+    const normalizedManifest = normalizeManifest(manifest);
+    if (!normalizedManifest.pluginId) {
+      throw new Error("Plugin manifest is missing pluginId.");
+    }
+    if (this.#plugins.has(normalizedManifest.pluginId)) {
+      throw new Error(
+        `Plugin "${normalizedManifest.pluginId}" is already registered.`,
+      );
+    }
+
+    const methodMap = new Map();
+    for (const method of normalizedManifest.methods) {
+      if (!method.methodId) {
+        throw new Error(
+          `Plugin "${normalizedManifest.pluginId}" contains a method without methodId.`,
+        );
+      }
+      const handler = handlers[method.methodId];
+      if (typeof handler !== "function") {
+        throw new Error(
+          `Plugin "${normalizedManifest.pluginId}" is missing a handler for method "${method.methodId}".`,
+        );
+      }
+      const descriptor = {
+        pluginId: normalizedManifest.pluginId,
+        manifest: normalizedManifest,
+        method,
+        handler,
+        plugin,
+        inputPorts: buildPortMap(method.inputPorts),
+        outputPorts: buildPortMap(method.outputPorts),
+      };
+      methodMap.set(method.methodId, descriptor);
+      this.#methods.set(
+        `${normalizedManifest.pluginId}:${method.methodId}`,
+        descriptor,
+      );
+    }
+
+    const record = {
+      pluginId: normalizedManifest.pluginId,
+      manifest: normalizedManifest,
+      methods: methodMap,
+      plugin,
+    };
+    this.#plugins.set(normalizedManifest.pluginId, record);
+    return record;
+  }
+
+  unregisterPlugin(pluginId) {
+    const record = this.#plugins.get(pluginId);
+    if (!record) {
+      return false;
+    }
+
+    this.#plugins.delete(pluginId);
+    for (const methodId of record.methods.keys()) {
+      this.#methods.delete(`${pluginId}:${methodId}`);
+    }
+    return true;
+  }
+
+  getPlugin(pluginId) {
+    return this.#plugins.get(pluginId) ?? null;
+  }
+
+  getMethod(pluginId, methodId) {
+    return this.#methods.get(`${pluginId}:${methodId}`) ?? null;
+  }
+
+  listPlugins() {
+    return Array.from(this.#plugins.values());
+  }
+
+  async invoke({
+    pluginId,
+    methodId,
+    inputs = [],
+    outputStreamCap = 0,
+    drainPolicy = undefined,
+    context = undefined,
+  }) {
+    const descriptor = this.getMethod(pluginId, methodId);
+    if (!descriptor) {
+      throw new Error(`Unknown method "${pluginId}:${methodId}".`);
+    }
+
+    const normalizedInputs = Array.isArray(inputs)
+      ? inputs.map((frame) => normalizeFrame(frame))
+      : [];
+    const inputsByPort = groupFramesByPort(normalizedInputs);
+
+    for (const [portId, port] of descriptor.inputPorts.entries()) {
+      const frames = inputsByPort.get(portId) ?? [];
+      if (port.required && frames.length === 0) {
+        throw new Error(
+          `Method "${pluginId}:${methodId}" requires input port "${portId}".`,
+        );
+      }
+      if (frames.length === 0) {
+        continue;
+      }
+      const distinctStreams = countDistinctStreams(frames);
+      if (distinctStreams < port.minStreams) {
+        throw new Error(
+          `Input port "${portId}" requires at least ${port.minStreams} stream(s).`,
+        );
+      }
+      if (port.maxStreams > 0 && distinctStreams > port.maxStreams) {
+        throw new Error(
+          `Input port "${portId}" allows at most ${port.maxStreams} stream(s).`,
+        );
+      }
+      for (const frame of frames) {
+        if (!portAcceptsFrame(port, frame)) {
+          const schemaName =
+            frame.typeRef?.schemaName ??
+            frame.typeRef?.fileIdentifier ??
+            "<unknown>";
+          throw new Error(
+            `Input port "${portId}" rejected frame type "${schemaName}".`,
+          );
+        }
+      }
+    }
+
+    for (const portId of inputsByPort.keys()) {
+      if (!descriptor.inputPorts.has(portId)) {
+        throw new Error(
+          `Method "${pluginId}:${methodId}" does not declare input port "${portId}".`,
+        );
+      }
+    }
+
+    const requestedDrainPolicy =
+      drainPolicy ??
+      descriptor.method.drainPolicy ??
+      DrainPolicy.DRAIN_UNTIL_YIELD;
+
+    const result = await descriptor.handler({
+      pluginId,
+      methodId,
+      manifest: descriptor.manifest,
+      method: descriptor.method,
+      plugin: descriptor.plugin,
+      inputs: normalizedInputs,
+      inputsByPort,
+      outputStreamCap,
+      drainPolicy: requestedDrainPolicy,
+      context,
+    });
+
+    return hydrateOutputPorts(result ?? {}, descriptor.method, outputStreamCap);
+  }
+
+  clear() {
+    this.#plugins.clear();
+    this.#methods.clear();
+  }
+}
+
+export default MethodRegistry;

package/src/runtime/constants.js

@@ -0,0 +1,44 @@
+import {
+  DefaultManifestExports as SharedDefaultManifestExports,
+  DrainPolicy as SharedDrainPolicy,
+  ExternalInterfaceDirection as SharedExternalInterfaceDirection,
+  ExternalInterfaceKind as SharedExternalInterfaceKind,
+  RuntimeTarget as SharedRuntimeTarget,
+} from "space-data-module-sdk";
+
+export const DrainPolicy = SharedDrainPolicy;
+
+export const BackpressurePolicy = Object.freeze({
+  DROP: "drop",
+  LATEST: "latest",
+  QUEUE: "queue",
+  BLOCK_REQUEST: "block-request",
+  COALESCE: "coalesce",
+  DRAIN_TO_EMPTY: "drain-to-empty",
+});
+
+export const TriggerKind = Object.freeze({
+  MANUAL: "manual",
+  TIMER: "timer",
+  PUBSUB_SUBSCRIPTION: "pubsub-subscription",
+  PROTOCOL_REQUEST: "protocol-request",
+  HTTP_REQUEST: "http-request",
+  ORBPRO_EVENT: "orbpro-event",
+  SCENE_EVENT: "scene-event",
+  SYSTEM_EVENT: "system-event",
+});
+
+export const NodeKind = Object.freeze({
+  TRIGGER: "trigger",
+  TRANSFORM: "transform",
+  ANALYZER: "analyzer",
+  PUBLISHER: "publisher",
+  RESPONDER: "responder",
+  RENDERER: "renderer",
+  SINK: "sink",
+});
+
+export const ExternalInterfaceDirection = SharedExternalInterfaceDirection;
+export const ExternalInterfaceKind = SharedExternalInterfaceKind;
+export const RuntimeTarget = SharedRuntimeTarget;
+export const DefaultManifestExports = SharedDefaultManifestExports;

package/src/runtime/index.js

@@ -0,0 +1,19 @@
+export {
+  BackpressurePolicy,
+  DefaultManifestExports,
+  DrainPolicy,
+  ExternalInterfaceDirection,
+  ExternalInterfaceKind,
+  NodeKind,
+  RuntimeTarget,
+  TriggerKind,
+} from "./constants.js";
+export {
+  normalizeArtifactDependency,
+  normalizeExternalInterface,
+  normalizeFrame,
+  normalizeManifest,
+  normalizeProgram,
+} from "./normalize.js";
+export { MethodRegistry } from "./MethodRegistry.js";
+export { FlowRuntime } from "./FlowRuntime.js";