stream-chat-react-native-core 9.12.1 → 9.13.0

package/src/ai-components/generative-ui/AIUIHost.ts

@@ -0,0 +1,358 @@
+import type { ReactNode } from 'react';
+import { Linking } from 'react-native';
+
+import { type AIUIAction, aiUIActionProblem } from './AIUIAction';
+import type { AIUIComponent } from './AIUIComponent';
+import { aiUIComponentDefinitionJSON } from './AIUIComponentDefinition';
+import { type AIUIDocument, isSameAIUIDocument } from './AIUIDocument';
+import { aiUILinkScheme } from './AIUIHelpers';
+import { type AIUIEngine, type AIUIRenderer, aiUIRendererReads } from './AIUIRenderer';
+
+import type { AIMessage } from '../conversation/types';
+import type { AIToolCallPart } from '../message-parts/types';
+import type { AIClientTool, AIClientToolDefinition } from '../tool-calls/AIClientToolRunner';
+
+/**
+ * Why one of the app's tools didn't run for a reply's UI, or failed, in words that name the tool.
+ */
+export class AIUIToolError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = 'AIUIToolError';
+  }
+}
+
+let toolRuns = 0;
+
+export type AIUIHostOptions = {
+  /** The renderer of each format the agent writes, the first one listed asked first. */
+  renderers: AIUIRenderer[];
+  /**
+   * Sends what the person did to the agent, as their next message, such as
+   * `session.responses.create(action.label, { uiAction: aiUIActionJSON(action) })`.
+   */
+  send: (action: AIUIAction) => Promise<unknown> | unknown;
+  /** The app's views the agent can place. */
+  components?: AIUIComponent[];
+  /** Tells the app that an action could not be sent. It is logged by default. */
+  onError?: (error: unknown, action: AIUIAction) => void;
+  /**
+   * Tells the app that a reply's UI, or a part of it, failed to show: a reply's UI that throws
+   * shows the reply's text instead, and one of the app's components that throws leaves only
+   * itself out. It is logged by default.
+   */
+  onRenderError?: (error: unknown, messageId: string) => void;
+  /**
+   * Opens a link from a reply's UI. By default, web and email links open and others don't: the
+   * agent writes the links, so another could call someone or act in another app.
+   */
+  open?: (url: string) => void;
+  /**
+   * The app's tools a reply's UI can run, the same ones an `AIClientToolRunner` takes. A tool
+   * whose definition doesn't say it is read-only (`annotations.readOnlyHint`) runs only when the
+   * person asks for it, such as with a tap, never by itself as a reply shows.
+   */
+  tools?: AIClientTool[];
+};
+
+const WEB_OR_EMAIL = new Set(['http', 'https', 'mailto']);
+
+/**
+ * Whether a link is one a reply's UI may open by default: a web link with a host, or an email
+ * link.
+ */
+export const isWebOrEmailLink = (url: string) => {
+  const scheme = aiUILinkScheme(url);
+  return scheme !== undefined && WEB_OR_EMAIL.has(scheme);
+};
+
+const openWebOrEmailLink = (url: string) => {
+  if (!isWebOrEmailLink(url)) {
+    console.warn(`A reply's UI asked to open ${url}, which isn't a web or email link`);
+    return;
+  }
+  Linking.openURL(url.trim()).catch(() => undefined);
+};
+
+/**
+ * The generated UI of one conversation: the renderers of the formats the agent writes, the
+ * app's components and tools, what each reply's UI was left in, and where what the person does
+ * goes. Give it to `AIConversation` as `ui`; keep one per conversation, for as long as it shows.
+ */
+export class AIUIHost {
+  readonly components: AIUIComponent[];
+  readonly renderers: AIUIRenderer[];
+
+  private readonly engines = new Map<AIUIRenderer, AIUIEngine>();
+  private readonly failed = new Set<string>();
+  private lastSend: Promise<void> = Promise.resolve();
+  private readonly listeners = new Set<() => void>();
+  private readonly options: AIUIHostOptions;
+  private readonly received = new Map<string, AIUIDocument>();
+  private readonly states = new Map<string, unknown>();
+  private readonly tools: Map<string, AIClientTool>;
+  private refreshing = false;
+  private version = 0;
+
+  constructor(options: AIUIHostOptions) {
+    this.options = options;
+    this.renderers = options.renderers;
+    this.components = options.components ?? [];
+    this.tools = new Map();
+    for (const tool of options.tools ?? []) {
+      if (!this.tools.has(tool.definition.name)) this.tools.set(tool.definition.name, tool);
+    }
+  }
+
+  /** The names of the app's tools a reply's UI can run with `runTool`, sorted. */
+  get toolNames() {
+    return [...this.tools.keys()].sort();
+  }
+
+  /** The app's tools a reply's UI can run, as the agent is told them, sorted by name. */
+  get toolDefinitions(): AIClientToolDefinition[] {
+    return this.toolNames.map((name) => this.tools.get(name)!.definition);
+  }
+
+  /** The app's component the agent places by `name`. */
+  component(name: string) {
+    return this.components.find((component) => component.definition.name === name);
+  }
+
+  /**
+   * Whether one of its renderers reads `document`. The one that does can still show nothing of
+   * its own for a reply, which then shows its text, as `content` tells.
+   */
+  canShow(document: AIUIDocument) {
+    return this.renderers.some((renderer) => aiUIRendererReads(renderer, document));
+  }
+
+  /**
+   * The view of a reply's document from the first renderer that reads its format and version,
+   * or `null` when none shows anything of its own, so the reply shows its text. `AIUIView` puts
+   * it on screen; `AIConversation` does it for every reply.
+   */
+  content(document: AIUIDocument, messageId: string): ReactNode | null {
+    if (this.failed.has(messageId)) return null;
+    const renderer = this.renderers.find((candidate) => aiUIRendererReads(candidate, document));
+    return renderer ? (this.engineOf(renderer).view(document, messageId) ?? null) : null;
+  }
+
+  /**
+   * Gives its renderers the UI of the conversation's replies, oldest first, as each arrives or
+   * changes, whether or not the reply is on screen, so a format whose later replies change an
+   * earlier one's UI follows them all. It also tells them when a reply leaves the conversation,
+   * or its UI does, such as when it is deleted. `AIConversation` does it: call it with the
+   * conversation's messages when you show replies yourself with `AIUIView`.
+   */
+  receive(messages: AIMessage[]) {
+    const documents = new Map<string, AIUIDocument>();
+    for (const message of messages) {
+      if (message.role !== 'user' && message.uiDocument) {
+        documents.set(message.id, message.uiDocument);
+      }
+    }
+    const left: [string, AIUIDocument][] = [];
+    for (const [messageId, document] of [...this.received]) {
+      if (documents.has(messageId)) continue;
+      this.received.delete(messageId);
+      left.push([messageId, document]);
+    }
+    for (const [messageId, document] of documents) {
+      const before = this.received.get(messageId);
+      if (isSameAIUIDocument(before, document)) continue;
+      this.received.set(messageId, document);
+      // A reply whose UI failed while it was written gets one more try once it is written.
+      if (before && !document.isStreaming) this.failed.delete(messageId);
+      const renderer = this.renderers.find((candidate) => aiUIRendererReads(candidate, document));
+      if (renderer) this.engineOf(renderer).receive?.(document, messageId);
+    }
+    // Forgets the replies that left once the conversation has rendered without them, since
+    // this runs while it renders, and a render React throws away must not lose what a reply
+    // was left in.
+    if (left.length) {
+      queueMicrotask(() => {
+        for (const [messageId, document] of left) {
+          if (this.received.has(messageId)) continue;
+          this.states.delete(messageId);
+          this.failed.delete(messageId);
+          const renderer = this.renderers.find((candidate) =>
+            aiUIRendererReads(candidate, document),
+          );
+          if (renderer) this.engineOf(renderer).remove?.(messageId);
+        }
+      });
+    }
+  }
+
+  /**
+   * Sends what the person did to the agent, after everything sent before it, so the agent gets
+   * a reply's actions in the order they happened. Resolves to whether it was sent: one the
+   * agent's backend would refuse, or that fails to send, goes to `onError`, resolves to
+   * `false`, and does not hold up the ones after it.
+   */
+  send(action: AIUIAction): Promise<boolean> {
+    const report = (error: unknown) => {
+      if (this.options.onError) this.options.onError(error, action);
+      else console.error("Couldn't send what the person did in a reply's UI", error);
+    };
+    const problem = aiUIActionProblem(action);
+    if (problem) {
+      report(new Error(problem));
+      return Promise.resolve(false);
+    }
+    const sent = this.lastSend.then(async () => {
+      try {
+        await this.options.send(action);
+        return true;
+      } catch (error) {
+        report(error);
+        return false;
+      }
+    });
+    this.lastSend = sent.then(() => undefined);
+    return sent;
+  }
+
+  /**
+   * Shows a reply's text in place of its UI, which failed to show, and tells the app. It lasts
+   * while the reply is in the conversation, or, for a reply still being written, until it is
+   * written. `AIConversation` and `AIUIView` do it.
+   */
+  fail(messageId: string, error: unknown) {
+    if (this.failed.has(messageId)) return;
+    this.failed.add(messageId);
+    if (this.options.onRenderError) this.options.onRenderError(error, messageId);
+    else console.error(`A reply's UI failed to show, so its text shows instead`, error);
+    this.refresh();
+  }
+
+  /**
+   * Tells the app that a part of a reply's UI failed to show, such as one of the app's
+   * components that threw, while the rest of it still shows. `AIUIComponentView` calls it.
+   */
+  reportRenderError(error: unknown, messageId: string) {
+    if (this.options.onRenderError) this.options.onRenderError(error, messageId);
+    else console.error(`A part of a reply's UI failed to show, so it is left out`, error);
+  }
+
+  /**
+   * Has the conversation show its replies again, because what one of them shows changed since
+   * it was last asked, such as a finished reply that turned out to show nothing of its own.
+   * Renderers call it; the conversation shows them again after the current render, however many
+   * times it is called in it.
+   */
+  refresh() {
+    if (this.refreshing) return;
+    this.refreshing = true;
+    queueMicrotask(() => {
+      this.refreshing = false;
+      this.version++;
+      this.listeners.forEach((listener) => listener());
+    });
+  }
+
+  /** Calls `listener` each time the conversation should show its replies again. */
+  subscribe = (listener: () => void) => {
+    this.listeners.add(listener);
+    return () => {
+      this.listeners.delete(listener);
+    };
+  };
+
+  /** A number that changes each time the conversation should show its replies again. */
+  getVersion = () => this.version;
+
+  /**
+   * Runs one of the app's tools for a reply's UI, such as to load data on this device, and
+   * resolves to its output. A tool that isn't read-only runs only when `userInitiated`, from
+   * something the person did. Rejects with an `AIUIToolError` when the tool can't run here, fails,
+   * or gives output that isn't JSON.
+   */
+  async runTool(
+    name: string,
+    args: Record<string, unknown> = {},
+    { userInitiated = false }: { userInitiated?: boolean } = {},
+  ): Promise<unknown> {
+    const tool = this.tools.get(name);
+    if (!tool) throw new AIUIToolError(`No tool named ${name}`);
+    if (!userInitiated && tool.definition.annotations?.readOnlyHint !== true) {
+      throw new AIUIToolError(
+        `${name} isn't read-only, so it runs only when the person asks for it`,
+      );
+    }
+    const call: AIToolCallPart = {
+      arguments: args,
+      executor: 'client',
+      id: `ui-${name}-${++toolRuns}`,
+      name,
+      status: 'awaiting_client',
+    };
+    let result;
+    try {
+      result = await tool.run(call);
+    } catch (error) {
+      throw new AIUIToolError(
+        `${name} failed${error instanceof Error && error.message ? `: ${error.message}` : ''}`,
+      );
+    }
+    if (result.failure !== undefined) throw new AIUIToolError(result.failure);
+    if (result.output === undefined) return undefined;
+    try {
+      const json = JSON.stringify(result.output);
+      if (json !== undefined) return JSON.parse(json) as unknown;
+    } catch {
+      // Not JSON, such as a value that refers to itself.
+    }
+    throw new AIUIToolError(`${name} gave output that isn't JSON`);
+  }
+
+  /** Opens a link from a reply's UI, as the host's `open` allows. */
+  open(url: string) {
+    if (aiUILinkScheme(url) === undefined) {
+      console.warn(`A reply's UI asked to open ${url}, which isn't a link`);
+      return;
+    }
+    (this.options.open ?? openWebOrEmailLink)(url.trim());
+  }
+
+  /** The state a reply's UI was left in, such as a form's values, kept while the host is. */
+  state(messageId: string): unknown {
+    return this.states.get(messageId);
+  }
+
+  /** Keeps the state a reply's UI was left in. `undefined` forgets it. */
+  setState(messageId: string, state: unknown) {
+    if (state === undefined) this.states.delete(messageId);
+    else this.states.set(messageId, state);
+  }
+
+  /**
+   * What the agent's backend needs to write UI this app shows: each format it reads, with what
+   * its renderer adds, such as the model's instructions, and the app's components and tools.
+   */
+  async capabilities() {
+    return {
+      components: this.components.map(({ definition }) => aiUIComponentDefinitionJSON(definition)),
+      // The app's tools as Model Context Protocol tools.
+      tools: this.toolDefinitions,
+      renderers: await Promise.all(
+        this.renderers.map(async (renderer) => ({
+          ...(await renderer.capabilities?.(this.components, this.toolDefinitions)),
+          format: renderer.format,
+        })),
+      ),
+      v: 1,
+    };
+  }
+
+  // The engine of a renderer, made the first time it is needed and kept as the host is.
+  private engineOf(renderer: AIUIRenderer) {
+    let engine = this.engines.get(renderer);
+    if (!engine) {
+      engine = renderer.createEngine(this);
+      this.engines.set(renderer, engine);
+    }
+    return engine;
+  }
+}

package/src/ai-components/generative-ui/AIUIMedia.ts

@@ -0,0 +1,45 @@
+import { aiUILinkScheme } from './AIUIHelpers';
+
+/**
+ * Decides what a reply's UI loads for an image, video or sound the agent linked to: the address
+ * to load, which may be another, or `null` not to load it. The agent writes these links, so one
+ * could track the person or reach their network.
+ */
+export type AIUIMediaURL = (url: string) => string | null | undefined;
+
+// A host on this device or its network, as a link names it.
+const PRIVATE_HOST =
+  /^(localhost|.*\.localhost|.*\.local|.*\.internal|0\.0\.0\.0|127\.\d+\.\d+\.\d+|10\.\d+\.\d+\.\d+|192\.168\.\d+\.\d+|169\.254\.\d+\.\d+|172\.(1[6-9]|2\d|3[01])\.\d+\.\d+|\[(::1?|f[cd][0-9a-f]*:.*|fe[89ab][0-9a-f]*:.*|::ffff:.*)\])$/i;
+
+// The host of a web link, past any user info and before its port: `//user@host:443/path`.
+const HOST = /^[a-z][a-z0-9+.-]*:\/\/(?:[^@/?#]*@)?(\[[^\]/?#]*\]|[^:/?#]*)/i;
+
+/**
+ * Whether `url` is a public web link: http or https, to a host that isn't this device or its
+ * network, such as `localhost` or `192.168.1.1`.
+ */
+export const isPublicWebURL = (url: string) => {
+  const scheme = aiUILinkScheme(url);
+  if (scheme !== 'http' && scheme !== 'https') return false;
+  const host = HOST.exec(url.trim())?.[1]?.replace(/\.$/, '');
+  return !!host && !PRIVATE_HOST.test(host);
+};
+
+/**
+ * The media a reply's UI loads by default: public web links. Others, such as `data:` or `file:`
+ * URLs, or addresses on the person's network, don't load.
+ */
+export const aiUIPublicMediaURL: AIUIMediaURL = (url) => (isPublicWebURL(url) ? url : null);
+
+/**
+ * The address to load for a link the agent wrote, as `mediaURL` decides it, or `undefined` not to
+ * load anything. Renderers call it before they draw an image, video or sound.
+ */
+export const aiUIMediaSource = (
+  value: unknown,
+  mediaURL: AIUIMediaURL = aiUIPublicMediaURL,
+): string | undefined => {
+  if (typeof value !== 'string' || value.trim() === '') return undefined;
+  const decided = mediaURL(value.trim());
+  return typeof decided === 'string' && decided.trim() !== '' ? decided : undefined;
+};

package/src/ai-components/generative-ui/AIUIRenderer.ts

@@ -0,0 +1,60 @@
+import type { ReactNode } from 'react';
+
+import type { AIUIComponent } from './AIUIComponent';
+import type { AIUIDocument, AIUIFormat } from './AIUIDocument';
+import type { AIUIHost } from './AIUIHost';
+
+import type { AIClientToolDefinition } from '../tool-calls/AIClientToolRunner';
+
+/**
+ * Shows the documents of one conversation in a renderer's format. A host makes one per
+ * renderer, the first time the renderer receives or shows something, and keeps it as long as
+ * the host lives, so what a reply's UI holds, such as a form's values, survives scrolling.
+ */
+export type AIUIEngine = {
+  /**
+   * The view of a reply's document, or `null` when it shows nothing of its own, such as a reply
+   * that only changed an earlier one's UI, so the reply's text shows. It is called while the
+   * conversation renders, so keep what it makes for a reply and return it again.
+   */
+  view: (document: AIUIDocument, messageId: string) => ReactNode | null;
+  /**
+   * Takes a reply's document as it arrives or changes, oldest reply first, whether or not the
+   * reply is on screen, so a format whose later replies change an earlier one's UI can follow
+   * them all. It comes while the conversation renders, before any reply's view is asked for:
+   * change what is already on screen afterwards, such as in a microtask.
+   */
+  receive?: (document: AIUIDocument, messageId: string) => void;
+  /**
+   * Forgets a reply that left the conversation, or whose UI did, such as one that was deleted,
+   * so the engine can let go of what it kept for it.
+   */
+  remove?: (messageId: string) => void;
+};
+
+/**
+ * Reads one format of generated UI, such as A2UI. A plug-in of `AIUIHost`: the AI components
+ * know no format themselves.
+ */
+export type AIUIRenderer = {
+  /** Makes what shows the documents of one conversation. */
+  createEngine: (host: AIUIHost) => AIUIEngine;
+  /** The format it reads. */
+  format: AIUIFormat;
+  /**
+   * What the agent's backend needs to have its model write UI this renderer shows with
+   * `components`, such as the model's instructions, which may take loading the format's
+   * packages to write. None by default.
+   */
+  capabilities?: (
+    components: AIUIComponent[],
+    /** The app's tools a reply's UI can run, which the agent can be told about too. */
+    tools: AIClientToolDefinition[],
+  ) => Record<string, unknown> | Promise<Record<string, unknown>>;
+  /** Whether it reads documents of `version`, or of no version. Every version by default. */
+  supports?: (version: string | undefined) => boolean;
+};
+
+/** Whether `renderer` reads `document`: its format, in a version it supports. */
+export const aiUIRendererReads = (renderer: AIUIRenderer, document: AIUIDocument) =>
+  renderer.format === document.format && (renderer.supports?.(document.version) ?? true);

package/src/ai-components/generative-ui/AIUIView.tsx

@@ -0,0 +1,87 @@
+import React, { type ReactNode, useSyncExternalStore } from 'react';
+import { StyleSheet } from 'react-native';
+
+import type { AIUIDocument } from './AIUIDocument';
+import { AIUIEntranceStage } from './AIUIEntrance';
+import { AIUIErrorBoundary } from './AIUIErrorBoundary';
+import type { AIUIHost } from './AIUIHost';
+
+import { useTheme } from '../contexts';
+
+/**
+ * A reply's generated UI as it shows in place of the reply's text: as wide as the reply, coming
+ * in as it first shows while the agent writes it. UI that throws while it renders shows nothing,
+ * and `onError` is told, such as to have the host show the reply's text. An app's component
+ * that throws only leaves itself out.
+ */
+export const AIUIGenerated = ({
+  children,
+  isStreaming,
+  messageId,
+  onError,
+}: {
+  children: ReactNode;
+  isStreaming: boolean;
+  messageId: string;
+  onError: (error: unknown) => void;
+}) => {
+  const {
+    theme: { generatedUI },
+  } = useTheme();
+  return (
+    <AIUIEntranceStage
+      isStreaming={isStreaming}
+      resetKey={messageId}
+      style={[styles.generated, generatedUI?.container]}
+    >
+      <AIUIErrorBoundary onError={onError} resetKey={messageId}>
+        {children}
+      </AIUIErrorBoundary>
+    </AIUIEntranceStage>
+  );
+};
+
+const subscribeToNothing = () => () => undefined;
+const noVersion = () => 0;
+
+/**
+ * Shows the conversation's replies again whenever the host says what one of them shows changed,
+ * such as a finished reply that turned out to show nothing of its own.
+ */
+export const useAIUIHostRefresh = (host: AIUIHost | undefined) =>
+  useSyncExternalStore(
+    host ? host.subscribe : subscribeToNothing,
+    host ? host.getVersion : noVersion,
+  );
+
+export type AIUIViewProps = {
+  document: AIUIDocument;
+  host: AIUIHost;
+  /** The reply the document belongs to. */
+  messageId: string;
+};
+
+/**
+ * One reply's generated UI, for showing replies outside `AIConversation`. It shows nothing when
+ * none of the host's renderers shows the document, and the reply's text should show instead:
+ * `host.content(document, messageId) !== null` tells which, and `host.subscribe` when that may
+ * have changed. Give the host the conversation's messages with `host.receive(messages)` as they
+ * change.
+ */
+export const AIUIView = ({ document, host, messageId }: AIUIViewProps) => {
+  useAIUIHostRefresh(host);
+  const content = host.content(document, messageId);
+  return content === null ? null : (
+    <AIUIGenerated
+      isStreaming={document.isStreaming}
+      messageId={messageId}
+      onError={(error) => host.fail(messageId, error)}
+    >
+      {content}
+    </AIUIGenerated>
+  );
+};
+
+const styles = StyleSheet.create({
+  generated: { alignSelf: 'stretch' },
+});

package/src/ai-components/generative-ui/index.ts

@@ -0,0 +1,48 @@
+// What apps and renderers of their own use. The helpers these are made with stay inside the
+// package, so changing them is no change to its API.
+export {
+  type AIUIAction,
+  type AIUIActionJSON,
+  AI_UI_ACTION_CUSTOM_DATA_KEY,
+  AI_UI_ACTION_MAX_BYTES,
+  aiUIActionJSON,
+  aiUIActionProblem,
+} from './AIUIAction';
+export { decodeAIUIArguments } from './AIUIArguments';
+export {
+  type AIUIComponent,
+  type AIUIComponentContext,
+  type AIUIComponentProps,
+  AIUIComponentView,
+  defineAIUIComponent,
+} from './AIUIComponent';
+export {
+  type AIUIComponentDefinition,
+  type AIUIComponentEvent,
+  type AIUIJSONSchema,
+  type AIUIParameter,
+  AIUIParameterType,
+  aiUIComponentDefinitionJSON,
+  optionalAIUIParameter,
+  parseAIUIComponentDefinition,
+  requiredAIUIParameter,
+} from './AIUIComponentDefinition';
+export {
+  type AIUIDocument,
+  type AIUIDocumentJSON,
+  type AIUIFormat,
+  AI_UI_CUSTOM_DATA_KEY,
+  AI_UI_SUPPORTED_VERSION,
+  aiUIDocumentText,
+  parseAIUIDocument,
+} from './AIUIDocument';
+export { AIUICascade, AIUIEntrance, AIUIEntranceStage, useAIUIEntrance } from './AIUIEntrance';
+export { AIUIHost, type AIUIHostOptions, AIUIToolError, isWebOrEmailLink } from './AIUIHost';
+export {
+  aiUIMediaSource,
+  type AIUIMediaURL,
+  aiUIPublicMediaURL,
+  isPublicWebURL,
+} from './AIUIMedia';
+export type { AIUIEngine, AIUIRenderer } from './AIUIRenderer';
+export { AIUIView, type AIUIViewProps, useAIUIHostRefresh } from './AIUIView';