stream-chat-react-native-core 9.12.1 → 9.13.0

package/src/a2ui/A2UISurface.tsx

@@ -0,0 +1,241 @@
+import React, {
+  createContext,
+  memo,
+  type ReactNode,
+  useCallback,
+  useContext,
+  useLayoutEffect,
+  useMemo,
+  useSyncExternalStore,
+} from 'react';
+import { type AccessibilityProps, View as RNView, type ViewProps } from 'react-native';
+import Animated from 'react-native-reanimated';
+
+import {
+  type ComponentApi,
+  type ComponentNode,
+  effect,
+  getValue,
+  isComponentNode,
+  isWritable,
+  NodeResolver,
+  peekValue,
+  ResolvedBinding,
+  type Signal,
+  type SurfaceModel,
+} from '@a2ui/web_core';
+
+import { A2UIEngineContext, A2UIReplyContext } from './A2UIContexts';
+import { keepSecretPath } from './secrets';
+
+import { AIUIComponentView } from '../ai-components/generative-ui/AIUIComponent';
+import { useAIUIComesIn, useAIUIEntrance } from '../ai-components/generative-ui/AIUIEntrance';
+import { isRecord } from '../ai-components/generative-ui/AIUIHelpers';
+
+/** The value of an A2UI signal, following it. */
+export const useSignalValue = <T,>(signal: Signal<T>) => {
+  const subscribe = useCallback(
+    (onChange: () => void) =>
+      effect(() => {
+        getValue(signal);
+        onChange();
+      }),
+    [signal],
+  );
+  const getSnapshot = useCallback(() => peekValue(signal), [signal]);
+  return useSyncExternalStore(subscribe, getSnapshot);
+};
+
+const isPlainObject = (value: unknown): value is Record<string, unknown> => {
+  if (!isRecord(value)) return false;
+  const prototype = Object.getPrototypeOf(value);
+  return prototype === Object.prototype || prototype === null;
+};
+
+const viewValueOf = (value: unknown): unknown => {
+  if (isComponentNode(value)) return value;
+  if (value instanceof ResolvedBinding) return viewValueOf(value.value);
+  if (Array.isArray(value)) return value.map(viewValueOf);
+  if (isPlainObject(value)) return viewPropsOf(value);
+  return value;
+};
+
+const noop = () => undefined;
+
+// What a component's view reads: bindings as their values, with a setter for each one the person
+// can change, named after it, such as `setValue` for `value`. Children stay nodes.
+const viewPropsOf = (props: Record<string, unknown>) => {
+  const view: Record<string, unknown> = {};
+  for (const [key, value] of Object.entries(props)) {
+    if (value instanceof ResolvedBinding) {
+      view[key] = viewValueOf(value.value);
+      view[`set${key.charAt(0).toUpperCase()}${key.slice(1)}`] = isWritable(value)
+        ? value.set
+        : noop;
+    } else view[key] = viewValueOf(value);
+  }
+  return view;
+};
+
+/** The axis the parent of a part lays its children out on, which a part's weight grows along. */
+export const A2UIAxisContext = createContext<'column' | 'row'>('column');
+
+const said = (value: unknown) =>
+  value === undefined || value === null || value === '' ? undefined : String(value);
+
+// What the agent tells assistive technology about a part, with its `accessibility`: what to call
+// it, what more to say about it, whether to announce its changes, and whether to leave it out.
+const accessibilityOf = (accessibility: unknown): AccessibilityProps => {
+  if (!isRecord(accessibility)) return {};
+  const { description, hidden, label, live } = accessibility;
+  const named = said(label);
+  const described = said(description);
+  return {
+    ...(named ? { accessibilityLabel: named } : {}),
+    ...(described ? { accessibilityHint: described } : {}),
+    ...(live === 'polite' || live === 'assertive' ? { accessibilityLiveRegion: live } : {}),
+    ...(hidden === true
+      ? { accessibilityElementsHidden: true, importantForAccessibility: 'no-hide-descendants' }
+      : {}),
+  };
+};
+
+// One of the app's components where the agent placed it, with the arguments it declares.
+const PlacedComponent = ({
+  node,
+  props,
+}: {
+  node: ComponentNode;
+  props: Record<string, unknown>;
+}) => {
+  const engine = useContext(A2UIEngineContext);
+  const reply = useContext(A2UIReplyContext);
+  const component = engine?.appComponent(node.type);
+  const names = component?.definition.parameters.map(({ name }) => name);
+  const args = useMemo(
+    () =>
+      Object.fromEntries(
+        (names ?? []).filter((name) => name in props).map((name) => [name, props[name]]),
+      ),
+    // eslint-disable-next-line react-hooks/exhaustive-deps -- the names change with the component
+    [component, props],
+  );
+  if (!engine || !reply || !component) return null;
+  const { messageId } = reply;
+  const { name } = component.definition;
+  return (
+    <AIUIComponentView
+      args={args}
+      component={component}
+      context={{
+        emit: (event, label, payload = {}) =>
+          void engine.host.send({
+            format: 'a2ui',
+            label,
+            messageId,
+            name: event,
+            payload: { ...payload, component: name },
+          }),
+        host: engine.host,
+        isStreaming: reply.isStreaming,
+        messageId,
+      }}
+    />
+  );
+};
+
+/**
+ * One part of a surface: its view from the basic catalog, or the app's component of its type,
+ * coming in when it shows, with its weight and what the agent tells assistive technology. A part
+ * whose type no catalog has, or that is still on its way, shows nothing.
+ */
+// The path of the data model a TextField writes what the person types to, when it is obscured,
+// such as a password, which must never reach the agent.
+const secretPathOf = (node: ComponentNode, resolved: Record<string, unknown>) => {
+  if (node.type !== 'TextField') return undefined;
+  const variant =
+    resolved.variant instanceof ResolvedBinding ? resolved.variant.value : resolved.variant;
+  const { value } = resolved;
+  return variant === 'obscured' && value instanceof ResolvedBinding && isWritable(value)
+    ? (node.context?.dataContext.resolvePath(value.path) ?? value.path)
+    : undefined;
+};
+
+const A2UINode = memo(({ node }: { node: ComponentNode }) => {
+  const engine = useContext(A2UIEngineContext);
+  const axis = useContext(A2UIAxisContext);
+  const resolved = useSignalValue(node.props) as Record<string, unknown>;
+  const props = useMemo(() => viewPropsOf(resolved), [resolved]);
+  // Before the person can type: whichever view draws the field, what they type stays theirs.
+  const secretPath = secretPathOf(node, resolved);
+  const surface = node.context?.dataContext.surface;
+  useLayoutEffect(() => {
+    if (secretPath && surface) keepSecretPath(surface, secretPath);
+  }, [secretPath, surface]);
+  const comesIn = useAIUIComesIn();
+  if (!engine || node.isPlaceholder || node.state !== 'resolved') return null;
+  const View = engine.components[node.type];
+  const isApp = !View && !!engine.appComponent(node.type);
+  if (!View && !isApp) return null;
+  // A weighted part shares the room its row leaves, as flex-grow from nothing does.
+  const weight = typeof props.weight === 'number' && axis === 'row' ? props.weight : undefined;
+  const Part = comesIn ? EnteringPart : ShownPart;
+  return (
+    <Part
+      {...accessibilityOf(props.accessibility)}
+      style={weight !== undefined ? { flexBasis: 0, flexGrow: weight } : undefined}
+      testID={`a2ui-${node.componentId}`}
+    >
+      {View ? <View node={node} props={props} /> : <PlacedComponent node={node} props={props} />}
+    </Part>
+  );
+});
+A2UINode.displayName = 'A2UINode';
+
+type PartProps = ViewProps & { children: ReactNode };
+
+// A part that comes in as it shows, such as while the agent writes it.
+const EnteringPart = ({ children, style, ...props }: PartProps) => {
+  const { scope, style: entrance } = useAIUIEntrance();
+  return (
+    <Animated.View {...props} style={[entrance, style]}>
+      {scope(children)}
+    </Animated.View>
+  );
+};
+
+// A part that shows at once, such as one of a reply from history: a plain view, since nothing of
+// it moves and a list that scrolls fast mounts it all the time.
+const ShownPart = ({ children, ...props }: PartProps) => <RNView {...props}>{children}</RNView>;
+
+/** A child of a part, as its view gets it: a node, or nothing while it is on its way. */
+export const A2UIChild = ({ node }: { node: unknown }) =>
+  isComponentNode(node) ? <A2UINode node={node} /> : null;
+
+/** The children of a part, as its view gets them. */
+export const A2UIChildren = ({ nodes }: { nodes: unknown }) => (
+  <>
+    {Array.isArray(nodes)
+      ? nodes.map((node) =>
+          isComponentNode(node) ? <A2UINode key={node.instanceId} node={node} /> : null,
+        )
+      : null}
+  </>
+);
+
+const resolvers = new WeakMap<SurfaceModel<ComponentApi>, NodeResolver<ComponentApi>>();
+
+const resolverOf = (surface: SurfaceModel<ComponentApi>) => {
+  let resolver = resolvers.get(surface);
+  if (!resolver) {
+    resolver = new NodeResolver<ComponentApi>(surface, surface.defaultCatalog);
+    resolvers.set(surface, resolver);
+  }
+  return resolver;
+};
+
+/** A surface of a reply: its tree of parts, from its root, as the agent writes them. */
+export const A2UISurfaceView = ({ surface }: { surface: SurfaceModel<ComponentApi> }) => {
+  const root = useSignalValue(resolverOf(surface).rootNode);
+  return root ? <A2UINode node={root} /> : null;
+};

package/src/a2ui/__tests__/A2UIInstructions.test.ts

@@ -0,0 +1,233 @@
+// What an agent is told, written in a script or a build too.
+import { parseAIUIComponentDefinition } from '../../ai-components';
+import { ohridLines, uiComponents } from '../../mock-builders/a2ui/ohrid';
+import {
+  A2UI_DEFAULT_CATALOG_ID,
+  a2uiInstructions,
+  A2UIInstructionsError,
+  a2uiProblems,
+} from '../index';
+
+const components = uiComponents.flatMap((json) => {
+  const definition = parseAIUIComponentDefinition(json);
+  return definition ? [definition] : [];
+});
+
+// The basic catalog's part the Swift assistant's `a2ui/prompt.json` keeps.
+const keep = [
+  'Text',
+  'Icon',
+  'Row',
+  'Column',
+  'List',
+  'Card',
+  'Tabs',
+  'Divider',
+  'Button',
+  'TextField',
+  'CheckBox',
+  'ChoicePicker',
+  'Slider',
+  'DateTimeInput',
+  'formatString',
+  'formatDate',
+  'formatNumber',
+  'openUrl',
+  'required',
+];
+
+const example = (lines: string[]) =>
+  [
+    'Example — a weekend trip:',
+    'A weekend in Ohrid by the lake: a map, your calendar and a plan for each day.',
+    `\`\`\`ui:a2ui@1.0\n${lines.join('\n')}\n\`\`\``,
+  ].join('\n\n');
+
+const line = (message: Record<string, unknown>) => JSON.stringify({ version: '1.0', ...message });
+
+const surface = (components: Record<string, unknown>[]) => [
+  line({ createSurface: { catalogId: A2UI_DEFAULT_CATALOG_ID, surfaceId: 's' } }),
+  line({ updateComponents: { components, surfaceId: 's' } }),
+];
+
+describe('a2uiInstructions', () => {
+  it('writes the instructions from prompt.json, in its order', async () => {
+    const text = await a2uiInstructions({
+      components,
+      examples: [example(ohridLines)],
+      keep,
+      notes: ['- PlaceCard, TripMap and CalendarCheck are the phone’s own views.'],
+      preamble: 'Some answers work better as interactive UI than as text.',
+      rules: ['Never add images for places.', 'Write the whole UI again.'],
+    });
+
+    const headings = text.split('\n').filter((row) => /^#{1,2} /.test(row));
+    expect(headings).toEqual([
+      '## Writing UI in A2UI',
+      '# Generating user interfaces with A2UI v1.0',
+      '## Catalog Guidelines',
+      "## The app's components",
+      '## The catalog `getstream.io:app-catalog`',
+      '## Rules',
+      '## Examples',
+    ]);
+    expect(text.startsWith('Some answers work better')).toBe(true);
+    expect(text).toContain('whose info string is `ui:a2ui@1.0`');
+    expect(text).toContain('the `catalogId` `getstream.io:app-catalog`');
+    expect(text).toContain('1. Never add images for places.\n2. Write the whole UI again.');
+
+    const catalog = JSON.parse((text.match(/```json\n(\{.*\})\n```/) ?? [])[1] ?? '{}') as {
+      catalogId: string;
+      components: Record<string, unknown>;
+    };
+    expect(catalog.catalogId).toBe(A2UI_DEFAULT_CATALOG_ID);
+    expect(Object.keys(catalog.components)).toEqual(
+      expect.arrayContaining(['PlaceCard', 'TripMap', 'CalendarCheck', 'Text', 'Tabs']),
+    );
+    expect(catalog.components).not.toHaveProperty('Video');
+    expect(catalog.components).not.toHaveProperty('Modal');
+    expect(catalog.components.CalendarCheck).toEqual({
+      description:
+        "The person's own calendar for the days of a plan, read on the phone. You never see it: use it whenever a plan has dates, so they can see what the trip would clash with.",
+      properties: {
+        component: { const: 'CalendarCheck' },
+        end: {
+          description: 'The last day, as YYYY-MM-DD',
+          format: 'date',
+          type: 'string',
+        },
+        start: {
+          description: 'The first day, as YYYY-MM-DD',
+          format: 'date',
+          type: 'string',
+        },
+      },
+      required: ['component', 'end', 'start'],
+      type: 'object',
+    });
+    expect(JSON.stringify(catalog.components.PlaceCard)).toContain(
+      'Tapping the card asks you about the place.',
+    );
+  });
+
+  it('writes nothing when an example would not render, and says why', async () => {
+    const broken = [
+      ...ohridLines.slice(0, 2),
+      line({
+        updateComponents: {
+          components: [{ component: 'Globe', id: 'map' }],
+          surfaceId: 'ohrid-weekend',
+        },
+      }),
+    ];
+    const asked = a2uiInstructions({ components, examples: [example(broken)], keep });
+    await expect(asked).rejects.toBeInstanceOf(A2UIInstructionsError);
+    await expect(asked).rejects.toMatchObject({
+      problems: expect.arrayContaining([expect.stringMatching(/^example: map: .*Globe/)]),
+    });
+  });
+});
+
+describe('a2uiProblems', () => {
+  it('finds nothing in the Ohrid plan', async () => {
+    expect(await a2uiProblems(ohridLines.join('\n'), { components })).toEqual([]);
+  });
+
+  it('says what doesn’t fit the catalog: components, properties and their values', async () => {
+    const problems = await a2uiProblems(
+      surface([
+        { children: ['title', 'map', 'go'], component: 'Column', id: 'root' },
+        { component: 'Text', id: 'title', size: 3, text: 'A weekend' },
+        { component: 'Globe', id: 'map' },
+        { child: 'title', component: 'Button', id: 'go' },
+      ]).join('\n'),
+      { components },
+    );
+    expect(problems).toEqual([
+      expect.stringMatching(/^go: .*action/),
+      expect.stringMatching(/^map: .*Globe/),
+      expect.stringMatching(/^title: .*size/),
+    ]);
+  });
+
+  it('says what the components make together: references, root, orphans, cycles and ids', async () => {
+    const dangling = await a2uiProblems(
+      surface([
+        { children: ['a', 'gone'], component: 'Column', id: 'root' },
+        { component: 'Text', id: 'a', text: 'Here' },
+        { component: 'Text', id: 'lost', text: 'Nobody holds this' },
+      ]).join('\n'),
+    );
+    expect(dangling).toEqual([
+      'root holds gone, which is not defined',
+      'lost is defined but nothing holds it',
+    ]);
+
+    expect(
+      await a2uiProblems(surface([{ component: 'Text', id: 'top', text: 'Hi' }]).join('\n')),
+    ).toEqual(['surface s has no component with the id root']);
+
+    const cycle = await a2uiProblems(
+      surface([
+        { children: ['loop'], component: 'Column', id: 'root' },
+        { children: ['root'], component: 'Column', id: 'loop' },
+      ]).join('\n'),
+    );
+    expect(cycle).toEqual([expect.stringMatching(/holds itself through its children/)]);
+
+    const named = await a2uiProblems(
+      surface([
+        { children: ['saturday-plan'], component: 'Column', id: 'root' },
+        { component: 'Text', id: 'saturday-plan', text: 'Morning' },
+      ]).join('\n'),
+    );
+    expect(named).toEqual([expect.stringMatching(/^saturday-plan: .*identifier/)]);
+  });
+
+  it('says what is wrong with the messages themselves', async () => {
+    const problems = await a2uiProblems(
+      [
+        '{not json}',
+        line({ updateComponents: { components: [], surfaceId: 'later' } }),
+        line({ callRendererFunction: { name: 'openUrl' } }),
+        line({ createSurface: { catalogId: 'someone-else', surfaceId: 's' } }),
+        line({
+          updateComponents: {
+            components: [
+              { component: 'Text', id: 'root', text: 'One' },
+              { component: 'Text', id: 'root', text: 'Two' },
+            ],
+            surfaceId: 's',
+          },
+        }),
+      ].join('\n'),
+    );
+    expect(problems).toEqual([
+      "this isn't JSON: {not json}",
+      'surface later gets components before it is created',
+      "a reply's UI calls no functions",
+      'surface s names the catalog someone-else, not getstream.io:app-catalog',
+      'surface s defines root more than once in one message',
+    ]);
+    expect(
+      await a2uiProblems(
+        [
+          ...surface([{ component: 'Text', id: 'root', text: 'One' }]),
+          line({ createSurface: { catalogId: A2UI_DEFAULT_CATALOG_ID, surfaceId: 's' } }),
+        ].join('\n'),
+      ),
+    ).toEqual([expect.stringMatching(/^surface s is created again/)]);
+    expect(
+      await a2uiProblems(
+        [
+          ...surface([{ component: 'Text', id: 'root', text: 'One' }]),
+          line({ deleteSurface: { surfaceId: 's' } }),
+          ...surface([{ component: 'Text', id: 'root', text: 'Two' }]),
+        ].join('\n'),
+      ),
+    ).toEqual([]);
+    expect(await a2uiProblems('Just words.')).toEqual([
+      'nothing renders: there is no createSurface',
+    ]);
+  });
+});