ui-soxo-bootstrap-core 2.6.26 → 2.6.28

package/core/modules/steps/voice-navigation.js

@@ -0,0 +1,709 @@
+const GEMINI_LIVE_WS_ENDPOINT =
+  process.env.GEMINI_LIVE_WS_ENDPOINT ||
+  process.env.REACT_APP_GEMINI_LIVE_WS_ENDPOINT ||
+  'wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent';
+
+const GEMINI_LIVE_MODEL =
+  process.env.GEMINI_LIVE_MODEL || process.env.REACT_APP_GEMINI_LIVE_MODEL || 'gemini-live-2.5-flash-preview';
+
+const OPENAI_REALTIME_WS_ENDPOINT =
+  process.env.OPENAI_REALTIME_WS_ENDPOINT ||
+  process.env.REACT_APP_OPENAI_REALTIME_WS_ENDPOINT ||
+  process.env.OPENAI_REALTIME_ENDPOINT ||
+  process.env.REACT_APP_OPENAI_REALTIME_ENDPOINT ||
+  'wss://api.openai.com/v1/realtime';
+
+const OPENAI_REALTIME_MODEL =
+  process.env.OPENAI_REALTIME_MODEL || process.env.REACT_APP_OPENAI_REALTIME_MODEL || 'gpt-realtime';
+
+const OPENAI_REALTIME_INSTRUCTIONS =
+  process.env.OPENAI_REALTIME_INSTRUCTIONS ||
+  process.env.REACT_APP_OPENAI_REALTIME_INSTRUCTIONS ||
+  'You are a calm and clear healthcare onboarding assistant in a guided step-by-step workflow.';
+
+const ALLOWED_ACTIONS = ['NEXT', 'PREVIOUS', 'SKIP', 'FINISH', 'GOTO', 'REPEAT', 'NONE'];
+
+function normalizeModelName(modelName) {
+  if (!modelName) {
+    return 'models/gemini-live-2.5-flash-preview';
+  }
+
+  if (modelName.indexOf('models/') === 0) {
+    return modelName;
+  }
+
+  return `models/${modelName}`;
+}
+
+function getGeminiApiKey() {
+  if (process.env.GEMINI_API_KEY) {
+    return process.env.GEMINI_API_KEY;
+  }
+
+  if (process.env.REACT_APP_GEMINI_API_KEY) {
+    return process.env.REACT_APP_GEMINI_API_KEY;
+  }
+
+  if (typeof window !== 'undefined') {
+    try {
+      if (window.localStorage) {
+        return window.localStorage.getItem('gemini_api_key');
+      }
+    } catch (error) {
+      return null;
+    }
+  }
+
+  return null;
+}
+
+function getFromStorage(storageKey) {
+  if (typeof window === 'undefined' || !window.localStorage) {
+    return null;
+  }
+
+  try {
+    return window.localStorage.getItem(storageKey);
+  } catch (error) {
+    return null;
+  }
+}
+
+export function getOpenAIRealtimeApiKey() {
+  return (
+    process.env.OPEN_AI_KEY ||
+    process.env.OPENAI_API_KEY ||
+    process.env.REACT_APP_OPEN_AI_KEY ||
+    process.env.REACT_APP_OPENAI_API_KEY ||
+    getFromStorage('open_ai_key') ||
+    getFromStorage('openai_api_key') ||
+    getFromStorage('OPEN_AI_KEY') ||
+    getFromStorage('OPENAI_API_KEY') ||
+    getFromStorage('REACT_APP_OPEN_AI_KEY') ||
+    getFromStorage('REACT_APP_OPENAI_API_KEY')
+  );
+}
+
+function sanitizeJsonText(text) {
+  if (!text) {
+    return '';
+  }
+
+  return text.replace(/```json/gi, '').replace(/```/g, '').trim();
+}
+
+function normalizeCommandPayload(payload = {}) {
+  const rawAction = String(payload.action || '').toUpperCase();
+  const action = ALLOWED_ACTIONS.indexOf(rawAction) !== -1 ? rawAction : 'NONE';
+
+  const rawStepNumber = payload.stepNumber !== undefined ? payload.stepNumber : payload.step_number;
+  const parsedStepNumber = Number.parseInt(rawStepNumber, 10);
+  const stepNumber = Number.isNaN(parsedStepNumber) ? null : parsedStepNumber;
+
+  const message = typeof payload.message === 'string' ? payload.message.trim() : '';
+
+  return {
+    action,
+    stepNumber,
+    message,
+  };
+}
+
+function extractJsonObject(text) {
+  const start = text.indexOf('{');
+  const end = text.lastIndexOf('}');
+
+  if (start === -1 || end === -1 || end <= start) {
+    return null;
+  }
+
+  return text.slice(start, end + 1);
+}
+
+function parseCommandFromModelText(modelText) {
+  const cleaned = sanitizeJsonText(modelText);
+  const directJson = extractJsonObject(cleaned);
+
+  if (directJson) {
+    try {
+      return normalizeCommandPayload(JSON.parse(directJson));
+    } catch (error) {
+      return null;
+    }
+  }
+
+  return null;
+}
+
+export function fallbackVoiceCommand(transcript = '') {
+  const text = transcript.toLowerCase().trim();
+
+  const gotoMatch = text.match(/(?:go to|goto|open|jump to)\s+(?:step\s+)?(\d+)/);
+  if (gotoMatch && gotoMatch[1]) {
+    return {
+      action: 'GOTO',
+      stepNumber: Number.parseInt(gotoMatch[1], 10),
+      message: 'Navigating to requested step',
+    };
+  }
+
+  if (/(^|\s)(repeat|again|read step|read this step)(\s|$)/.test(text)) {
+    return { action: 'REPEAT', stepNumber: null, message: 'Repeating current step' };
+  }
+
+  if (/(^|\s)(previous|back|go back|last step)(\s|$)/.test(text)) {
+    return { action: 'PREVIOUS', stepNumber: null, message: 'Going to previous step' };
+  }
+
+  if (/(^|\s)(next|continue|forward|go next)(\s|$)/.test(text)) {
+    return { action: 'NEXT', stepNumber: null, message: 'Going to next step' };
+  }
+
+  if (/(^|\s)(skip|skip step)(\s|$)/.test(text)) {
+    return { action: 'SKIP', stepNumber: null, message: 'Skipping current step' };
+  }
+
+  if (/(^|\s)(finish|complete process|submit process|done)(\s|$)/.test(text)) {
+    return { action: 'FINISH', stepNumber: null, message: 'Finishing process' };
+  }
+
+  return { action: 'NONE', stepNumber: null, message: 'Command not recognized' };
+}
+
+function buildCommandPrompt({ transcript, context }) {
+  const {
+    activeStepNumber = 1,
+    totalSteps = 1,
+    isLastStep = false,
+    canSkip = false,
+    isStepCompleted = false,
+  } = context || {};
+
+  return [
+    'You are a strict command interpreter for a step wizard.',
+    'Return ONLY JSON with keys: action, stepNumber, message.',
+    'action must be one of: NEXT, PREVIOUS, SKIP, FINISH, GOTO, REPEAT, NONE.',
+    'stepNumber must be null unless action is GOTO.',
+    `Current step: ${activeStepNumber} of ${totalSteps}.`,
+    `Is last step: ${isLastStep ? 'yes' : 'no'}.`,
+    `Can skip current step: ${canSkip ? 'yes' : 'no'}.`,
+    `Current step completed: ${isStepCompleted ? 'yes' : 'no'}.`,
+    `User voice transcript: "${transcript}".`,
+  ].join(' ');
+}
+
+export function createGeminiLiveNavigator({ apiKey, model, endpoint } = {}) {
+  const resolvedApiKey = apiKey || getGeminiApiKey();
+  const resolvedModel = normalizeModelName(model || GEMINI_LIVE_MODEL);
+  const resolvedEndpoint = endpoint || GEMINI_LIVE_WS_ENDPOINT;
+
+  let socket = null;
+  let isSetupComplete = false;
+  let setupInFlight = null;
+  let pendingCommand = null;
+
+  function cleanupPending(errorMessage) {
+    if (!pendingCommand) {
+      return;
+    }
+
+    clearTimeout(pendingCommand.timeoutId);
+
+    const reject = pendingCommand.reject;
+    pendingCommand = null;
+
+    reject(new Error(errorMessage || 'Voice command interrupted'));
+  }
+
+  function close() {
+    cleanupPending('Gemini Live session closed');
+
+    if (socket && (socket.readyState === WebSocket.OPEN || socket.readyState === WebSocket.CONNECTING)) {
+      socket.close();
+    }
+
+    socket = null;
+    isSetupComplete = false;
+    setupInFlight = null;
+  }
+
+  async function connect() {
+    if (!resolvedApiKey) {
+      throw new Error('Gemini API key is missing');
+    }
+
+    if (socket && socket.readyState === WebSocket.OPEN && isSetupComplete) {
+      return;
+    }
+
+    if (setupInFlight) {
+      return setupInFlight;
+    }
+
+    setupInFlight = new Promise((resolve, reject) => {
+      const ws = new WebSocket(`${resolvedEndpoint}?key=${encodeURIComponent(resolvedApiKey)}`);
+      let settled = false;
+      let setupTimeoutId = null;
+
+      function safeResolve() {
+        if (settled) {
+          return;
+        }
+
+        settled = true;
+        clearTimeout(setupTimeoutId);
+        resolve();
+      }
+
+      function safeReject(error) {
+        if (settled) {
+          return;
+        }
+
+        settled = true;
+        clearTimeout(setupTimeoutId);
+        reject(error);
+      }
+
+      socket = ws;
+
+      ws.onopen = () => {
+        ws.send(
+          JSON.stringify({
+            setup: {
+              model: resolvedModel,
+              generationConfig: {
+                responseModalities: ['TEXT'],
+                temperature: 0.1,
+                maxOutputTokens: 256,
+              },
+              systemInstruction: {
+                parts: [
+                  {
+                    text:
+                      'You convert spoken step navigation requests into strict JSON command output only.',
+                  },
+                ],
+              },
+            },
+          })
+        );
+      };
+
+      ws.onmessage = (event) => {
+        let message;
+        try {
+          message = JSON.parse(event.data);
+        } catch (error) {
+          return;
+        }
+
+        if (message.setupComplete) {
+          isSetupComplete = true;
+          safeResolve();
+          return;
+        }
+
+        if (message.goAway) {
+          close();
+          safeReject(new Error('Gemini Live server closed the session'));
+          return;
+        }
+
+        if (pendingCommand && message.serverContent) {
+          const parts =
+            message.serverContent.modelTurn &&
+            message.serverContent.modelTurn.parts &&
+            Array.isArray(message.serverContent.modelTurn.parts)
+              ? message.serverContent.modelTurn.parts
+              : [];
+
+          parts.forEach((part) => {
+            if (part && typeof part.text === 'string') {
+              pendingCommand.buffer += part.text;
+            }
+          });
+
+          if (message.serverContent.turnComplete) {
+            const resolveCommand = pendingCommand.resolve;
+            const rejectCommand = pendingCommand.reject;
+            const buffer = pendingCommand.buffer;
+
+            clearTimeout(pendingCommand.timeoutId);
+            pendingCommand = null;
+
+            const parsed = parseCommandFromModelText(buffer);
+            if (parsed) {
+              resolveCommand(parsed);
+            } else {
+              rejectCommand(new Error('Unable to parse Gemini voice command response'));
+            }
+          }
+        }
+      };
+
+      ws.onerror = () => {
+        close();
+        safeReject(new Error('Gemini Live WebSocket error'));
+      };
+
+      ws.onclose = () => {
+        if (!settled) {
+          safeReject(new Error('Gemini Live WebSocket closed before setup'));
+        }
+
+        cleanupPending('Gemini Live WebSocket closed');
+
+        socket = null;
+        isSetupComplete = false;
+      };
+
+      setupTimeoutId = setTimeout(() => {
+        close();
+        safeReject(new Error('Gemini Live setup timed out'));
+      }, 10000);
+    });
+
+    try {
+      await setupInFlight;
+    } finally {
+      setupInFlight = null;
+    }
+  }
+
+  async function interpretCommand({ transcript, context }) {
+    await connect();
+
+    if (!socket || socket.readyState !== WebSocket.OPEN) {
+      throw new Error('Gemini Live socket is not connected');
+    }
+
+    if (pendingCommand) {
+      cleanupPending('A new command interrupted the previous command');
+    }
+
+    const prompt = buildCommandPrompt({ transcript, context });
+
+    const response = await new Promise((resolve, reject) => {
+      const timeoutId = setTimeout(() => {
+        pendingCommand = null;
+        reject(new Error('Gemini Live command timed out'));
+      }, 12000);
+
+      pendingCommand = {
+        buffer: '',
+        timeoutId,
+        resolve,
+        reject,
+      };
+
+      socket.send(
+        JSON.stringify({
+          clientContent: {
+            turns: [
+              {
+                role: 'user',
+                parts: [{ text: prompt }],
+              },
+            ],
+            turnComplete: true,
+          },
+        })
+      );
+    });
+
+    return normalizeCommandPayload(response);
+  }
+
+  return {
+    interpretCommand,
+    close,
+  };
+}
+
+function extractOpenAIRealtimeText(message = {}) {
+  if (typeof message.delta === 'string' && message.delta.trim()) {
+    return message.delta;
+  }
+
+  if (typeof message.text === 'string' && message.text.trim()) {
+    return message.text;
+  }
+
+  if (message.item && Array.isArray(message.item.content)) {
+    return message.item.content
+      .map((part) => {
+        if (typeof part?.text === 'string') {
+          return part.text;
+        }
+
+        if (typeof part?.transcript === 'string') {
+          return part.transcript;
+        }
+
+        return '';
+      })
+      .filter(Boolean)
+      .join(' ');
+  }
+
+  if (message.response && Array.isArray(message.response.output)) {
+    return message.response.output
+      .flatMap((outputItem) => (Array.isArray(outputItem?.content) ? outputItem.content : []))
+      .map((part) => {
+        if (typeof part?.text === 'string') {
+          return part.text;
+        }
+
+        if (typeof part?.transcript === 'string') {
+          return part.transcript;
+        }
+
+        return '';
+      })
+      .filter(Boolean)
+      .join(' ');
+  }
+
+  return '';
+}
+
+export function createOpenAIRealtimeConversation({ apiKey, model, endpoint, instructions } = {}) {
+  const resolvedApiKey = apiKey || getOpenAIRealtimeApiKey();
+  const resolvedModel = model || OPENAI_REALTIME_MODEL;
+  const resolvedEndpoint = endpoint || OPENAI_REALTIME_WS_ENDPOINT;
+  const resolvedInstructions = instructions || OPENAI_REALTIME_INSTRUCTIONS;
+
+  const openAIOrganizationId = process.env.OPENAI_ORG_ID || process.env.REACT_APP_OPENAI_ORG_ID || '';
+  const openAIProjectId = process.env.OPENAI_PROJECT_ID || process.env.REACT_APP_OPENAI_PROJECT_ID || '';
+
+  let socket = null;
+  let sessionReady = false;
+  let connectInFlight = null;
+  let pendingResponse = null;
+
+  function clearPending(errorMessage) {
+    if (!pendingResponse) {
+      return;
+    }
+
+    clearTimeout(pendingResponse.timeoutId);
+
+    const rejectTurn = pendingResponse.reject;
+    pendingResponse = null;
+
+    rejectTurn(new Error(errorMessage || 'OpenAI Realtime conversation interrupted'));
+  }
+
+  function close() {
+    clearPending('OpenAI Realtime session closed');
+
+    if (socket && (socket.readyState === WebSocket.OPEN || socket.readyState === WebSocket.CONNECTING)) {
+      socket.close();
+    }
+
+    socket = null;
+    sessionReady = false;
+    connectInFlight = null;
+  }
+
+  async function connect() {
+    if (!resolvedApiKey) {
+      throw new Error('OpenAI Realtime API key is missing');
+    }
+
+    if (socket && socket.readyState === WebSocket.OPEN && sessionReady) {
+      return;
+    }
+
+    if (connectInFlight) {
+      return connectInFlight;
+    }
+
+    connectInFlight = new Promise((resolve, reject) => {
+      const protocols = ['realtime', `openai-insecure-api-key.${resolvedApiKey}`];
+      if (openAIOrganizationId) {
+        protocols.push(`openai-organization.${openAIOrganizationId}`);
+      }
+
+      if (openAIProjectId) {
+        protocols.push(`openai-project.${openAIProjectId}`);
+      }
+
+      const ws = new WebSocket(`${resolvedEndpoint}?model=${encodeURIComponent(resolvedModel)}`, protocols);
+      let settled = false;
+      let setupTimeoutId = null;
+
+      function safeResolve() {
+        if (settled) {
+          return;
+        }
+
+        settled = true;
+        clearTimeout(setupTimeoutId);
+        resolve();
+      }
+
+      function safeReject(error) {
+        if (settled) {
+          return;
+        }
+
+        settled = true;
+        clearTimeout(setupTimeoutId);
+        reject(error);
+      }
+
+      socket = ws;
+
+      ws.onopen = () => {
+        ws.send(
+          JSON.stringify({
+            type: 'session.update',
+            session: {
+              instructions: resolvedInstructions,
+              modalities: ['text'],
+              temperature: 0.6,
+            },
+          })
+        );
+      };
+
+      ws.onmessage = (event) => {
+        let message;
+        try {
+          message = JSON.parse(event.data);
+        } catch (error) {
+          return;
+        }
+
+        if (message.type === 'session.updated') {
+          sessionReady = true;
+          safeResolve();
+          return;
+        }
+
+        if (message.type === 'error') {
+          const errorMessage = message.error?.message || 'OpenAI Realtime WebSocket error';
+          if (!sessionReady) {
+            safeReject(new Error(errorMessage));
+          } else {
+            clearPending(errorMessage);
+          }
+          return;
+        }
+
+        if (!pendingResponse) {
+          return;
+        }
+
+        const chunk = extractOpenAIRealtimeText(message);
+        if (chunk) {
+          pendingResponse.buffer += chunk;
+        }
+
+        if (message.type === 'response.done') {
+          const resolveTurn = pendingResponse.resolve;
+          const rejectTurn = pendingResponse.reject;
+          const buffer = pendingResponse.buffer.trim();
+
+          clearTimeout(pendingResponse.timeoutId);
+          pendingResponse = null;
+
+          if (buffer) {
+            resolveTurn(buffer);
+          } else {
+            rejectTurn(new Error('OpenAI Realtime response did not include text'));
+          }
+        }
+      };
+
+      ws.onerror = () => {
+        close();
+        safeReject(new Error('OpenAI Realtime WebSocket connection error'));
+      };
+
+      ws.onclose = () => {
+        if (!settled && !sessionReady) {
+          safeReject(new Error('OpenAI Realtime WebSocket closed before setup'));
+        }
+
+        clearPending('OpenAI Realtime WebSocket closed');
+        socket = null;
+        sessionReady = false;
+      };
+
+      setupTimeoutId = setTimeout(() => {
+        close();
+        safeReject(new Error('OpenAI Realtime setup timed out'));
+      }, 10000);
+    });
+
+    try {
+      await connectInFlight;
+    } finally {
+      connectInFlight = null;
+    }
+  }
+
+  async function sendUserText({ text, context = '', responseInstructions = '' } = {}) {
+    const normalizedText = typeof text === 'string' ? text.trim() : '';
+
+    if (!normalizedText) {
+      throw new Error('User text is required');
+    }
+
+    await connect();
+
+    if (!socket || socket.readyState !== WebSocket.OPEN) {
+      throw new Error('OpenAI Realtime socket is not connected');
+    }
+
+    if (pendingResponse) {
+      clearPending('A new turn interrupted the previous response');
+    }
+
+    const prompt = context ? `${context}\n\nUser: ${normalizedText}` : normalizedText;
+
+    return new Promise((resolve, reject) => {
+      const timeoutId = setTimeout(() => {
+        pendingResponse = null;
+        reject(new Error('OpenAI Realtime response timed out'));
+      }, 25000);
+
+      pendingResponse = {
+        buffer: '',
+        timeoutId,
+        resolve,
+        reject,
+      };
+
+      socket.send(
+        JSON.stringify({
+          type: 'conversation.item.create',
+          item: {
+            type: 'message',
+            role: 'user',
+            content: [{ type: 'input_text', text: prompt }],
+          },
+        })
+      );
+
+      socket.send(
+        JSON.stringify({
+          type: 'response.create',
+          response: {
+            modalities: ['text'],
+            ...(responseInstructions ? { instructions: responseInstructions } : {}),
+          },
+        })
+      );
+    });
+  }
+
+  return {
+    connect,
+    sendUserText,
+    close,
+  };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ui-soxo-bootstrap-core",
-  "version": "2.6.26",
+  "version": "2.6.28",
   "description": "All the Core Components for you to start",
   "keywords": [
     "all in one"