reneco-advanced-input-module 0.0.1

package/src/index.ts

@@ -0,0 +1,12 @@
+/**
+ * @fileoverview entry point for your component library
+ *
+ * This is the entry point for your component library. Use this file to export utilities,
+ * constants or data structure that accompany your components.
+ *
+ * DO NOT use this file to export your components. Instead, use the recommended approaches
+ * to consume components of this package as outlined in the `README.md`.
+ */
+
+// export { format } from './utils/utils';
+// export type * from './components.d.ts';

package/src/services/audio-recorder.service.ts

@@ -0,0 +1,74 @@
+export class AudioRecorderService {
+  private mediaRecorder: MediaRecorder | null = null;
+  private audioChunks: Blob[] = [];
+  private stream: MediaStream | null = null;
+
+  async startRecording(): Promise<void> {
+    try {
+      // Check if the API exists before calling
+      if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
+        console.error(
+          'Failed to start recording:', 'Microphone access is not supported in this browser or the page is not served over HTTPS/localhost.'
+        );
+        return; // Exit gracefully instead of throwing
+      }
+      
+      this.stream = await navigator.mediaDevices.getUserMedia({
+        audio: {
+          echoCancellation: true,
+          noiseSuppression: true,
+          autoGainControl: true
+        }
+      });
+
+      this.audioChunks = [];
+      this.mediaRecorder = new MediaRecorder(this.stream, {
+        mimeType: 'audio/webm;codecs=opus'
+      });
+
+      this.mediaRecorder.ondataavailable = (event) => {
+        if (event.data && event.data.size > 0) {
+          this.audioChunks.push(event.data);
+        }
+      };
+
+      this.mediaRecorder.start(100); // Collect data every 100ms
+    } catch (error: any) {
+      console.error('Failed to start recording:', error);
+    }
+  }
+
+  async stopRecording(): Promise<Blob> {
+    return new Promise((resolve, reject) => {
+      if (!this.mediaRecorder) {
+        reject(new Error('No active recording found'));
+        return;
+      }
+
+      this.mediaRecorder.onstop = () => {
+        const audioBlob = new Blob(this.audioChunks, { type: 'audio/webm' });
+        this.cleanup();
+        resolve(audioBlob);
+      };
+
+      this.mediaRecorder.onerror = (event) => {
+        reject(new Error(`Recording error: ${event}`));
+      };
+
+      this.mediaRecorder.stop();
+    });
+  }
+
+  isRecording(): boolean {
+    return this.mediaRecorder?.state === 'recording';
+  }
+
+  private cleanup(): void {
+    if (this.stream) {
+      this.stream.getTracks().forEach(track => track.stop());
+      this.stream = null;
+    }
+    this.mediaRecorder = null;
+    this.audioChunks = [];
+  }
+}

package/src/services/llm.service.ts

@@ -0,0 +1,221 @@
+import { LLMProvider, ServiceProviderConfig } from '../types/service-providers.types';
+import { FormSchema } from '../types/form-schema.types';
+
+export class OpenAILLMService implements LLMProvider {
+  private apiKey: string;
+  private model: string;
+  private baseUrl: string;
+
+  constructor(config: ServiceProviderConfig['llm']) {
+    // Get API key from config or environment
+    this.apiKey = config?.apiKey || this.getEnvironmentVariable('OPENAI_API_KEY') || '';
+    // the newest OpenAI model is "gpt-4.1-mini". do not change this unless explicitly requested by the user
+    // this.model = config?.model || 'gpt-4.1-mini';
+    this.model = config?.model || 'gpt-4.1';
+    this.baseUrl = config?.baseUrl || 'https://api.openai.com/v1';
+    
+    if (!this.apiKey) {
+      throw new Error('OpenAI API key is required for LLM service');
+    }
+  }
+
+  private getEnvironmentVariable(name: string): string | undefined {
+    // In browser environment, we might get env vars through other means
+    if (typeof process !== 'undefined' && process.env) {
+      return process.env[name];
+    }
+    // Check if it's available as a global variable or through other means
+    return (window as any)[name] || undefined;
+  }
+
+  private getOptimizeFieldsDescription(schema: any){
+    return Object.values(schema).map((field: any) => 
+      `- ${field.name ?? field.title} ` +
+      `(${field.type}` +
+      `${field.required ? ', required' : ''}` +
+      `${field.readonly ? ', readonly' : ''}` +
+      `${field.min && field.min !== "" ? ', min='+field.min : ''}` +
+      `${field.max && field.max !== "" ? ', max='+field.max : ''}` +
+      `)` +
+      `${field.options ? ` - options: ${field.options.join(', ')}` : ''}`
+    ).join('\n')
+  }
+
+  async fillFormFromTranscription(transcription: string, schema: any): Promise<any> {
+    return this.fillForm(transcription, schema, true);
+  }
+
+  async fillFormFromJson(json: string, schema: any): Promise<any> {
+    return this.fillForm(json, schema, false);
+  }
+
+  private async fillForm(data: string, schema: any, dataIsTranscription: boolean = true): Promise<any> {
+    try {
+      // Handle complex schema format with fields array
+      if (schema?.fields || schema?.schema) {
+        const finalSchema = schema?.fields || schema?.schema;
+        const systemPrompt = `You are an expert form-filling assistant. You will receive a voice transcription and form field definitions. Your task is to extract values from the spoken content.
+          Rules:
+          1. Only extract values that can be confidently determined from the transcription
+          2. Respect field types (string, number, datetime, boolean, select)
+          3. For datetime fields, use ISO format (YYYY-MM-DDTHH:MM)
+          4. For date fields, use the following format: DD/MM/YYY
+          5. For boolean fields, interpret yes/no, true/false, positive/negative responses
+          6. For select fields, use exact option values from the provided choices
+          7. Only include fields where relevant information is found
+          8. The current GMT datetime is ${(new Date() as any).toGMTString()}
+          9. Respect the constraints written between parenthesis for readonly status (readonly fields must not be filled with values), min and max values, whatever the transcription says
+
+          Respond with JSON in this exact format: {"fields": [{"name": "field_name", "value": "extracted_value"}]}`;
+
+        let userPrompt = `
+          Voice transcription: "${data}"
+
+          Form fields:
+          ${this.getOptimizeFieldsDescription(finalSchema)}
+
+          Please extract values from the Transcription for these fields.
+        `;
+
+        //TODO
+
+        // const userPrompt = (
+        //   dataIsTranscription 
+        //   ?
+        //   `Voice transcription: "${data}"`
+        //   : 
+        //   `Json datas: "${JSON.stringify(data)}"`
+        //   )+`
+
+        //   Form fields:
+        //   ${this.getOptimizeFieldsDescription(finalSchema)}
+
+        //   Please extract values from the ` + 
+        //   (
+        //   dataIsTranscription 
+        //   ?
+        //   `Transcription`
+        //   : 
+        //   `Json generated file`
+        //   )+` for these fields.`;
+
+        const response = await fetch(`${this.baseUrl}/chat/completions`, {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+            'Authorization': `Bearer ${this.apiKey}`,
+          },
+          body: JSON.stringify({
+            model: this.model,
+            messages: [
+              { role: 'system', content: systemPrompt },
+              { role: 'user', content: userPrompt }
+            ],
+            response_format: { type: 'json_object' },
+            temperature: 0.1,
+          }),
+        });
+
+        if (!response.ok) {
+          const errorData = await response.json().catch(() => ({ error: 'Unknown error' }));
+          throw new Error(`LLM API failed: ${errorData.error?.message || response.statusText}`);
+        }
+
+        const result = await response.json();
+        return JSON.parse(result.choices[0].message.content);
+      }
+
+      // Handle simple schema format (backward compatibility)
+      const systemPrompt = `You are an expert form-filling assistant. You will receive a voice transcription and a JSON form schema. Your task is to intelligently fill the form fields based on the spoken content.
+        Rules:
+        1. Only fill fields that can be confidently determined from the transcription
+        2. Respect field types (string, number, date, boolean, select)
+        3. For datetime fields, use ISO format (YYYY-MM-DDTHH:MM)
+        4. For date fields, use the following format: DD/MM/YYY
+        5. For boolean fields, interpret yes/no, true/false, positive/negative responses
+        6. For select fields, match the closest option from the provided choices
+        7. Leave fields empty if no relevant information is found
+        8. Return the same schema structure with 'default' values filled
+        9. The current GMT datetime is ${(new Date() as any).toGMTString()}
+        10. Respect the constraints written between parenthesis for readonly status (readonly fields must not be filled with values), min and max values, whatever the transcription says
+
+        Respond with JSON in this exact format: {"schema": {...}}`;
+
+        let userPrompt = `
+          Voice transcription: "${data}"
+
+          Form schema to fill:
+          ${JSON.stringify(schema, null, 2)}
+
+          Please fill the form fields based on the Transcription and return the schema with default values populated.
+        `;
+
+        // const userPrompt = (
+        //   dataIsTranscription 
+        //   ?
+        //   `Voice transcription: "${data}"`
+        //   : 
+        //   `Json datas: "${JSON.stringify(data)}"`
+        //   )+`
+
+        //   Form schema to fill:
+        //   ${JSON.stringify(schema, null, 2)}
+
+        //   Please fill the form fields based on the ` + 
+        //   (
+        //   dataIsTranscription 
+        //   ?
+        //   `Transcription`
+        //   : 
+        //   `Json generated file`
+        //   )+` nd return the schema with default values populated.`;
+
+      const response = await fetch(`${this.baseUrl}/chat/completions`, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          'Authorization': `Bearer ${this.apiKey}`,
+        },
+        body: JSON.stringify({
+          model: this.model,
+          messages: [
+            { role: 'system', content: systemPrompt },
+            { role: 'user', content: userPrompt }
+          ],
+          response_format: { type: 'json_object' },
+          temperature: 0.1, // Low temperature for consistency
+        }),
+      });
+
+      if (!response.ok) {
+        const errorData = await response.json().catch(() => ({ error: 'Unknown error' }));
+        throw new Error(`LLM API failed: ${errorData.error?.message || response.statusText}`);
+      }
+
+      const result = await response.json();
+      const filledSchema = JSON.parse(result.choices[0].message.content);
+      
+      // Validate that the response has the correct structure
+      if (!filledSchema.schema) {
+        throw new Error('Invalid response format from LLM service');
+      }
+
+      return filledSchema;
+    } catch (error) {
+      throw new Error(`Form filling failed: ${error.message}`);
+    }
+  }
+}
+
+export class LLMServiceFactory {
+  static create(config: ServiceProviderConfig): LLMProvider {
+    const provider = config.llm?.provider || 'openai';
+    
+    switch (provider) {
+      case 'openai':
+        return new OpenAILLMService(config.llm);
+      default:
+        throw new Error(`Unsupported LLM provider: ${provider}`);
+    }
+  }
+}

package/src/services/speech-to-text.service.ts

@@ -0,0 +1,72 @@
+import { SpeechToTextProvider, ServiceProviderConfig } from '../types/service-providers.types';
+
+export class WhisperSpeechToTextService implements SpeechToTextProvider {
+  private apiKey: string;
+  private baseUrl: string;
+
+  constructor(config: ServiceProviderConfig['speechToText']) {
+    // Get API key from config or environment
+    this.apiKey = config?.apiKey || this.getEnvironmentVariable('OPENAI_API_KEY') || '';
+    this.baseUrl = config?.baseUrl || 'https://api.openai.com/v1';
+    
+    if (!this.apiKey) {
+      throw new Error('OpenAI API key is required for Whisper service');
+    }
+  }
+
+  private getEnvironmentVariable(name: string): string | undefined {
+    // In browser environment, we might get env vars through other means
+    if (typeof process !== 'undefined' && process.env) {
+      return process.env[name];
+    }
+    // Check if it's available as a global variable or through other means
+    return (window as any)[name] || undefined;
+  }
+
+  async transcribe(audioBlob: Blob, lang: string = 'en'): Promise<string> {
+    try {
+      const formData = new FormData();
+      
+      // Convert webm to a format Whisper can handle
+      const audioFile = new File([audioBlob], 'audio.webm', { type: 'audio/webm' });
+      formData.append('file', audioFile);
+      formData.append('model', 'gpt-4o-transcribe');// >>> tronque le texte ?
+      // formData.append('model', 'gpt-4o-mini-transcribe');// >>> tronque le texte ?
+      // formData.append('model', 'whisper-1');
+      formData.append('language', lang);
+      formData.append('response_format', 'json');
+      formData.append('max_output_tokens', '2000');
+
+      const response = await fetch(`${this.baseUrl}/audio/transcriptions`, {
+        method: 'POST',
+        headers: {
+          'Authorization': `Bearer ${this.apiKey}`,
+        },
+        body: formData,
+      });
+
+      if (!response.ok) {
+        const errorData = await response.json().catch(() => ({ error: 'Unknown error' }));
+        throw new Error(`Transcription failed: ${errorData.error?.message || response.statusText}`);
+      }
+
+      const result = await response.json();
+      return result.text || '';
+    } catch (error) {
+      throw new Error(`Speech-to-text transcription failed: ${error.message}`);
+    }
+  }
+}
+
+export class SpeechToTextServiceFactory {
+  static create(config: ServiceProviderConfig): SpeechToTextProvider {
+    const provider = config.speechToText?.provider || 'whisper';
+    
+    switch (provider) {
+      case 'whisper':
+        return new WhisperSpeechToTextService(config.speechToText);
+      default:
+        throw new Error(`Unsupported speech-to-text provider: ${provider}`);
+    }
+  }
+}

package/src/types/form-schema.types.ts

@@ -0,0 +1,78 @@
+export interface FormField {
+  type: 'string' | 'number' | 'date' | 'boolean' | 'select' | 'header' | 'dbpicker';
+  title: string;
+  description?: string;
+  required?: boolean;
+  default?: any;
+  options?: string[]; // For select fields
+  pattern?: string; // For validation
+  readonly?: boolean;
+  min?: number;
+  max?: number;
+}
+export interface FormFieldExtended extends FormField {
+  realType?: string;
+  pickerOptions?: any[];
+
+  ID?: string;
+  TFie_PK_ID?: string;
+  TVal_PK_ID?: string;
+  TFIn_PK_ID?: string;
+  Enabled?: boolean;
+  isForm?: string;
+  LabelText?: string;
+  SystemName?: string;
+  HelpText?: string;
+  CssStyle?: string;
+  ControlType?: string;
+  ValidationRequired?: string;
+  ValidationMin?: string;
+  ValidationMax?: string;
+  Mask?: string;
+  TypeId?: string;
+  Unit?: string;
+  TFie_Fullpath?: string;
+  TVal_FK_Parent_ID?: string;
+  value?: any;
+}
+
+export interface FormSchema {
+  schema: {
+    [fieldName: string]: FormField;
+  };
+  title?: string;
+  description?: string;
+}
+
+export interface FormSchemaFieldsOnly {
+  fields: {
+    [fieldName: string]: FormField;
+  };
+}
+
+export interface FormSchemaExtended {
+  schema: {
+    [fieldName: string]: FormFieldExtended;
+  };
+  title?: string;
+  description?: string;
+}
+
+
+export interface FormSchemaFieldsOnlyExtended {
+  fields: {
+    [fieldName: string]: FormFieldExtended;
+  };
+}
+
+export interface FilledFormData {
+  [fieldName: string]: any;
+}
+
+export interface VoiceFormRecorderResult {
+  success: boolean;
+  data?: FilledFormData;
+  error?: string;
+  transcription?: string;
+  jsonForm?: string;
+}

package/src/types/service-providers.types.ts

@@ -0,0 +1,22 @@
+export interface SpeechToTextProvider {
+  transcribe(audioBlob: Blob, lang: string): Promise<string>;
+}
+
+export interface LLMProvider {
+  fillFormFromTranscription(transcription: string, schema: any): Promise<any>;
+  fillFormFromJson(json: string, schema: any): Promise<any>;
+}
+
+export interface ServiceProviderConfig {
+  speechToText?: {
+    provider: 'whisper' | 'custom';
+    apiKey?: string;
+    baseUrl?: string;
+  };
+  llm?: {
+    provider: 'openai' | 'custom';
+    apiKey?: string;
+    model?: string;
+    baseUrl?: string;
+  };
+}