symposium 0.13.6 → 0.13.8

package/Agent.js

@@ -205,6 +205,19 @@ export default class Agent {
 	async generateCompletion(thread, options = {}, retry_counter = 1) {
 		try {
 			const model = Symposium.getModelByName(thread.state.model);
+
+			for (let message of thread.messages) {
+				for (let c of message.content) {
+					if (c.type === 'audio' && !model.supports_audio) {
+						const words = await this.getPromptWordsForTranscription(thread);
+						const prompt = words.length ? 'Possibili parole usate: ' + words.join(', ') : null;
+						const transcribed = await Symposium.transcribe(c.content, prompt);
+						c.type = 'text';
+						c.content = '[voice message] ' + transcribed;
+					}
+				}
+			}
+
 			const messages = await model.generate(thread, await this.getFunctions(), options);
 			return model.supports_functions ? messages : messages.map(m => this.parseFunctions(m));
 		} catch (error) {

package/Model.js

@@ -6,6 +6,7 @@ export default class Model {
 	supports_functions = false;
 	supports_structured_output = false;
 	system_role_name = 'system';
+	supports_audio = false;
 
 	constructor() {
 		if (!this.label)

package/Symposium.js

@@ -1,3 +1,5 @@
+import fs from 'fs';
+
 import Gpt35 from "./models/Gpt35.js";
 import Gpt4 from "./models/Gpt4.js";
 import Gpt4Turbo from "./models/Gpt4Turbo.js";
@@ -17,6 +19,7 @@ import DeepSeekReasoner from "./models/DeepSeekReasoner.js";
 export default class Symposium {
 	static models = new Map();
 	static storage = null;
+	static transcription_model = null;
 
 	/*
 	* Storage must expose the following methods:
@@ -74,4 +77,52 @@ export default class Symposium {
 
 		return functions;
 	}
+
+	static async transcribe(audio, prompt = null) {
+		if (!process.env.TRANSCRIPTION_MODEL)
+			throw new Error('Transcription is not enabled');
+
+		let file;
+		switch (audio.type) {
+			case 'url':
+				if (!audio.url)
+					throw new Error('Audio URL is required');
+
+				if (audio.url.startsWith('/')) { // Local path
+					// Get with fs
+					if (!fs.existsSync(audio.url))
+						throw new Error('Audio file does not exist at the specified path: ' + audio.url);
+
+					file = fs.readFileSync(audio.url);
+				} else {
+					file = await fetch(audio.url).then(res => res.blob());
+				}
+
+				file = new File([file], 'audio.' + this.getExtFromMime(file.type), {type: file.type});
+				break;
+
+			case 'base64':
+				file = new File([Buffer.from(audio.data, 'base64')], 'audio.' + this.getExtFromMime(audio.type), {type: audio.type});
+				break;
+		}
+
+		if (!this.transcription_model)
+			this.transcription_model = Symposium.getModelByName(process.env.TRANSCRIPTION_MODEL);
+
+		return this.transcription_model.transcribe(file, prompt);
+	}
+
+	static getExtFromMime(mime) {
+		const mimeToExt = {
+			'audio/mpeg': 'mp3',
+			'audio/wav': 'wav',
+			'audio/ogg': 'ogg',
+			'audio/flac': 'flac',
+			'audio/aac': 'aac',
+			'audio/mp4': 'm4a',
+			'audio/webm': 'webm',
+		};
+
+		return mimeToExt[mime] || null;
+	}
 }

package/models/Whisper.js

@@ -4,14 +4,13 @@ export default class Whisper extends OpenAIModel {
 	type = 'stt';
 	name = 'whisper';
 
-	async transcribe(agent, thread, file) {
-		const words = await agent.getPromptWordsForTranscription(thread);
-
+	async transcribe(file, prompt = null) {
 		const response = await this.getOpenAi().audio.transcriptions.create({
+			model: 'gpt-4o-transcribe',
 			file,
-			model: 'whisper-1',
-			prompt: words.join(', '),
+			prompt,
 		});
+
 		return response.text;
 	}
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "type": "module",
   "name": "symposium",
-  "version": "0.13.6",
+  "version": "0.13.8",
   "description": "Agents",
   "main": "index.js",
   "author": "Domenico Giambra",