PraisonAI 0.0.72__cp312-cp312-manylinux_2_35_x86_64.whl → 0.0.74__cp312-cp312-manylinux_2_35_x86_64.whl

praisonai/cli.py

@@ -130,6 +130,10 @@ class PraisonAI:
             self.create_code_interface()
             return
         
+        if getattr(args, 'realtime', False):
+            self.create_realtime_interface()
+            return
+        
         if args.agent_file == 'train':
             package_root = os.path.dirname(os.path.abspath(__file__))
             config_yaml_destination = os.path.join(os.getcwd(), 'config.yaml')
@@ -256,6 +260,7 @@ class PraisonAI:
         parser.add_argument("--hf", type=str, help="Hugging Face model name")
         parser.add_argument("--ollama", type=str, help="Ollama model name")
         parser.add_argument("--dataset", type=str, help="Dataset name for training", default="yahma/alpaca-cleaned")
+        parser.add_argument("--realtime", action="store_true", help="Start the realtime voice interaction interface")
         args, unknown_args = parser.parse_known_args()
 
         if unknown_args and unknown_args[0] == '-b' and unknown_args[1] == 'api:app':
@@ -270,6 +275,8 @@ class PraisonAI:
         if args.agent_file == 'code':
             args.ui = 'chainlit'
             args.code = True
+        if args.agent_file == 'realtime':
+            args.realtime = True
 
         return args
     
@@ -416,6 +423,29 @@ class PraisonAI:
         else:
             print("ERROR: Chainlit is not installed. Please install it with 'pip install \"praisonai\[ui]\"' to use the UI.")        
 
+    def create_realtime_interface(self):
+        """
+        Create a Chainlit interface for the realtime voice interaction application.
+        """
+        if CHAINLIT_AVAILABLE:
+            import praisonai
+            os.environ["CHAINLIT_PORT"] = "8088"  # Ensure this port is not in use by another service
+            root_path = os.path.join(os.path.expanduser("~"), ".praison")
+            os.environ["CHAINLIT_APP_ROOT"] = root_path
+            public_folder = os.path.join(os.path.dirname(praisonai.__file__), 'public')
+            if not os.path.exists(os.path.join(root_path, "public")):
+                if os.path.exists(public_folder):
+                    shutil.copytree(public_folder, os.path.join(root_path, "public"), dirs_exist_ok=True)
+                    logging.info("Public folder copied successfully!")
+                else:
+                    logging.info("Public folder not found in the package.")
+            else:
+                logging.info("Public folder already exists.")
+            realtime_ui_path = os.path.join(os.path.dirname(praisonai.__file__), 'ui', 'realtime.py')
+            chainlit_run([realtime_ui_path])
+        else:
+            print("ERROR: Realtime UI is not installed. Please install it with 'pip install \"praisonai[realtime]\"' to use the realtime UI.")
+
 if __name__ == "__main__":
     praison_ai = PraisonAI()
     praison_ai.main()

praisonai/deploy.py

@@ -56,7 +56,7 @@ class CloudDeployer:
             file.write("FROM python:3.11-slim\n")
             file.write("WORKDIR /app\n")
             file.write("COPY . .\n")
-            file.write("RUN pip install flask praisonai==0.0.72 gunicorn markdown\n")
+            file.write("RUN pip install flask praisonai==0.0.74 gunicorn markdown\n")
             file.write("EXPOSE 8080\n")
             file.write('CMD ["gunicorn", "-b", "0.0.0.0:8080", "api:app"]\n')
             

praisonai/ui/chat.py

@@ -17,6 +17,9 @@ from sql_alchemy import SQLAlchemyDataLayer
 from tavily import TavilyClient
 from crawl4ai import WebCrawler
 import asyncio
+from PIL import Image
+import io
+import base64
 
 # Set up logging
 logger = logging.getLogger(__name__)
@@ -292,11 +295,32 @@ async def main(message: cl.Message):
     message_history = cl.user_session.get("message_history", [])
     now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
     
-    # Add the current date and time to the user's message
+    # Check if an image was uploaded with this message
+    image = None
+    if message.elements and isinstance(message.elements[0], cl.Image):
+        image_element = message.elements[0]
+        try:
+            # Open the image and keep it in memory
+            image = Image.open(image_element.path)
+            image.load()  # This ensures the file is fully loaded into memory
+            cl.user_session.set("image", image)
+        except Exception as e:
+            logger.error(f"Error processing image: {str(e)}")
+            await cl.Message(content="There was an error processing the uploaded image. Please try again.").send()
+            return
+
+    # Prepare user message
     user_message = f"""
-Answer the question and use tools if needed:\n{message.content}.\n\n
+Answer the question and use tools if needed:\n
+
 Current Date and Time: {now}
+
+User Question: {message.content}
 """
+
+    if image:
+        user_message = f"Image uploaded. {user_message}"
+
     message_history.append({"role": "user", "content": user_message})
 
     msg = cl.Message(content="")
@@ -309,6 +333,19 @@ Current Date and Time: {now}
         "stream": True,
     }
 
+    # If an image is uploaded, include it in the message
+    if image:
+        buffered = io.BytesIO()
+        image.save(buffered, format="PNG")
+        img_str = base64.b64encode(buffered.getvalue()).decode()
+        
+        completion_params["messages"][-1] = {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": user_message},
+                {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{img_str}"}}
+            ]
+        }
     # Only add tools and tool_choice if Tavily API key is available
     if tavily_api_key:
         completion_params["tools"] = tools
@@ -359,6 +396,7 @@ Current Date and Time: {now}
     cl.user_session.set("message_history", message_history)
     await msg.update()
 
+    # Handle tool calls if any
     if tavily_api_key and tool_calls:
         available_functions = {
             "tavily_web_search": tavily_web_search,
@@ -411,7 +449,7 @@ Current Date and Time: {now}
         msg.content = full_response
         await msg.update()
     else:
-        # If no tool calls or Tavily API key is not set, the full_response is already set
+        # If no tool calls, the full_response is already set
         msg.content = full_response
         await msg.update()
 
@@ -433,7 +471,7 @@ async def send_count():
     ).send()
 
 @cl.on_chat_resume
-async def on_chat_resume(thread: ThreadDict):  # Change the type hint here
+async def on_chat_resume(thread: ThreadDict):
     logger.info(f"Resuming chat: {thread['id']}")
     model_name = load_setting("model_name") or os.getenv("MODEL_NAME") or "gpt-4o-mini"
     logger.debug(f"Model name: {model_name}")
@@ -481,3 +519,10 @@ async def on_chat_resume(thread: ThreadDict):  # Change the type hint here
             logger.warning(f"Message without recognized type: {message}")
 
     cl.user_session.set("message_history", message_history)
+
+    # Check if there's an image in the thread metadata
+    image_data = metadata.get("image")
+    if image_data:
+        image = Image.open(io.BytesIO(base64.b64decode(image_data)))
+        cl.user_session.set("image", image)
+        await cl.Message(content="Previous image loaded. You can continue asking questions about it or upload a new image.").send()

praisonai/ui/code.py

@@ -18,6 +18,9 @@ from context import ContextGatherer
 from tavily import TavilyClient
 from datetime import datetime
 from crawl4ai import WebCrawler
+from PIL import Image
+import io
+import base64
 
 # Set up logging
 logger = logging.getLogger(__name__)
@@ -303,16 +306,37 @@ tools = [{
 async def main(message: cl.Message):
     model_name = load_setting("model_name") or os.getenv("MODEL_NAME") or "gpt-4o-mini"
     message_history = cl.user_session.get("message_history", [])
-    message_history.append({"role": "user", "content": message.content})
     gatherer = ContextGatherer()
     context, token_count, context_tree = gatherer.run()
     now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-    prompt_history = message_history
-    prompt_history.append({"role": "user", "content": """
-                           Answer the question and use tools if needed:\n{question}.\n\n
-                           Current Date and Time: {now}
-                           Below is the Context:\n{context}\n\n"""
-                           .format(context=context, question=message.content, now=now)})
+
+    # Check if an image was uploaded with this message
+    image = None
+    if message.elements and isinstance(message.elements[0], cl.Image):
+        image_element = message.elements[0]
+        try:
+            # Open the image and keep it in memory
+            image = Image.open(image_element.path)
+            image.load()  # This ensures the file is fully loaded into memory
+            cl.user_session.set("image", image)
+        except Exception as e:
+            logger.error(f"Error processing image: {str(e)}")
+            await cl.Message(content="There was an error processing the uploaded image. Please try again.").send()
+            return
+
+    # Prepare user message
+    user_message = f"""
+Answer the question and use tools if needed:\n{message.content}.\n\n
+Current Date and Time: {now}
+
+Context:
+{context}
+"""
+
+    if image:
+        user_message = f"Image uploaded. {user_message}"
+
+    message_history.append({"role": "user", "content": user_message})
 
     msg = cl.Message(content="")
     await msg.send()
@@ -320,11 +344,27 @@ async def main(message: cl.Message):
     # Prepare the completion parameters
     completion_params = {
         "model": model_name,
-        "messages": prompt_history,
+        "messages": message_history,
         "stream": True,
     }
 
-    # Only add tools and tool_choice if Tavily API key is available
+    # If an image is uploaded, include it in the message
+    if image:
+        buffered = io.BytesIO()
+        image.save(buffered, format="PNG")
+        img_str = base64.b64encode(buffered.getvalue()).decode()
+        
+        completion_params["messages"][-1] = {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": user_message},
+                {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{img_str}"}}
+            ]
+        }
+        # Use a vision-capable model when an image is present
+        completion_params["model"] = "gpt-4-vision-preview"  # Adjust this to your actual vision-capable model
+
+    # Only add tools and tool_choice if Tavily API key is available and no image is uploaded
     if tavily_api_key:
         completion_params["tools"] = tools
         completion_params["tool_choice"] = "auto"
@@ -380,7 +420,7 @@ async def main(message: cl.Message):
         available_functions = {
             "tavily_web_search": tavily_web_search,
         }
-        messages = prompt_history + [{"role": "assistant", "content": None, "function_call": {
+        messages = message_history + [{"role": "assistant", "content": None, "function_call": {
             "name": tool_calls[0]['function']['name'],
             "arguments": tool_calls[0]['function']['arguments']
         }}]
@@ -497,3 +537,10 @@ async def on_chat_resume(thread: ThreadDict):
             logger.warning(f"Message without recognized type: {message}")
 
     cl.user_session.set("message_history", message_history)
+
+    # Check if there's an image in the thread metadata
+    image_data = metadata.get("image")
+    if image_data:
+        image = Image.open(io.BytesIO(base64.b64decode(image_data)))
+        cl.user_session.set("image", image)
+        await cl.Message(content="Previous image loaded. You can continue asking questions about it, upload a new image, or just chat.").send()

praisonai/ui/realtime.py

@@ -0,0 +1,368 @@
+import os
+import asyncio
+import sqlite3
+from datetime import datetime
+from uuid import uuid4
+
+from openai import AsyncOpenAI
+import chainlit as cl
+from chainlit.logger import logger
+from chainlit.input_widget import TextInput
+from chainlit.types import ThreadDict
+
+from realtimeclient import RealtimeClient
+from realtimeclient.tools import tools
+from sql_alchemy import SQLAlchemyDataLayer
+import chainlit.data as cl_data
+from literalai.helper import utc_now
+import json
+
+# Set up CHAINLIT_AUTH_SECRET
+CHAINLIT_AUTH_SECRET = os.getenv("CHAINLIT_AUTH_SECRET")
+
+if not CHAINLIT_AUTH_SECRET:
+    os.environ["CHAINLIT_AUTH_SECRET"] = "p8BPhQChpg@J>jBz$wGxqLX2V>yTVgP*7Ky9H$aV:axW~ANNX-7_T:o@lnyCBu^U"
+    CHAINLIT_AUTH_SECRET = os.getenv("CHAINLIT_AUTH_SECRET")
+
+# Database path
+DB_PATH = os.path.expanduser("~/.praison/database.sqlite")
+
+def initialize_db():
+    os.makedirs(os.path.dirname(DB_PATH), exist_ok=True)
+    conn = sqlite3.connect(DB_PATH)
+    cursor = conn.cursor()
+    cursor.execute('''
+        CREATE TABLE IF NOT EXISTS users (
+            id UUID PRIMARY KEY,
+            identifier TEXT NOT NULL UNIQUE,
+            metadata JSONB NOT NULL,
+            createdAt TEXT
+        )
+    ''')
+    cursor.execute('''
+        CREATE TABLE IF NOT EXISTS threads (
+            id UUID PRIMARY KEY,
+            createdAt TEXT,
+            name TEXT,
+            userId UUID,
+            userIdentifier TEXT,
+            tags TEXT[],
+            metadata JSONB NOT NULL DEFAULT '{}',
+            FOREIGN KEY (userId) REFERENCES users(id) ON DELETE CASCADE
+        )
+    ''')
+    cursor.execute('''
+        CREATE TABLE IF NOT EXISTS steps (
+            id UUID PRIMARY KEY,
+            name TEXT NOT NULL,
+            type TEXT NOT NULL,
+            threadId UUID NOT NULL,
+            parentId UUID,
+            disableFeedback BOOLEAN NOT NULL DEFAULT 0,
+            streaming BOOLEAN NOT NULL DEFAULT 0,
+            waitForAnswer BOOLEAN DEFAULT 0,
+            isError BOOLEAN NOT NULL DEFAULT 0,
+            metadata JSONB DEFAULT '{}',
+            tags TEXT[],
+            input TEXT,
+            output TEXT,
+            createdAt TEXT,
+            start TEXT,
+            end TEXT,
+            generation JSONB,
+            showInput TEXT,
+            language TEXT,
+            indent INT,
+            FOREIGN KEY (threadId) REFERENCES threads (id) ON DELETE CASCADE
+        )
+    ''')
+    cursor.execute('''
+        CREATE TABLE IF NOT EXISTS elements (
+            id UUID PRIMARY KEY,
+            threadId UUID,
+            type TEXT,
+            url TEXT,
+            chainlitKey TEXT,
+            name TEXT NOT NULL,
+            display TEXT,
+            objectKey TEXT,
+            size TEXT,
+            page INT,
+            language TEXT,
+            forId UUID,
+            mime TEXT,
+            FOREIGN KEY (threadId) REFERENCES threads (id) ON DELETE CASCADE
+        )
+    ''')
+    cursor.execute('''
+        CREATE TABLE IF NOT EXISTS feedbacks (
+            id UUID PRIMARY KEY,
+            forId UUID NOT NULL,
+            value INT NOT NULL,
+            threadId UUID,
+            comment TEXT
+        )
+    ''')
+    cursor.execute('''
+        CREATE TABLE IF NOT EXISTS settings (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            key TEXT UNIQUE,
+            value TEXT
+        )
+    ''')
+    conn.commit()
+    conn.close()
+
+def save_setting(key: str, value: str):
+    """Saves a setting to the database."""
+    conn = sqlite3.connect(DB_PATH)
+    cursor = conn.cursor()
+    cursor.execute(
+        """
+        INSERT OR REPLACE INTO settings (id, key, value)
+        VALUES ((SELECT id FROM settings WHERE key = ?), ?, ?)
+    """,
+        (key, key, value),
+    )
+    conn.commit()
+    conn.close()
+
+def load_setting(key: str) -> str:
+    """Loads a setting from the database."""
+    conn = sqlite3.connect(DB_PATH)
+    cursor = conn.cursor()
+    cursor.execute('SELECT value FROM settings WHERE key = ?', (key,))
+    result = cursor.fetchone()
+    conn.close()
+    return result[0] if result else None
+
+# Initialize the database
+initialize_db()
+
+# Set up SQLAlchemy data layer
+cl_data._data_layer = SQLAlchemyDataLayer(conninfo=f"sqlite+aiosqlite:///{DB_PATH}")
+
+client = AsyncOpenAI()
+
+@cl.on_chat_start
+async def start():
+    initialize_db()
+    model_name = load_setting("model_name") or os.getenv("MODEL_NAME", "gpt-4o-mini")
+    cl.user_session.set("model_name", model_name)
+    cl.user_session.set("message_history", [])  # Initialize message history
+    logger.debug(f"Model name: {model_name}")
+    # settings = cl.ChatSettings(
+    #     [
+    #         TextInput(
+    #             id="model_name",
+    #             label="Enter the Model Name",
+    #             placeholder="e.g., gpt-4o-mini",
+    #             initial=model_name
+    #         )
+    #     ]
+    # )
+    # cl.user_session.set("settings", settings)
+    # await settings.send()
+    await cl.Message(
+        content="Welcome to the PraisonAI realtime. Press `P` to talk!"
+    ).send()
+    await setup_openai_realtime()
+
+@cl.on_message
+async def on_message(message: cl.Message):
+    openai_realtime: RealtimeClient = cl.user_session.get("openai_realtime")
+    message_history = cl.user_session.get("message_history", [])
+    
+    if openai_realtime and openai_realtime.is_connected():
+        current_date = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        prompt = f"Current time Just for reference: {current_date}\n\n{message.content}"
+        
+        # Add user message to history
+        message_history.append({"role": "user", "content": prompt})
+        cl.user_session.set("message_history", message_history)
+        
+        await openai_realtime.send_user_message_content([{ "type": 'input_text', "text": message.content }])
+    else:
+        await cl.Message(content="Please activate voice mode before sending messages!").send()
+
+async def setup_openai_realtime():
+    """Instantiate and configure the OpenAI Realtime Client"""
+    openai_realtime = RealtimeClient(api_key=os.getenv("OPENAI_API_KEY"))
+    cl.user_session.set("track_id", str(uuid4()))
+
+    async def handle_conversation_updated(event):
+        item = event.get("item")
+        delta = event.get("delta")
+        """Currently used to stream audio back to the client."""
+        if delta:
+            if 'audio' in delta:
+                audio = delta['audio']  # Int16Array, audio added
+                await cl.context.emitter.send_audio_chunk(cl.OutputAudioChunk(mimeType="pcm16", data=audio, track=cl.user_session.get("track_id")))
+            if 'transcript' in delta:
+                transcript = delta['transcript']  # string, transcript added
+                logger.debug(f"Transcript delta: {transcript}")
+            if 'text' in delta:
+                text = delta['text']  # string, text added
+                logger.debug(f"Text delta: {text}")
+            if 'arguments' in delta:
+                arguments = delta['arguments']  # string, function arguments added
+                logger.debug(f"Function arguments delta: {arguments}")
+
+    async def handle_item_completed(event):
+        """Used to populate the chat context with transcription once an item is completed."""
+        try:
+            item = event.get("item")
+            logger.debug(f"Item completed: {json.dumps(item, indent=2, default=str)}")
+            await openai_realtime._send_chainlit_message(item)
+            
+            # Add assistant message to history
+            message_history = cl.user_session.get("message_history", [])
+            content = item.get("formatted", {}).get("text", "") or item.get("formatted", {}).get("transcript", "")
+            if content:
+                message_history.append({"role": "assistant", "content": content})
+                cl.user_session.set("message_history", message_history)
+        except Exception as e:
+            error_message = f"Error in handle_item_completed: {str(e)}"
+            logger.error(error_message)
+            debug_item = json.dumps(item, indent=2, default=str)
+            logger.error(f"Item causing error: {debug_item}")
+
+    async def handle_conversation_interrupt(event):
+        """Used to cancel the client previous audio playback."""
+        cl.user_session.set("track_id", str(uuid4()))
+        await cl.context.emitter.send_audio_interrupt()
+
+    async def handle_error(event):
+        logger.error(event)
+        await cl.Message(content=f"Error: {event}", author="System").send()
+
+    openai_realtime.on('conversation.updated', handle_conversation_updated)
+    openai_realtime.on('conversation.item.completed', handle_item_completed)
+    openai_realtime.on('conversation.interrupted', handle_conversation_interrupt)
+    openai_realtime.on('error', handle_error)
+
+    cl.user_session.set("openai_realtime", openai_realtime)
+    coros = [openai_realtime.add_tool(tool_def, tool_handler) for tool_def, tool_handler in tools]
+    await asyncio.gather(*coros)
+
+@cl.on_settings_update
+async def setup_agent(settings):
+    logger.debug(settings)
+    cl.user_session.set("settings", settings)
+    model_name = settings["model_name"]
+    cl.user_session.set("model_name", model_name)
+    
+    # Save in settings table
+    save_setting("model_name", model_name)
+    
+    # Save in thread metadata
+    thread_id = cl.user_session.get("thread_id")
+    if thread_id:
+        thread = await cl_data._data_layer.get_thread(thread_id)
+        if thread:
+            metadata = thread.get("metadata", {})
+            if isinstance(metadata, str):
+                try:
+                    metadata = json.loads(metadata)
+                except json.JSONDecodeError:
+                    metadata = {}
+            
+            metadata["model_name"] = model_name
+            
+            # Always store metadata as a dictionary
+            await cl_data._data_layer.update_thread(thread_id, metadata=metadata)
+            
+            # Update the user session with the new metadata
+            cl.user_session.set("metadata", metadata)
+
+@cl.on_audio_start
+async def on_audio_start():
+    try:
+        openai_realtime: RealtimeClient = cl.user_session.get("openai_realtime")
+        await openai_realtime.connect()
+        logger.info("Connected to OpenAI realtime")
+        return True
+    except Exception as e:
+        await cl.ErrorMessage(content=f"Failed to connect to OpenAI realtime: {e}").send()
+        return False
+
+@cl.on_audio_chunk
+async def on_audio_chunk(chunk: cl.InputAudioChunk):
+    openai_realtime: RealtimeClient = cl.user_session.get("openai_realtime")            
+    if openai_realtime.is_connected():
+        await openai_realtime.append_input_audio(chunk.data)
+    else:
+        logger.info("RealtimeClient is not connected")
+
+@cl.on_audio_end
+@cl.on_chat_end
+@cl.on_stop
+async def on_end():
+    openai_realtime: RealtimeClient = cl.user_session.get("openai_realtime")
+    if openai_realtime and openai_realtime.is_connected():
+        await openai_realtime.disconnect()
+
+@cl.password_auth_callback
+def auth_callback(username: str, password: str):
+    # You can customize this function to use your own authentication logic
+    expected_username = os.getenv("CHAINLIT_USERNAME", "admin")
+    expected_password = os.getenv("CHAINLIT_PASSWORD", "admin")
+    if (username, password) == (expected_username, expected_password):
+        return cl.User(
+            identifier=username, metadata={"role": "ADMIN", "provider": "credentials"}
+        )
+    else:
+        return None
+
+@cl.on_chat_resume
+async def on_chat_resume(thread: ThreadDict):
+    logger.info(f"Resuming chat: {thread['id']}")
+    model_name = load_setting("model_name") or os.getenv("MODEL_NAME") or "gpt-4o-mini"
+    logger.debug(f"Model name: {model_name}")
+    settings = cl.ChatSettings(
+        [
+            TextInput(
+                id="model_name",
+                label="Enter the Model Name",
+                placeholder="e.g., gpt-4o-mini",
+                initial=model_name
+            )
+        ]
+    )
+    await settings.send()
+    thread_id = thread["id"]
+    cl.user_session.set("thread_id", thread["id"])
+    
+    # Ensure metadata is a dictionary
+    metadata = thread.get("metadata", {})
+    if isinstance(metadata, str):
+        try:
+            metadata = json.loads(metadata)
+        except json.JSONDecodeError:
+            metadata = {}
+    
+    cl.user_session.set("metadata", metadata)
+    
+    message_history = []
+    steps = thread["steps"]
+
+    for message in steps:
+        msg_type = message.get("type")
+        if msg_type == "user_message":
+            message_history.append({"role": "user", "content": message.get("output", "")})
+        elif msg_type == "assistant_message":
+            message_history.append({"role": "assistant", "content": message.get("output", "")})
+        elif msg_type == "run":
+            # Handle 'run' type messages
+            if message.get("isError"):
+                message_history.append({"role": "system", "content": f"Error: {message.get('output', '')}"})
+            else:
+                # You might want to handle non-error 'run' messages differently
+                pass
+        else:
+            logger.warning(f"Message without recognized type: {message}")
+
+    cl.user_session.set("message_history", message_history)
+
+    # Reconnect to OpenAI realtime
+    await setup_openai_realtime()