khoj 1.30.11.dev13__py3-none-any.whl → 1.30.11.dev56__py3-none-any.whl

khoj/interface/compiled/share/chat/index.txt

@@ -1,7 +1,7 @@
 2:I[66513,[],"ClientPageRoot"]
-3:I[5506,["3954","static/chunks/d3ac728e-a9e3522eef9b6b28.js","3072","static/chunks/3072-be830e4f8412b9d2.js","4752","static/chunks/4752-554a3db270186ce3.js","7592","static/chunks/7592-a09c39a38e60634b.js","3690","static/chunks/3690-51312931ba1eae30.js","3463","static/chunks/3463-081c031e873b7966.js","1603","static/chunks/1603-c68d44bc4ae6039a.js","8423","static/chunks/8423-1dda16bc56236523.js","5538","static/chunks/5538-e5f3c9f4d67a64b9.js","3111","static/chunks/app/share/chat/page-cd5757199539bbf2.js"],"default",1]
+3:I[5506,["3954","static/chunks/d3ac728e-a9e3522eef9b6b28.js","3072","static/chunks/3072-be830e4f8412b9d2.js","4752","static/chunks/4752-554a3db270186ce3.js","7592","static/chunks/7592-a09c39a38e60634b.js","3690","static/chunks/3690-51312931ba1eae30.js","3463","static/chunks/3463-081c031e873b7966.js","1603","static/chunks/1603-e40aadd1e56ab030.js","8423","static/chunks/8423-1dda16bc56236523.js","5538","static/chunks/5538-0ea2d3944ca051e1.js","3111","static/chunks/app/share/chat/page-635635e4fb39fe29.js"],"default",1]
 4:I[39275,[],""]
 5:I[61343,[],""]
-0:["K_WyVARSz0loPVvwOW1gg",[[["",{"children":["share",{"children":["chat",{"children":["__PAGE__",{}]}]}]},"$undefined","$undefined",true],["",{"children":["share",{"children":["chat",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],[["$","link","0",{"rel":"stylesheet","href":"/_next/static/css/3cf13271869a4aeb.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/_next/static/css/2d097a35da6bfe8d.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","2",{"rel":"stylesheet","href":"/_next/static/css/1f293605f2871853.css","precedence":"next","crossOrigin":"$undefined"}]]],null],null]},[[null,["$","html",null,{"lang":"en","className":"__variable_a693a0 __variable_702545","children":[["$","meta",null,{"httpEquiv":"Content-Security-Policy","content":"default-src 'self' https://assets.khoj.dev; script-src 'self' https://assets.khoj.dev 'unsafe-inline' 'unsafe-eval'; connect-src 'self' blob: https://ipapi.co/json ws://localhost:42110; style-src 'self' https://assets.khoj.dev 'unsafe-inline' https://fonts.googleapis.com; img-src 'self' data: blob: https://*.khoj.dev https://*.googleusercontent.com https://*.google.com/ https://*.gstatic.com; font-src 'self' https://assets.khoj.dev https://fonts.gstatic.com; child-src 'none'; object-src 'none';"}],["$","body",null,{"children":[["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","share","children","chat","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}],["$","script",null,{"dangerouslySetInnerHTML":{"__html":"window.EXCALIDRAW_ASSET_PATH = 'https://assets.khoj.dev/@excalidraw/excalidraw/dist/';"}}]]}]]}]],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","share","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/_next/static/css/bedf49fbfc598358.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/_next/static/css/3c34171b174cc381.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","className":"__variable_a693a0 __variable_702545","children":[["$","meta",null,{"httpEquiv":"Content-Security-Policy","content":"default-src 'self' https://assets.khoj.dev; media-src * blob:; script-src 'self' https://assets.khoj.dev 'unsafe-inline' 'unsafe-eval'; connect-src 'self' blob: https://ipapi.co/json ws://localhost:42110; style-src 'self' https://assets.khoj.dev 'unsafe-inline' https://fonts.googleapis.com; img-src 'self' data: blob: https://*.khoj.dev https://*.googleusercontent.com https://*.google.com/ https://*.gstatic.com; font-src 'self' https://assets.khoj.dev https://fonts.gstatic.com; child-src 'none'; object-src 'none';"}],["$","body",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]]}]],null],null],["$L6",null]]]]
+0:["J7Vqh1vjCleYuVLeTaJL6",[[["",{"children":["share",{"children":["chat",{"children":["__PAGE__",{}]}]}]},"$undefined","$undefined",true],["",{"children":["share",{"children":["chat",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],[["$","link","0",{"rel":"stylesheet","href":"/_next/static/css/3cf13271869a4aeb.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/_next/static/css/edd3abaf11580924.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","2",{"rel":"stylesheet","href":"/_next/static/css/1f293605f2871853.css","precedence":"next","crossOrigin":"$undefined"}]]],null],null]},[[null,["$","html",null,{"lang":"en","className":"__variable_f36179 __variable_702545","children":[["$","meta",null,{"httpEquiv":"Content-Security-Policy","content":"default-src 'self' https://assets.khoj.dev; media-src * blob:; script-src 'self' https://assets.khoj.dev https://app.chatwoot.com 'unsafe-inline' 'unsafe-eval'; connect-src 'self' blob: https://ipapi.co/json ws://localhost:42110; style-src 'self' https://assets.khoj.dev 'unsafe-inline' https://fonts.googleapis.com; img-src 'self' data: blob: https://*.khoj.dev https://*.googleusercontent.com https://*.google.com/ https://*.gstatic.com; font-src 'self' https://assets.khoj.dev https://fonts.gstatic.com; child-src 'self' https://app.chatwoot.com; object-src 'none';"}],["$","body",null,{"children":[["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","share","children","chat","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}],["$","script",null,{"dangerouslySetInnerHTML":{"__html":"window.EXCALIDRAW_ASSET_PATH = 'https://assets.khoj.dev/@excalidraw/excalidraw/dist/';"}}]]}]]}]],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","share","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/_next/static/css/089de1d8526b96e9.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/_next/static/css/3c34171b174cc381.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","className":"__variable_f36179 __variable_702545","children":[["$","meta",null,{"httpEquiv":"Content-Security-Policy","content":"default-src 'self' https://assets.khoj.dev; media-src * blob:; script-src 'self' https://assets.khoj.dev https://app.chatwoot.com 'unsafe-inline' 'unsafe-eval'; connect-src 'self' blob: https://ipapi.co/json ws://localhost:42110; style-src 'self' https://assets.khoj.dev 'unsafe-inline' https://fonts.googleapis.com; img-src 'self' data: blob: https://*.khoj.dev https://*.googleusercontent.com https://*.google.com/ https://*.gstatic.com; font-src 'self' https://assets.khoj.dev https://fonts.gstatic.com; child-src 'self' https://app.chatwoot.com; object-src 'none';"}],["$","body",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]]}]],null],null],["$L6",null]]]]
 6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"Khoj AI - Chat"}],["$","meta","3",{"name":"description","content":"Use this page to view a chat with Khoj AI."}],["$","link","4",{"rel":"manifest","href":"/static/khoj.webmanifest","crossOrigin":"use-credentials"}],["$","meta","5",{"property":"og:title","content":"Khoj AI"}],["$","meta","6",{"property":"og:description","content":"Your Second Brain."}],["$","meta","7",{"property":"og:url","content":"https://app.khoj.dev/"}],["$","meta","8",{"property":"og:site_name","content":"Khoj AI"}],["$","meta","9",{"property":"og:image","content":"https://assets.khoj.dev/khoj_lantern_256x256.png"}],["$","meta","10",{"property":"og:image:width","content":"256"}],["$","meta","11",{"property":"og:image:height","content":"256"}],["$","meta","12",{"property":"og:image","content":"https://assets.khoj.dev/khoj_lantern_logomarktype_1200x630.png"}],["$","meta","13",{"property":"og:image:width","content":"1200"}],["$","meta","14",{"property":"og:image:height","content":"630"}],["$","meta","15",{"property":"og:type","content":"website"}],["$","meta","16",{"name":"twitter:card","content":"summary_large_image"}],["$","meta","17",{"name":"twitter:title","content":"Khoj AI"}],["$","meta","18",{"name":"twitter:description","content":"Your Second Brain."}],["$","meta","19",{"name":"twitter:image","content":"https://assets.khoj.dev/khoj_lantern_256x256.png"}],["$","meta","20",{"name":"twitter:image:width","content":"256"}],["$","meta","21",{"name":"twitter:image:height","content":"256"}],["$","meta","22",{"name":"twitter:image","content":"https://assets.khoj.dev/khoj_lantern_logomarktype_1200x630.png"}],["$","meta","23",{"name":"twitter:image:width","content":"1200"}],["$","meta","24",{"name":"twitter:image:height","content":"630"}],["$","link","25",{"rel":"icon","href":"/static/assets/icons/khoj_lantern.ico"}],["$","link","26",{"rel":"apple-touch-icon","href":"/static/assets/icons/khoj_lantern_256x256.png"}],["$","meta","27",{"name":"next-size-adjust"}]]
 1:null

khoj/migrations/migrate_server_pg.py

@@ -60,11 +60,7 @@ import logging
 
 from packaging import version
 
-from khoj.database.models import (
-    ChatModelOptions,
-    OpenAIProcessorConversationConfig,
-    SearchModelConfig,
-)
+from khoj.database.models import AiModelApi, ChatModelOptions, SearchModelConfig
 from khoj.utils.yaml import load_config_from_file, save_config_to_file
 
 logger = logging.getLogger(__name__)
@@ -121,16 +117,14 @@ def migrate_server_pg(args):
                     if openai.get("chat-model") is None:
                         openai["chat-model"] = "gpt-3.5-turbo"
 
-                    openai_config = OpenAIProcessorConversationConfig.objects.create(
-                        api_key=openai.get("api-key"), name="default"
-                    )
+                    openai_model_api = AiModelApi.objects.create(api_key=openai.get("api-key"), name="default")
 
                     ChatModelOptions.objects.create(
                         chat_model=openai.get("chat-model"),
                         tokenizer=processor_conversation.get("tokenizer"),
                         max_prompt_size=processor_conversation.get("max-prompt-size"),
                         model_type=ChatModelOptions.ModelType.OPENAI,
-                        openai_config=openai_config,
+                        ai_model_api=openai_model_api,
                     )
 
         save_config_to_file(raw_config, args.config_file)

khoj/processor/conversation/anthropic/anthropic_chat.py

@@ -1,6 +1,6 @@
 import logging
 from datetime import datetime, timedelta
-from typing import Dict, Optional
+from typing import Dict, List, Optional
 
 import pyjson5
 from langchain.schema import ChatMessage
@@ -23,7 +23,7 @@ from khoj.utils.helpers import (
     is_none_or_empty,
     truncate_code_context,
 )
-from khoj.utils.rawconfig import LocationData
+from khoj.utils.rawconfig import FileAttachment, LocationData
 from khoj.utils.yaml import yaml_dump
 
 logger = logging.getLogger(__name__)
@@ -55,7 +55,7 @@ def extract_questions_anthropic(
         [
             f'User: {chat["intent"]["query"]}\nAssistant: {{"queries": {chat["intent"].get("inferred-queries") or list([chat["intent"]["query"]])}}}\nA: {chat["message"]}\n\n'
             for chat in conversation_log.get("chat", [])[-4:]
-            if chat["by"] == "khoj" and "text-to-image" not in chat["intent"].get("type")
+            if chat["by"] == "khoj"
         ]
     )
 
@@ -157,6 +157,10 @@ def converse_anthropic(
     query_images: Optional[list[str]] = None,
     vision_available: bool = False,
     query_files: str = None,
+    generated_images: Optional[list[str]] = None,
+    generated_files: List[FileAttachment] = None,
+    generated_excalidraw_diagram: Optional[str] = None,
+    program_execution_context: Optional[List[str]] = None,
     tracer: dict = {},
 ):
     """
@@ -217,6 +221,10 @@ def converse_anthropic(
         vision_enabled=vision_available,
         model_type=ChatModelOptions.ModelType.ANTHROPIC,
         query_files=query_files,
+        generated_excalidraw_diagram=generated_excalidraw_diagram,
+        generated_files=generated_files,
+        generated_images=generated_images,
+        program_execution_context=program_execution_context,
     )
 
     messages, system_prompt = format_messages_for_anthropic(messages, system_prompt)

khoj/processor/conversation/google/gemini_chat.py

@@ -1,6 +1,6 @@
 import logging
 from datetime import datetime, timedelta
-from typing import Dict, Optional
+from typing import Dict, List, Optional
 
 import pyjson5
 from langchain.schema import ChatMessage
@@ -23,7 +23,7 @@ from khoj.utils.helpers import (
     is_none_or_empty,
     truncate_code_context,
 )
-from khoj.utils.rawconfig import LocationData
+from khoj.utils.rawconfig import FileAttachment, LocationData
 from khoj.utils.yaml import yaml_dump
 
 logger = logging.getLogger(__name__)
@@ -56,7 +56,7 @@ def extract_questions_gemini(
         [
             f'User: {chat["intent"]["query"]}\nAssistant: {{"queries": {chat["intent"].get("inferred-queries") or list([chat["intent"]["query"]])}}}\nA: {chat["message"]}\n\n'
             for chat in conversation_log.get("chat", [])[-4:]
-            if chat["by"] == "khoj" and "text-to-image" not in chat["intent"].get("type")
+            if chat["by"] == "khoj"
         ]
     )
 
@@ -167,6 +167,10 @@ def converse_gemini(
     query_images: Optional[list[str]] = None,
     vision_available: bool = False,
     query_files: str = None,
+    generated_images: Optional[list[str]] = None,
+    generated_files: List[FileAttachment] = None,
+    generated_excalidraw_diagram: Optional[str] = None,
+    program_execution_context: List[str] = None,
     tracer={},
 ):
     """
@@ -228,6 +232,10 @@ def converse_gemini(
         vision_enabled=vision_available,
         model_type=ChatModelOptions.ModelType.GOOGLE,
         query_files=query_files,
+        generated_excalidraw_diagram=generated_excalidraw_diagram,
+        generated_files=generated_files,
+        generated_images=generated_images,
+        program_execution_context=program_execution_context,
     )
 
     messages, system_prompt = format_messages_for_gemini(messages, system_prompt)

khoj/processor/conversation/offline/chat_model.py

@@ -28,7 +28,7 @@ from khoj.utils.helpers import (
     is_promptrace_enabled,
     truncate_code_context,
 )
-from khoj.utils.rawconfig import LocationData
+from khoj.utils.rawconfig import FileAttachment, LocationData
 from khoj.utils.yaml import yaml_dump
 
 logger = logging.getLogger(__name__)
@@ -69,7 +69,7 @@ def extract_questions_offline(
 
     if use_history:
         for chat in conversation_log.get("chat", [])[-4:]:
-            if chat["by"] == "khoj" and "text-to-image" not in chat["intent"].get("type"):
+            if chat["by"] == "khoj":
                 chat_history += f"Q: {chat['intent']['query']}\n"
                 chat_history += f"Khoj: {chat['message']}\n\n"
 
@@ -164,6 +164,8 @@ def converse_offline(
     user_name: str = None,
     agent: Agent = None,
     query_files: str = None,
+    generated_files: List[FileAttachment] = None,
+    additional_context: List[str] = None,
     tracer: dict = {},
 ) -> Union[ThreadedGenerator, Iterator[str]]:
     """
@@ -231,6 +233,8 @@ def converse_offline(
         tokenizer_name=tokenizer_name,
         model_type=ChatModelOptions.ModelType.OFFLINE,
         query_files=query_files,
+        generated_files=generated_files,
+        program_execution_context=additional_context,
     )
 
     logger.debug(f"Conversation Context for {model}: {messages_to_print(messages)}")

khoj/processor/conversation/openai/gpt.py

@@ -1,6 +1,6 @@
 import logging
 from datetime import datetime, timedelta
-from typing import Dict, Optional
+from typing import Dict, List, Optional
 
 import pyjson5
 from langchain.schema import ChatMessage
@@ -22,7 +22,7 @@ from khoj.utils.helpers import (
     is_none_or_empty,
     truncate_code_context,
 )
-from khoj.utils.rawconfig import LocationData
+from khoj.utils.rawconfig import FileAttachment, LocationData
 from khoj.utils.yaml import yaml_dump
 
 logger = logging.getLogger(__name__)
@@ -157,6 +157,10 @@ def converse(
     query_images: Optional[list[str]] = None,
     vision_available: bool = False,
     query_files: str = None,
+    generated_images: Optional[list[str]] = None,
+    generated_files: List[FileAttachment] = None,
+    generated_excalidraw_diagram: Optional[str] = None,
+    program_execution_context: List[str] = None,
     tracer: dict = {},
 ):
     """
@@ -219,6 +223,10 @@ def converse(
         vision_enabled=vision_available,
         model_type=ChatModelOptions.ModelType.OPENAI,
         query_files=query_files,
+        generated_excalidraw_diagram=generated_excalidraw_diagram,
+        generated_files=generated_files,
+        generated_images=generated_images,
+        program_execution_context=program_execution_context,
     )
     logger.debug(f"Conversation Context for GPT: {messages_to_print(messages)}")
 

khoj/processor/conversation/openai/utils.py

@@ -19,12 +19,7 @@ from khoj.processor.conversation.utils import (
     ThreadedGenerator,
     commit_conversation_trace,
 )
-from khoj.utils import state
-from khoj.utils.helpers import (
-    get_chat_usage_metrics,
-    in_debug_mode,
-    is_promptrace_enabled,
-)
+from khoj.utils.helpers import get_chat_usage_metrics, is_promptrace_enabled
 
 logger = logging.getLogger(__name__)
 

khoj/processor/conversation/prompts.py

@@ -178,6 +178,18 @@ Improved Prompt:
 """.strip()
 )
 
+generated_image_attachment = PromptTemplate.from_template(
+    f"""
+Here is the image you generated based on my query. You can follow-up with a general response to my query. Limit to 1-2 sentences.
+""".strip()
+)
+
+generated_diagram_attachment = PromptTemplate.from_template(
+    f"""
+I've successfully created a diagram based on the user's query. The diagram will automatically be shared with the user. I can follow-up with a general response or summary. Limit to 1-2 sentences.
+""".strip()
+)
+
 ## Diagram Generation
 ## --
 
@@ -1029,6 +1041,12 @@ A:
 """.strip()
 )
 
+additional_program_context = PromptTemplate.from_template(
+    """
+Here are some additional results from the query execution:
+{context}
+""".strip()
+)
 
 personality_prompt_safety_expert_lax = PromptTemplate.from_template(
     """

khoj/processor/conversation/utils.py

@@ -154,7 +154,7 @@ def construct_chat_history(conversation_history: dict, n: int = 4, agent_name="A
                 chat_history += f'{agent_name}: {{"queries": {chat["intent"].get("inferred-queries")}}}\n'
 
             chat_history += f"{agent_name}: {chat['message']}\n\n"
-        elif chat["by"] == "khoj" and ("text-to-image" in chat["intent"].get("type")):
+        elif chat["by"] == "khoj" and chat.get("images"):
             chat_history += f"User: {chat['intent']['query']}\n"
             chat_history += f"{agent_name}: [generated image redacted for space]\n"
         elif chat["by"] == "khoj" and ("excalidraw" in chat["intent"].get("type")):
@@ -213,6 +213,7 @@ class ChatEvent(Enum):
     END_LLM_RESPONSE = "end_llm_response"
     MESSAGE = "message"
     REFERENCES = "references"
+    GENERATED_ASSETS = "generated_assets"
     STATUS = "status"
     METADATA = "metadata"
     USAGE = "usage"
@@ -225,7 +226,6 @@ def message_to_log(
     user_message_metadata={},
     khoj_message_metadata={},
     conversation_log=[],
-    train_of_thought=[],
 ):
     """Create json logs from messages, metadata for conversation log"""
     default_khoj_message_metadata = {
@@ -234,6 +234,10 @@ def message_to_log(
     }
     khoj_response_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
 
+    # Filter out any fields that are set to None
+    user_message_metadata = {k: v for k, v in user_message_metadata.items() if v is not None}
+    khoj_message_metadata = {k: v for k, v in khoj_message_metadata.items() if v is not None}
+
     # Create json log from Human's message
     human_log = merge_dicts({"message": user_message, "by": "you"}, user_message_metadata)
 
@@ -261,31 +265,41 @@ def save_to_conversation_log(
     automation_id: str = None,
     query_images: List[str] = None,
     raw_query_files: List[FileAttachment] = [],
+    generated_images: List[str] = [],
+    raw_generated_files: List[FileAttachment] = [],
+    generated_excalidraw_diagram: str = None,
     train_of_thought: List[Any] = [],
     tracer: Dict[str, Any] = {},
 ):
     user_message_time = user_message_time or datetime.now().strftime("%Y-%m-%d %H:%M:%S")
     turn_id = tracer.get("mid") or str(uuid.uuid4())
+
+    user_message_metadata = {"created": user_message_time, "images": query_images, "turnId": turn_id}
+
+    if raw_query_files and len(raw_query_files) > 0:
+        user_message_metadata["queryFiles"] = [file.model_dump(mode="json") for file in raw_query_files]
+
+    khoj_message_metadata = {
+        "context": compiled_references,
+        "intent": {"inferred-queries": inferred_queries, "type": intent_type},
+        "onlineContext": online_results,
+        "codeContext": code_results,
+        "automationId": automation_id,
+        "trainOfThought": train_of_thought,
+        "turnId": turn_id,
+        "images": generated_images,
+        "queryFiles": [file.model_dump(mode="json") for file in raw_generated_files],
+    }
+
+    if generated_excalidraw_diagram:
+        khoj_message_metadata["excalidrawDiagram"] = generated_excalidraw_diagram
+
     updated_conversation = message_to_log(
         user_message=q,
         chat_response=chat_response,
-        user_message_metadata={
-            "created": user_message_time,
-            "images": query_images,
-            "turnId": turn_id,
-            "queryFiles": [file.model_dump(mode="json") for file in raw_query_files],
-        },
-        khoj_message_metadata={
-            "context": compiled_references,
-            "intent": {"inferred-queries": inferred_queries, "type": intent_type},
-            "onlineContext": online_results,
-            "codeContext": code_results,
-            "automationId": automation_id,
-            "trainOfThought": train_of_thought,
-            "turnId": turn_id,
-        },
+        user_message_metadata=user_message_metadata,
+        khoj_message_metadata=khoj_message_metadata,
         conversation_log=meta_log.get("chat", []),
-        train_of_thought=train_of_thought,
     )
     ConversationAdapters.save_conversation(
         user,
@@ -303,13 +317,13 @@ def save_to_conversation_log(
 Saved Conversation Turn
 You ({user.username}): "{q}"
 
-Khoj: "{inferred_queries if ("text-to-image" in intent_type) else chat_response}"
+Khoj: "{chat_response}"
 """.strip()
     )
 
 
 def construct_structured_message(
-    message: str, images: list[str], model_type: str, vision_enabled: bool, attached_file_context: str
+    message: str, images: list[str], model_type: str, vision_enabled: bool, attached_file_context: str = None
 ):
     """
     Format messages into appropriate multimedia format for supported chat model types
@@ -327,7 +341,8 @@ def construct_structured_message(
             constructed_messages.append({"type": "text", "text": attached_file_context})
         if vision_enabled and images:
             for image in images:
-                constructed_messages.append({"type": "image_url", "image_url": {"url": image}})
+                if image.startswith("https://"):
+                    constructed_messages.append({"type": "image_url", "image_url": {"url": image}})
         return constructed_messages
 
     if not is_none_or_empty(attached_file_context):
@@ -365,6 +380,10 @@ def generate_chatml_messages_with_context(
     model_type="",
     context_message="",
     query_files: str = None,
+    generated_images: Optional[list[str]] = None,
+    generated_files: List[FileAttachment] = None,
+    generated_excalidraw_diagram: str = None,
+    program_execution_context: List[str] = [],
 ):
     """Generate chat messages with appropriate context from previous conversation to send to the chat model"""
     # Set max prompt size from user config or based on pre-configured for model and machine specs
@@ -384,6 +403,7 @@ def generate_chatml_messages_with_context(
         message_attached_files = ""
 
         chat_message = chat.get("message")
+        role = "user" if chat["by"] == "you" else "assistant"
 
         if chat["by"] == "khoj" and "excalidraw" in chat["intent"].get("type", ""):
             chat_message = chat["intent"].get("inferred-queries")[0]
@@ -404,7 +424,7 @@ def generate_chatml_messages_with_context(
                 query_files_dict[file["name"]] = file["content"]
 
             message_attached_files = gather_raw_query_files(query_files_dict)
-            chatml_messages.append(ChatMessage(content=message_attached_files, role="user"))
+            chatml_messages.append(ChatMessage(content=message_attached_files, role=role))
 
         if not is_none_or_empty(chat.get("onlineContext")):
             message_context += f"{prompts.online_search_conversation.format(online_results=chat.get('onlineContext'))}"
@@ -413,10 +433,20 @@ def generate_chatml_messages_with_context(
             reconstructed_context_message = ChatMessage(content=message_context, role="user")
             chatml_messages.insert(0, reconstructed_context_message)
 
-        role = "user" if chat["by"] == "you" else "assistant"
-        message_content = construct_structured_message(
-            chat_message, chat.get("images"), model_type, vision_enabled, attached_file_context=query_files
-        )
+        if chat.get("images"):
+            if role == "assistant":
+                # Issue: the assistant role cannot accept an image as a message content, so send it in a separate user message.
+                file_attachment_message = construct_structured_message(
+                    message=prompts.generated_image_attachment.format(),
+                    images=chat.get("images"),
+                    model_type=model_type,
+                    vision_enabled=vision_enabled,
+                )
+                chatml_messages.append(ChatMessage(content=file_attachment_message, role="user"))
+            else:
+                message_content = construct_structured_message(
+                    chat_message, chat.get("images"), model_type, vision_enabled
+                )
 
         reconstructed_message = ChatMessage(content=message_content, role=role)
         chatml_messages.insert(0, reconstructed_message)
@@ -425,6 +455,7 @@ def generate_chatml_messages_with_context(
             break
 
     messages = []
+
     if not is_none_or_empty(user_message):
         messages.append(
             ChatMessage(
@@ -437,6 +468,31 @@ def generate_chatml_messages_with_context(
     if not is_none_or_empty(context_message):
         messages.append(ChatMessage(content=context_message, role="user"))
 
+    if generated_images:
+        messages.append(
+            ChatMessage(
+                content=construct_structured_message(
+                    prompts.generated_image_attachment.format(), generated_images, model_type, vision_enabled
+                ),
+                role="user",
+            )
+        )
+
+    if generated_files:
+        message_attached_files = gather_raw_query_files({file.name: file.content for file in generated_files})
+        messages.append(ChatMessage(content=message_attached_files, role="assistant"))
+
+    if generated_excalidraw_diagram:
+        messages.append(ChatMessage(content=prompts.generated_diagram_attachment.format(), role="assistant"))
+
+    if program_execution_context:
+        messages.append(
+            ChatMessage(
+                content=prompts.additional_program_context.format(context="\n".join(program_execution_context)),
+                role="assistant",
+            )
+        )
+
     if len(chatml_messages) > 0:
         messages += chatml_messages
 

khoj/processor/image/generate.py

@@ -12,7 +12,7 @@ from khoj.database.models import Agent, KhojUser, TextToImageModelConfig
 from khoj.routers.helpers import ChatEvent, generate_better_image_prompt
 from khoj.routers.storage import upload_image
 from khoj.utils import state
-from khoj.utils.helpers import ImageIntentType, convert_image_to_webp, timer
+from khoj.utils.helpers import convert_image_to_webp, timer
 from khoj.utils.rawconfig import LocationData
 
 logger = logging.getLogger(__name__)
@@ -34,14 +34,13 @@ async def text_to_image(
     status_code = 200
     image = None
     image_url = None
-    intent_type = ImageIntentType.TEXT_TO_IMAGE_V3
 
     text_to_image_config = await ConversationAdapters.aget_user_text_to_image_model(user)
     if not text_to_image_config:
         # If the user has not configured a text to image model, return an unsupported on server error
         status_code = 501
         message = "Failed to generate image. Setup image generation on the server."
-        yield image_url or image, status_code, message, intent_type.value
+        yield image_url or image, status_code, message
         return
 
     text2image_model = text_to_image_config.model_name
@@ -50,8 +49,8 @@ async def text_to_image(
         if chat["by"] == "khoj" and chat["intent"].get("type") in ["remember", "reminder"]:
             chat_history += f"Q: {chat['intent']['query']}\n"
             chat_history += f"A: {chat['message']}\n"
-        elif chat["by"] == "khoj" and "text-to-image" in chat["intent"].get("type"):
-            chat_history += f"Q: Prompt: {chat['intent']['query']}\n"
+        elif chat["by"] == "khoj" and chat.get("images"):
+            chat_history += f"Q: {chat['intent']['query']}\n"
             chat_history += f"A: Improved Prompt: {chat['intent']['inferred-queries'][0]}\n"
 
     if send_status_func:
@@ -92,31 +91,29 @@ async def text_to_image(
                 logger.error(f"Image Generation blocked by OpenAI: {e}")
                 status_code = e.status_code  # type: ignore
                 message = f"Image generation blocked by OpenAI due to policy violation"  # type: ignore
-                yield image_url or image, status_code, message, intent_type.value
+                yield image_url or image, status_code, message
                 return
             else:
                 logger.error(f"Image Generation failed with {e}", exc_info=True)
                 message = f"Image generation failed using OpenAI"  # type: ignore
                 status_code = e.status_code  # type: ignore
-                yield image_url or image, status_code, message, intent_type.value
+                yield image_url or image, status_code, message
                 return
         except requests.RequestException as e:
             logger.error(f"Image Generation failed with {e}", exc_info=True)
             message = f"Image generation using {text2image_model} via {text_to_image_config.model_type} failed due to a network error."
             status_code = 502
-            yield image_url or image, status_code, message, intent_type.value
+            yield image_url or image, status_code, message
             return
 
     # Decide how to store the generated image
     with timer("Upload image to S3", logger):
         image_url = upload_image(webp_image_bytes, user.uuid)
-    if image_url:
-        intent_type = ImageIntentType.TEXT_TO_IMAGE2
-    else:
-        intent_type = ImageIntentType.TEXT_TO_IMAGE_V3
+
+    if not image_url:
         image = base64.b64encode(webp_image_bytes).decode("utf-8")
 
-    yield image_url or image, status_code, image_prompt, intent_type.value
+    yield image_url or image, status_code, image_prompt
 
 
 def generate_image_with_openai(
@@ -127,8 +124,8 @@ def generate_image_with_openai(
     # Get the API key from the user's configuration
     if text_to_image_config.api_key:
         api_key = text_to_image_config.api_key
-    elif text_to_image_config.openai_config:
-        api_key = text_to_image_config.openai_config.api_key
+    elif text_to_image_config.ai_model_api:
+        api_key = text_to_image_config.ai_model_api.api_key
     elif state.openai_client:
         api_key = state.openai_client.api_key
     auth_header = {"Authorization": f"Bearer {api_key}"} if api_key else {}

khoj/routers/api.py

@@ -430,9 +430,8 @@ async def extract_references_and_questions(
                 tracer=tracer,
             )
         elif conversation_config.model_type == ChatModelOptions.ModelType.OPENAI:
-            openai_chat_config = conversation_config.openai_config
-            api_key = openai_chat_config.api_key
-            base_url = openai_chat_config.api_base_url
+            api_key = conversation_config.ai_model_api.api_key
+            base_url = conversation_config.ai_model_api.api_base_url
             chat_model = conversation_config.chat_model
             inferred_queries = extract_questions(
                 defiltered_query,
@@ -449,7 +448,7 @@ async def extract_references_and_questions(
                 tracer=tracer,
             )
         elif conversation_config.model_type == ChatModelOptions.ModelType.ANTHROPIC:
-            api_key = conversation_config.openai_config.api_key
+            api_key = conversation_config.ai_model_api.api_key
             chat_model = conversation_config.chat_model
             inferred_queries = extract_questions_anthropic(
                 defiltered_query,
@@ -465,7 +464,7 @@ async def extract_references_and_questions(
                 tracer=tracer,
             )
         elif conversation_config.model_type == ChatModelOptions.ModelType.GOOGLE:
-            api_key = conversation_config.openai_config.api_key
+            api_key = conversation_config.ai_model_api.api_key
             chat_model = conversation_config.chat_model
             inferred_queries = extract_questions_gemini(
                 defiltered_query,
@@ -537,6 +536,7 @@ def user_info(request: Request) -> Response:
         "photo": user_picture,
         "is_active": is_active,
         "has_documents": has_documents,
+        "khoj_version": state.khoj_version,
     }
 
     # Return user information as a JSON response