khoj 1.24.2.dev3__py3-none-any.whl → 1.24.2.dev16__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- khoj/database/adapters/__init__.py +139 -16
- khoj/database/admin.py +2 -0
- khoj/database/migrations/0065_remove_agent_avatar_remove_agent_public_and_more.py +49 -0
- khoj/database/migrations/0066_remove_agent_tools_agent_input_tools_and_more.py +69 -0
- khoj/database/migrations/0067_alter_agent_style_icon.py +50 -0
- khoj/database/models/__init__.py +60 -18
- khoj/interface/compiled/404/index.html +1 -1
- khoj/interface/compiled/_next/static/chunks/1269-2e52d48e7d0e5c61.js +1 -0
- khoj/interface/compiled/_next/static/chunks/1603-67a89278e2c5dbe6.js +1 -0
- khoj/interface/compiled/_next/static/chunks/2697-a38d01981ad3bdf8.js +1 -0
- khoj/interface/compiled/_next/static/chunks/3110-ef2cacd1b8d79ad8.js +1 -0
- khoj/interface/compiled/_next/static/chunks/4086-2c74808ba38a5a0f.js +1 -0
- khoj/interface/compiled/_next/static/chunks/477-ec86e93db10571c1.js +1 -0
- khoj/interface/compiled/_next/static/chunks/51-e8f5bdb69b5ea421.js +1 -0
- khoj/interface/compiled/_next/static/chunks/9178-899fe9a6b754ecfe.js +1 -0
- khoj/interface/compiled/_next/static/chunks/9417-29502e39c3e7d60c.js +1 -0
- khoj/interface/compiled/_next/static/chunks/9479-7eed36fc954ef804.js +1 -0
- khoj/interface/compiled/_next/static/chunks/app/agents/page-df26b497b7356151.js +1 -0
- khoj/interface/compiled/_next/static/chunks/app/automations/page-1688dead2f21270d.js +1 -0
- khoj/interface/compiled/_next/static/chunks/app/chat/page-91abcb71846922b7.js +1 -0
- khoj/interface/compiled/_next/static/chunks/app/factchecker/page-7ab093711c27041c.js +1 -0
- khoj/interface/compiled/_next/static/chunks/app/page-fada198096eab47f.js +1 -0
- khoj/interface/compiled/_next/static/chunks/app/search/page-a7e036689b6507ff.js +1 -0
- khoj/interface/compiled/_next/static/chunks/app/settings/page-fa11cafaec7ab39f.js +1 -0
- khoj/interface/compiled/_next/static/chunks/app/share/chat/page-c5d2b9076e5390b2.js +1 -0
- khoj/interface/compiled/_next/static/chunks/{webpack-d4781cada9b58e75.js → webpack-f52083d548d804fa.js} +1 -1
- khoj/interface/compiled/_next/static/css/50d972a8c787730b.css +25 -0
- khoj/interface/compiled/_next/static/css/592ca99f5122e75a.css +1 -0
- khoj/interface/compiled/_next/static/css/dfb67a9287720a2b.css +1 -0
- khoj/interface/compiled/agents/index.html +1 -1
- khoj/interface/compiled/agents/index.txt +2 -2
- khoj/interface/compiled/automations/index.html +1 -1
- khoj/interface/compiled/automations/index.txt +2 -2
- khoj/interface/compiled/chat/index.html +1 -1
- khoj/interface/compiled/chat/index.txt +2 -2
- khoj/interface/compiled/factchecker/index.html +1 -1
- khoj/interface/compiled/factchecker/index.txt +2 -2
- khoj/interface/compiled/index.html +1 -1
- khoj/interface/compiled/index.txt +2 -2
- khoj/interface/compiled/search/index.html +1 -1
- khoj/interface/compiled/search/index.txt +2 -2
- khoj/interface/compiled/settings/index.html +1 -1
- khoj/interface/compiled/settings/index.txt +2 -2
- khoj/interface/compiled/share/chat/index.html +1 -1
- khoj/interface/compiled/share/chat/index.txt +2 -2
- khoj/processor/content/notion/notion_to_entries.py +2 -1
- khoj/processor/conversation/anthropic/anthropic_chat.py +2 -0
- khoj/processor/conversation/google/gemini_chat.py +2 -0
- khoj/processor/conversation/offline/chat_model.py +3 -1
- khoj/processor/conversation/openai/gpt.py +2 -0
- khoj/processor/conversation/prompts.py +56 -5
- khoj/processor/image/generate.py +3 -1
- khoj/processor/tools/online_search.py +9 -7
- khoj/routers/api.py +34 -5
- khoj/routers/api_agents.py +232 -4
- khoj/routers/api_chat.py +46 -17
- khoj/routers/api_content.py +14 -0
- khoj/routers/helpers.py +113 -13
- khoj/search_type/text_search.py +4 -1
- khoj/utils/helpers.py +15 -2
- {khoj-1.24.2.dev3.dist-info → khoj-1.24.2.dev16.dist-info}/METADATA +1 -8
- {khoj-1.24.2.dev3.dist-info → khoj-1.24.2.dev16.dist-info}/RECORD +67 -64
- khoj/interface/compiled/_next/static/chunks/1603-3e2e1528e3b6ea1d.js +0 -1
- khoj/interface/compiled/_next/static/chunks/2697-a29cb9191a9e339c.js +0 -1
- khoj/interface/compiled/_next/static/chunks/6648-ee109f4ea33a74e2.js +0 -1
- khoj/interface/compiled/_next/static/chunks/7071-b4711cecca6619a8.js +0 -1
- khoj/interface/compiled/_next/static/chunks/743-1a64254447cda71f.js +0 -1
- khoj/interface/compiled/_next/static/chunks/8423-62ac6c832be2461b.js +0 -1
- khoj/interface/compiled/_next/static/chunks/9162-0be016519a18568b.js +0 -1
- khoj/interface/compiled/_next/static/chunks/9178-7e815211edcb3657.js +0 -1
- khoj/interface/compiled/_next/static/chunks/9417-5d14ac74aaab2c66.js +0 -1
- khoj/interface/compiled/_next/static/chunks/9984-e410179c6fac7cf1.js +0 -1
- khoj/interface/compiled/_next/static/chunks/app/agents/page-d302911777a3e027.js +0 -1
- khoj/interface/compiled/_next/static/chunks/app/automations/page-0a5de8c254c29a1c.js +0 -1
- khoj/interface/compiled/_next/static/chunks/app/chat/page-d96bf6a84bb05290.js +0 -1
- khoj/interface/compiled/_next/static/chunks/app/factchecker/page-32e61af29e6b431d.js +0 -1
- khoj/interface/compiled/_next/static/chunks/app/page-96cab08c985716f4.js +0 -1
- khoj/interface/compiled/_next/static/chunks/app/search/page-b3193d46c65571c5.js +0 -1
- khoj/interface/compiled/_next/static/chunks/app/settings/page-0db9b708366606ec.js +0 -1
- khoj/interface/compiled/_next/static/chunks/app/share/chat/page-f06ac16cfe5b5a16.js +0 -1
- khoj/interface/compiled/_next/static/css/24f141a6e37cd204.css +0 -25
- khoj/interface/compiled/_next/static/css/3e1f1fdd70775091.css +0 -1
- khoj/interface/compiled/_next/static/css/f768dddada62459d.css +0 -1
- /khoj/interface/compiled/_next/static/{_29ceahp81LhuIHo5QgOD → MyYNlmGMz32TGV_-febR4}/_buildManifest.js +0 -0
- /khoj/interface/compiled/_next/static/{_29ceahp81LhuIHo5QgOD → MyYNlmGMz32TGV_-febR4}/_ssgManifest.js +0 -0
- {khoj-1.24.2.dev3.dist-info → khoj-1.24.2.dev16.dist-info}/WHEEL +0 -0
- {khoj-1.24.2.dev3.dist-info → khoj-1.24.2.dev16.dist-info}/entry_points.txt +0 -0
- {khoj-1.24.2.dev3.dist-info → khoj-1.24.2.dev16.dist-info}/licenses/LICENSE +0 -0
@@ -10,6 +10,7 @@ from enum import Enum
|
|
10
10
|
from typing import Callable, Iterable, List, Optional, Type
|
11
11
|
|
12
12
|
import cron_descriptor
|
13
|
+
import django
|
13
14
|
from apscheduler.job import Job
|
14
15
|
from asgiref.sync import sync_to_async
|
15
16
|
from django.contrib.sessions.backends.db import SessionStore
|
@@ -551,26 +552,62 @@ class ClientApplicationAdapters:
|
|
551
552
|
|
552
553
|
class AgentAdapters:
|
553
554
|
DEFAULT_AGENT_NAME = "Khoj"
|
554
|
-
DEFAULT_AGENT_AVATAR = "https://assets.khoj.dev/lamp-128.png"
|
555
555
|
DEFAULT_AGENT_SLUG = "khoj"
|
556
556
|
|
557
|
+
@staticmethod
|
558
|
+
async def aget_readonly_agent_by_slug(agent_slug: str, user: KhojUser):
|
559
|
+
return await Agent.objects.filter(
|
560
|
+
(Q(slug__iexact=agent_slug.lower()))
|
561
|
+
& (
|
562
|
+
Q(privacy_level=Agent.PrivacyLevel.PUBLIC)
|
563
|
+
| Q(privacy_level=Agent.PrivacyLevel.PROTECTED)
|
564
|
+
| Q(creator=user)
|
565
|
+
)
|
566
|
+
).afirst()
|
567
|
+
|
568
|
+
@staticmethod
|
569
|
+
async def adelete_agent_by_slug(agent_slug: str, user: KhojUser):
|
570
|
+
agent = await AgentAdapters.aget_agent_by_slug(agent_slug, user)
|
571
|
+
if agent:
|
572
|
+
await agent.adelete()
|
573
|
+
return True
|
574
|
+
return False
|
575
|
+
|
557
576
|
@staticmethod
|
558
577
|
async def aget_agent_by_slug(agent_slug: str, user: KhojUser):
|
559
578
|
return await Agent.objects.filter(
|
560
|
-
(Q(slug__iexact=agent_slug.lower())) & (Q(
|
579
|
+
(Q(slug__iexact=agent_slug.lower())) & (Q(privacy_level=Agent.PrivacyLevel.PUBLIC) | Q(creator=user))
|
580
|
+
).afirst()
|
581
|
+
|
582
|
+
@staticmethod
|
583
|
+
async def aget_agent_by_name(agent_name: str, user: KhojUser):
|
584
|
+
return await Agent.objects.filter(
|
585
|
+
(Q(name__iexact=agent_name.lower())) & (Q(privacy_level=Agent.PrivacyLevel.PUBLIC) | Q(creator=user))
|
561
586
|
).afirst()
|
562
587
|
|
563
588
|
@staticmethod
|
564
589
|
def get_agent_by_slug(slug: str, user: KhojUser = None):
|
565
590
|
if user:
|
566
|
-
return Agent.objects.filter(
|
567
|
-
|
591
|
+
return Agent.objects.filter(
|
592
|
+
(Q(slug__iexact=slug.lower())) & (Q(privacy_level=Agent.PrivacyLevel.PUBLIC) | Q(creator=user))
|
593
|
+
).first()
|
594
|
+
return Agent.objects.filter(slug__iexact=slug.lower(), privacy_level=Agent.PrivacyLevel.PUBLIC).first()
|
568
595
|
|
569
596
|
@staticmethod
|
570
597
|
def get_all_accessible_agents(user: KhojUser = None):
|
598
|
+
public_query = Q(privacy_level=Agent.PrivacyLevel.PUBLIC)
|
571
599
|
if user:
|
572
|
-
return
|
573
|
-
|
600
|
+
return (
|
601
|
+
Agent.objects.filter(public_query | Q(creator=user))
|
602
|
+
.distinct()
|
603
|
+
.order_by("created_at")
|
604
|
+
.prefetch_related("creator", "chat_model", "fileobject_set")
|
605
|
+
)
|
606
|
+
return (
|
607
|
+
Agent.objects.filter(public_query)
|
608
|
+
.order_by("created_at")
|
609
|
+
.prefetch_related("creator", "chat_model", "fileobject_set")
|
610
|
+
)
|
574
611
|
|
575
612
|
@staticmethod
|
576
613
|
async def aget_all_accessible_agents(user: KhojUser = None) -> List[Agent]:
|
@@ -604,17 +641,19 @@ class AgentAdapters:
|
|
604
641
|
agent.chat_model = default_conversation_config
|
605
642
|
agent.slug = AgentAdapters.DEFAULT_AGENT_SLUG
|
606
643
|
agent.name = AgentAdapters.DEFAULT_AGENT_NAME
|
644
|
+
agent.privacy_level = Agent.PrivacyLevel.PUBLIC
|
645
|
+
agent.managed_by_admin = True
|
646
|
+
agent.input_tools = []
|
647
|
+
agent.output_modes = []
|
607
648
|
agent.save()
|
608
649
|
else:
|
609
650
|
# The default agent is public and managed by the admin. It's handled a little differently than other agents.
|
610
651
|
agent = Agent.objects.create(
|
611
652
|
name=AgentAdapters.DEFAULT_AGENT_NAME,
|
612
|
-
|
653
|
+
privacy_level=Agent.PrivacyLevel.PUBLIC,
|
613
654
|
managed_by_admin=True,
|
614
655
|
chat_model=default_conversation_config,
|
615
656
|
personality=default_personality,
|
616
|
-
tools=["*"],
|
617
|
-
avatar=AgentAdapters.DEFAULT_AGENT_AVATAR,
|
618
657
|
slug=AgentAdapters.DEFAULT_AGENT_SLUG,
|
619
658
|
)
|
620
659
|
Conversation.objects.filter(agent=None).update(agent=agent)
|
@@ -625,6 +664,68 @@ class AgentAdapters:
|
|
625
664
|
async def aget_default_agent():
|
626
665
|
return await Agent.objects.filter(name=AgentAdapters.DEFAULT_AGENT_NAME).afirst()
|
627
666
|
|
667
|
+
@staticmethod
|
668
|
+
async def aupdate_agent(
|
669
|
+
user: KhojUser,
|
670
|
+
name: str,
|
671
|
+
personality: str,
|
672
|
+
privacy_level: str,
|
673
|
+
icon: str,
|
674
|
+
color: str,
|
675
|
+
chat_model: str,
|
676
|
+
files: List[str],
|
677
|
+
input_tools: List[str],
|
678
|
+
output_modes: List[str],
|
679
|
+
):
|
680
|
+
chat_model_option = await ChatModelOptions.objects.filter(chat_model=chat_model).afirst()
|
681
|
+
|
682
|
+
agent, created = await Agent.objects.filter(name=name, creator=user).aupdate_or_create(
|
683
|
+
defaults={
|
684
|
+
"name": name,
|
685
|
+
"creator": user,
|
686
|
+
"personality": personality,
|
687
|
+
"privacy_level": privacy_level,
|
688
|
+
"style_icon": icon,
|
689
|
+
"style_color": color,
|
690
|
+
"chat_model": chat_model_option,
|
691
|
+
"input_tools": input_tools,
|
692
|
+
"output_modes": output_modes,
|
693
|
+
}
|
694
|
+
)
|
695
|
+
|
696
|
+
# Delete all existing files and entries
|
697
|
+
await FileObject.objects.filter(agent=agent).adelete()
|
698
|
+
await Entry.objects.filter(agent=agent).adelete()
|
699
|
+
|
700
|
+
for file in files:
|
701
|
+
reference_file = await FileObject.objects.filter(file_name=file, user=agent.creator).afirst()
|
702
|
+
if reference_file:
|
703
|
+
await FileObject.objects.acreate(file_name=file, agent=agent, raw_text=reference_file.raw_text)
|
704
|
+
|
705
|
+
# Duplicate all entries associated with the file
|
706
|
+
entries: List[Entry] = []
|
707
|
+
async for entry in Entry.objects.filter(file_path=file, user=agent.creator).aiterator():
|
708
|
+
entries.append(
|
709
|
+
Entry(
|
710
|
+
agent=agent,
|
711
|
+
embeddings=entry.embeddings,
|
712
|
+
raw=entry.raw,
|
713
|
+
compiled=entry.compiled,
|
714
|
+
heading=entry.heading,
|
715
|
+
file_source=entry.file_source,
|
716
|
+
file_type=entry.file_type,
|
717
|
+
file_path=entry.file_path,
|
718
|
+
file_name=entry.file_name,
|
719
|
+
url=entry.url,
|
720
|
+
hashed_value=entry.hashed_value,
|
721
|
+
)
|
722
|
+
)
|
723
|
+
|
724
|
+
# Bulk create entries
|
725
|
+
await Entry.objects.abulk_create(entries)
|
726
|
+
|
727
|
+
return agent
|
728
|
+
|
628
729
|
|
629
730
|
class PublicConversationAdapters:
|
630
731
|
@staticmethod
|
@@ -1113,8 +1214,8 @@ class FileObjectAdapters:
|
|
1113
1214
|
return await FileObject.objects.acreate(user=user, file_name=file_name, raw_text=raw_text)
|
1114
1215
|
|
1115
1216
|
@staticmethod
|
1116
|
-
async def async_get_file_objects_by_name(user: KhojUser, file_name: str):
|
1117
|
-
return await sync_to_async(list)(FileObject.objects.filter(user=user, file_name=file_name))
|
1217
|
+
async def async_get_file_objects_by_name(user: KhojUser, file_name: str, agent: Agent = None):
|
1218
|
+
return await sync_to_async(list)(FileObject.objects.filter(user=user, file_name=file_name, agent=agent))
|
1118
1219
|
|
1119
1220
|
@staticmethod
|
1120
1221
|
async def async_get_all_file_objects(user: KhojUser):
|
@@ -1196,10 +1297,18 @@ class EntryAdapters:
|
|
1196
1297
|
def user_has_entries(user: KhojUser):
|
1197
1298
|
return Entry.objects.filter(user=user).exists()
|
1198
1299
|
|
1300
|
+
@staticmethod
|
1301
|
+
def agent_has_entries(agent: Agent):
|
1302
|
+
return Entry.objects.filter(agent=agent).exists()
|
1303
|
+
|
1199
1304
|
@staticmethod
|
1200
1305
|
async def auser_has_entries(user: KhojUser):
|
1201
1306
|
return await Entry.objects.filter(user=user).aexists()
|
1202
1307
|
|
1308
|
+
@staticmethod
|
1309
|
+
async def aagent_has_entries(agent: Agent):
|
1310
|
+
return await Entry.objects.filter(agent=agent).aexists()
|
1311
|
+
|
1203
1312
|
@staticmethod
|
1204
1313
|
async def adelete_entry_by_file(user: KhojUser, file_path: str):
|
1205
1314
|
return await Entry.objects.filter(user=user, file_path=file_path).adelete()
|
@@ -1214,6 +1323,10 @@ class EntryAdapters:
|
|
1214
1323
|
|
1215
1324
|
return deleted_count
|
1216
1325
|
|
1326
|
+
@staticmethod
|
1327
|
+
async def aget_agent_entry_filepaths(agent: Agent):
|
1328
|
+
return await sync_to_async(list)(Entry.objects.filter(agent=agent).values_list("file_path", flat=True))
|
1329
|
+
|
1217
1330
|
@staticmethod
|
1218
1331
|
def get_all_filenames_by_source(user: KhojUser, file_source: str):
|
1219
1332
|
return (
|
@@ -1229,15 +1342,19 @@ class EntryAdapters:
|
|
1229
1342
|
return total_size / 1024 / 1024
|
1230
1343
|
|
1231
1344
|
@staticmethod
|
1232
|
-
def apply_filters(user: KhojUser, query: str, file_type_filter: str = None):
|
1345
|
+
def apply_filters(user: KhojUser, query: str, file_type_filter: str = None, agent: Agent = None):
|
1233
1346
|
q_filter_terms = Q()
|
1234
1347
|
|
1235
1348
|
word_filters = EntryAdapters.word_filter.get_filter_terms(query)
|
1236
1349
|
file_filters = EntryAdapters.file_filter.get_filter_terms(query)
|
1237
1350
|
date_filters = EntryAdapters.date_filter.get_query_date_range(query)
|
1238
1351
|
|
1352
|
+
user_or_agent = Q(user=user)
|
1353
|
+
if agent != None:
|
1354
|
+
user_or_agent |= Q(agent=agent)
|
1355
|
+
|
1239
1356
|
if len(word_filters) == 0 and len(file_filters) == 0 and len(date_filters) == 0:
|
1240
|
-
return Entry.objects.filter(
|
1357
|
+
return Entry.objects.filter(user_or_agent)
|
1241
1358
|
|
1242
1359
|
for term in word_filters:
|
1243
1360
|
if term.startswith("+"):
|
@@ -1273,7 +1390,7 @@ class EntryAdapters:
|
|
1273
1390
|
formatted_max_date = date.fromtimestamp(max_date).strftime("%Y-%m-%d")
|
1274
1391
|
q_filter_terms &= Q(embeddings_dates__date__lte=formatted_max_date)
|
1275
1392
|
|
1276
|
-
relevant_entries = Entry.objects.filter(
|
1393
|
+
relevant_entries = Entry.objects.filter(user_or_agent).filter(q_filter_terms)
|
1277
1394
|
if file_type_filter:
|
1278
1395
|
relevant_entries = relevant_entries.filter(file_type=file_type_filter)
|
1279
1396
|
return relevant_entries
|
@@ -1286,9 +1403,15 @@ class EntryAdapters:
|
|
1286
1403
|
file_type_filter: str = None,
|
1287
1404
|
raw_query: str = None,
|
1288
1405
|
max_distance: float = math.inf,
|
1406
|
+
agent: Agent = None,
|
1289
1407
|
):
|
1290
|
-
|
1291
|
-
|
1408
|
+
user_or_agent = Q(user=user)
|
1409
|
+
|
1410
|
+
if agent != None:
|
1411
|
+
user_or_agent |= Q(agent=agent)
|
1412
|
+
|
1413
|
+
relevant_entries = EntryAdapters.apply_filters(user, raw_query, file_type_filter, agent)
|
1414
|
+
relevant_entries = relevant_entries.filter(user_or_agent).annotate(
|
1292
1415
|
distance=CosineDistance("embeddings", embeddings)
|
1293
1416
|
)
|
1294
1417
|
relevant_entries = relevant_entries.filter(distance__lte=max_distance)
|
khoj/database/admin.py
CHANGED
@@ -27,6 +27,7 @@ from khoj.database.models import (
|
|
27
27
|
Subscription,
|
28
28
|
TextToImageModelConfig,
|
29
29
|
UserConversationConfig,
|
30
|
+
UserRequests,
|
30
31
|
UserSearchModelConfig,
|
31
32
|
UserVoiceModelConfig,
|
32
33
|
VoiceModelOption,
|
@@ -103,6 +104,7 @@ admin.site.register(NotionConfig)
|
|
103
104
|
admin.site.register(UserVoiceModelConfig)
|
104
105
|
admin.site.register(VoiceModelOption)
|
105
106
|
admin.site.register(UserConversationConfig)
|
107
|
+
admin.site.register(UserRequests)
|
106
108
|
|
107
109
|
|
108
110
|
@admin.register(Agent)
|
@@ -0,0 +1,49 @@
|
|
1
|
+
# Generated by Django 5.0.8 on 2024-09-18 02:54
|
2
|
+
|
3
|
+
import django.db.models.deletion
|
4
|
+
from django.db import migrations, models
|
5
|
+
|
6
|
+
|
7
|
+
class Migration(migrations.Migration):
|
8
|
+
dependencies = [
|
9
|
+
("database", "0064_remove_conversation_temp_id_alter_conversation_id"),
|
10
|
+
]
|
11
|
+
|
12
|
+
operations = [
|
13
|
+
migrations.RemoveField(
|
14
|
+
model_name="agent",
|
15
|
+
name="avatar",
|
16
|
+
),
|
17
|
+
migrations.RemoveField(
|
18
|
+
model_name="agent",
|
19
|
+
name="public",
|
20
|
+
),
|
21
|
+
migrations.AddField(
|
22
|
+
model_name="agent",
|
23
|
+
name="privacy_level",
|
24
|
+
field=models.CharField(
|
25
|
+
choices=[("public", "Public"), ("private", "Private"), ("protected", "Protected")],
|
26
|
+
default="private",
|
27
|
+
max_length=30,
|
28
|
+
),
|
29
|
+
),
|
30
|
+
migrations.AddField(
|
31
|
+
model_name="entry",
|
32
|
+
name="agent",
|
33
|
+
field=models.ForeignKey(
|
34
|
+
blank=True, default=None, null=True, on_delete=django.db.models.deletion.CASCADE, to="database.agent"
|
35
|
+
),
|
36
|
+
),
|
37
|
+
migrations.AddField(
|
38
|
+
model_name="fileobject",
|
39
|
+
name="agent",
|
40
|
+
field=models.ForeignKey(
|
41
|
+
blank=True, default=None, null=True, on_delete=django.db.models.deletion.CASCADE, to="database.agent"
|
42
|
+
),
|
43
|
+
),
|
44
|
+
migrations.AlterField(
|
45
|
+
model_name="agent",
|
46
|
+
name="slug",
|
47
|
+
field=models.CharField(max_length=200, unique=True),
|
48
|
+
),
|
49
|
+
]
|
@@ -0,0 +1,69 @@
|
|
1
|
+
# Generated by Django 5.0.8 on 2024-10-01 00:42
|
2
|
+
|
3
|
+
import django.contrib.postgres.fields
|
4
|
+
from django.db import migrations, models
|
5
|
+
|
6
|
+
|
7
|
+
class Migration(migrations.Migration):
|
8
|
+
dependencies = [
|
9
|
+
("database", "0065_remove_agent_avatar_remove_agent_public_and_more"),
|
10
|
+
]
|
11
|
+
|
12
|
+
operations = [
|
13
|
+
migrations.RemoveField(
|
14
|
+
model_name="agent",
|
15
|
+
name="tools",
|
16
|
+
),
|
17
|
+
migrations.AddField(
|
18
|
+
model_name="agent",
|
19
|
+
name="input_tools",
|
20
|
+
field=django.contrib.postgres.fields.ArrayField(
|
21
|
+
base_field=models.CharField(
|
22
|
+
choices=[
|
23
|
+
("general", "General"),
|
24
|
+
("online", "Online"),
|
25
|
+
("notes", "Notes"),
|
26
|
+
("summarize", "Summarize"),
|
27
|
+
("webpage", "Webpage"),
|
28
|
+
],
|
29
|
+
max_length=200,
|
30
|
+
),
|
31
|
+
default=list,
|
32
|
+
size=None,
|
33
|
+
),
|
34
|
+
),
|
35
|
+
migrations.AddField(
|
36
|
+
model_name="agent",
|
37
|
+
name="output_modes",
|
38
|
+
field=django.contrib.postgres.fields.ArrayField(
|
39
|
+
base_field=models.CharField(choices=[("text", "Text"), ("image", "Image")], max_length=200),
|
40
|
+
default=list,
|
41
|
+
size=None,
|
42
|
+
),
|
43
|
+
),
|
44
|
+
migrations.AlterField(
|
45
|
+
model_name="agent",
|
46
|
+
name="style_icon",
|
47
|
+
field=models.CharField(
|
48
|
+
choices=[
|
49
|
+
("Lightbulb", "Lightbulb"),
|
50
|
+
("Health", "Health"),
|
51
|
+
("Robot", "Robot"),
|
52
|
+
("Aperture", "Aperture"),
|
53
|
+
("GraduationCap", "Graduation Cap"),
|
54
|
+
("Jeep", "Jeep"),
|
55
|
+
("Island", "Island"),
|
56
|
+
("MathOperations", "Math Operations"),
|
57
|
+
("Asclepius", "Asclepius"),
|
58
|
+
("Couch", "Couch"),
|
59
|
+
("Code", "Code"),
|
60
|
+
("Atom", "Atom"),
|
61
|
+
("ClockCounterClockwise", "Clock Counter Clockwise"),
|
62
|
+
("PencilLine", "Pencil Line"),
|
63
|
+
("Chalkboard", "Chalkboard"),
|
64
|
+
],
|
65
|
+
default="Lightbulb",
|
66
|
+
max_length=200,
|
67
|
+
),
|
68
|
+
),
|
69
|
+
]
|
@@ -0,0 +1,50 @@
|
|
1
|
+
# Generated by Django 5.0.8 on 2024-10-01 18:42
|
2
|
+
|
3
|
+
from django.db import migrations, models
|
4
|
+
|
5
|
+
|
6
|
+
class Migration(migrations.Migration):
|
7
|
+
dependencies = [
|
8
|
+
("database", "0066_remove_agent_tools_agent_input_tools_and_more"),
|
9
|
+
]
|
10
|
+
|
11
|
+
operations = [
|
12
|
+
migrations.AlterField(
|
13
|
+
model_name="agent",
|
14
|
+
name="style_icon",
|
15
|
+
field=models.CharField(
|
16
|
+
choices=[
|
17
|
+
("Lightbulb", "Lightbulb"),
|
18
|
+
("Health", "Health"),
|
19
|
+
("Robot", "Robot"),
|
20
|
+
("Aperture", "Aperture"),
|
21
|
+
("GraduationCap", "Graduation Cap"),
|
22
|
+
("Jeep", "Jeep"),
|
23
|
+
("Island", "Island"),
|
24
|
+
("MathOperations", "Math Operations"),
|
25
|
+
("Asclepius", "Asclepius"),
|
26
|
+
("Couch", "Couch"),
|
27
|
+
("Code", "Code"),
|
28
|
+
("Atom", "Atom"),
|
29
|
+
("ClockCounterClockwise", "Clock Counter Clockwise"),
|
30
|
+
("PencilLine", "Pencil Line"),
|
31
|
+
("Chalkboard", "Chalkboard"),
|
32
|
+
("Cigarette", "Cigarette"),
|
33
|
+
("CraneTower", "Crane Tower"),
|
34
|
+
("Heart", "Heart"),
|
35
|
+
("Leaf", "Leaf"),
|
36
|
+
("NewspaperClipping", "Newspaper Clipping"),
|
37
|
+
("OrangeSlice", "Orange Slice"),
|
38
|
+
("SmileyMelting", "Smiley Melting"),
|
39
|
+
("YinYang", "Yin Yang"),
|
40
|
+
("SneakerMove", "Sneaker Move"),
|
41
|
+
("Student", "Student"),
|
42
|
+
("Oven", "Oven"),
|
43
|
+
("Gavel", "Gavel"),
|
44
|
+
("Broadcast", "Broadcast"),
|
45
|
+
],
|
46
|
+
default="Lightbulb",
|
47
|
+
max_length=200,
|
48
|
+
),
|
49
|
+
),
|
50
|
+
]
|
khoj/database/models/__init__.py
CHANGED
@@ -3,6 +3,7 @@ import uuid
|
|
3
3
|
from random import choice
|
4
4
|
|
5
5
|
from django.contrib.auth.models import AbstractUser
|
6
|
+
from django.contrib.postgres.fields import ArrayField
|
6
7
|
from django.core.exceptions import ValidationError
|
7
8
|
from django.db import models
|
8
9
|
from django.db.models.signals import pre_save
|
@@ -10,6 +11,8 @@ from django.dispatch import receiver
|
|
10
11
|
from pgvector.django import VectorField
|
11
12
|
from phonenumber_field.modelfields import PhoneNumberField
|
12
13
|
|
14
|
+
from khoj.utils.helpers import ConversationCommand
|
15
|
+
|
13
16
|
|
14
17
|
class BaseModel(models.Model):
|
15
18
|
created_at = models.DateTimeField(auto_now_add=True)
|
@@ -125,7 +128,7 @@ class Agent(BaseModel):
|
|
125
128
|
EMERALD = "emerald"
|
126
129
|
|
127
130
|
class StyleIconTypes(models.TextChoices):
|
128
|
-
|
131
|
+
LIGHTBULB = "Lightbulb"
|
129
132
|
HEALTH = "Health"
|
130
133
|
ROBOT = "Robot"
|
131
134
|
APERTURE = "Aperture"
|
@@ -140,20 +143,64 @@ class Agent(BaseModel):
|
|
140
143
|
CLOCK_COUNTER_CLOCKWISE = "ClockCounterClockwise"
|
141
144
|
PENCIL_LINE = "PencilLine"
|
142
145
|
CHALKBOARD = "Chalkboard"
|
146
|
+
CIGARETTE = "Cigarette"
|
147
|
+
CRANE_TOWER = "CraneTower"
|
148
|
+
HEART = "Heart"
|
149
|
+
LEAF = "Leaf"
|
150
|
+
NEWSPAPER_CLIPPING = "NewspaperClipping"
|
151
|
+
ORANGE_SLICE = "OrangeSlice"
|
152
|
+
SMILEY_MELTING = "SmileyMelting"
|
153
|
+
YIN_YANG = "YinYang"
|
154
|
+
SNEAKER_MOVE = "SneakerMove"
|
155
|
+
STUDENT = "Student"
|
156
|
+
OVEN = "Oven"
|
157
|
+
GAVEL = "Gavel"
|
158
|
+
BROADCAST = "Broadcast"
|
159
|
+
|
160
|
+
class PrivacyLevel(models.TextChoices):
|
161
|
+
PUBLIC = "public"
|
162
|
+
PRIVATE = "private"
|
163
|
+
PROTECTED = "protected"
|
164
|
+
|
165
|
+
class InputToolOptions(models.TextChoices):
|
166
|
+
# These map to various ConversationCommand types
|
167
|
+
GENERAL = "general"
|
168
|
+
ONLINE = "online"
|
169
|
+
NOTES = "notes"
|
170
|
+
SUMMARIZE = "summarize"
|
171
|
+
WEBPAGE = "webpage"
|
172
|
+
|
173
|
+
class OutputModeOptions(models.TextChoices):
|
174
|
+
# These map to various ConversationCommand types
|
175
|
+
TEXT = "text"
|
176
|
+
IMAGE = "image"
|
143
177
|
|
144
178
|
creator = models.ForeignKey(
|
145
179
|
KhojUser, on_delete=models.CASCADE, default=None, null=True, blank=True
|
146
180
|
) # Creator will only be null when the agents are managed by admin
|
147
181
|
name = models.CharField(max_length=200)
|
148
182
|
personality = models.TextField()
|
149
|
-
|
150
|
-
|
151
|
-
public = models.BooleanField(default=False)
|
183
|
+
input_tools = ArrayField(models.CharField(max_length=200, choices=InputToolOptions.choices), default=list)
|
184
|
+
output_modes = ArrayField(models.CharField(max_length=200, choices=OutputModeOptions.choices), default=list)
|
152
185
|
managed_by_admin = models.BooleanField(default=False)
|
153
186
|
chat_model = models.ForeignKey(ChatModelOptions, on_delete=models.CASCADE)
|
154
|
-
slug = models.CharField(max_length=200)
|
187
|
+
slug = models.CharField(max_length=200, unique=True)
|
155
188
|
style_color = models.CharField(max_length=200, choices=StyleColorTypes.choices, default=StyleColorTypes.BLUE)
|
156
|
-
style_icon = models.CharField(max_length=200, choices=StyleIconTypes.choices, default=StyleIconTypes.
|
189
|
+
style_icon = models.CharField(max_length=200, choices=StyleIconTypes.choices, default=StyleIconTypes.LIGHTBULB)
|
190
|
+
privacy_level = models.CharField(max_length=30, choices=PrivacyLevel.choices, default=PrivacyLevel.PRIVATE)
|
191
|
+
|
192
|
+
def save(self, *args, **kwargs):
|
193
|
+
is_new = self._state.adding
|
194
|
+
|
195
|
+
if self.creator is None:
|
196
|
+
self.managed_by_admin = True
|
197
|
+
|
198
|
+
if is_new:
|
199
|
+
random_sequence = "".join(choice("0123456789") for i in range(6))
|
200
|
+
slug = f"{self.name.lower().replace(' ', '-')}-{random_sequence}"
|
201
|
+
self.slug = slug
|
202
|
+
|
203
|
+
super().save(*args, **kwargs)
|
157
204
|
|
158
205
|
|
159
206
|
class ProcessLock(BaseModel):
|
@@ -173,22 +220,11 @@ class ProcessLock(BaseModel):
|
|
173
220
|
def verify_agent(sender, instance, **kwargs):
|
174
221
|
# check if this is a new instance
|
175
222
|
if instance._state.adding:
|
176
|
-
if Agent.objects.filter(name=instance.name,
|
223
|
+
if Agent.objects.filter(name=instance.name, privacy_level=Agent.PrivacyLevel.PUBLIC).exists():
|
177
224
|
raise ValidationError(f"A public Agent with the name {instance.name} already exists.")
|
178
225
|
if Agent.objects.filter(name=instance.name, creator=instance.creator).exists():
|
179
226
|
raise ValidationError(f"A private Agent with the name {instance.name} already exists.")
|
180
227
|
|
181
|
-
slug = instance.name.lower().replace(" ", "-")
|
182
|
-
observed_random_numbers = set()
|
183
|
-
while Agent.objects.filter(slug=slug).exists():
|
184
|
-
try:
|
185
|
-
random_number = choice([i for i in range(0, 1000) if i not in observed_random_numbers])
|
186
|
-
except IndexError:
|
187
|
-
raise ValidationError("Unable to generate a unique slug for the Agent. Please try again later.")
|
188
|
-
observed_random_numbers.add(random_number)
|
189
|
-
slug = f"{slug}-{random_number}"
|
190
|
-
instance.slug = slug
|
191
|
-
|
192
228
|
|
193
229
|
class NotionConfig(BaseModel):
|
194
230
|
token = models.CharField(max_length=200)
|
@@ -406,6 +442,7 @@ class Entry(BaseModel):
|
|
406
442
|
GITHUB = "github"
|
407
443
|
|
408
444
|
user = models.ForeignKey(KhojUser, on_delete=models.CASCADE, default=None, null=True, blank=True)
|
445
|
+
agent = models.ForeignKey(Agent, on_delete=models.CASCADE, default=None, null=True, blank=True)
|
409
446
|
embeddings = VectorField(dimensions=None)
|
410
447
|
raw = models.TextField()
|
411
448
|
compiled = models.TextField()
|
@@ -418,12 +455,17 @@ class Entry(BaseModel):
|
|
418
455
|
hashed_value = models.CharField(max_length=100)
|
419
456
|
corpus_id = models.UUIDField(default=uuid.uuid4, editable=False)
|
420
457
|
|
458
|
+
def save(self, *args, **kwargs):
|
459
|
+
if self.user and self.agent:
|
460
|
+
raise ValidationError("An Entry cannot be associated with both a user and an agent.")
|
461
|
+
|
421
462
|
|
422
463
|
class FileObject(BaseModel):
|
423
464
|
# Same as Entry but raw will be a much larger string
|
424
465
|
file_name = models.CharField(max_length=400, default=None, null=True, blank=True)
|
425
466
|
raw_text = models.TextField()
|
426
467
|
user = models.ForeignKey(KhojUser, on_delete=models.CASCADE, default=None, null=True, blank=True)
|
468
|
+
agent = models.ForeignKey(Agent, on_delete=models.CASCADE, default=None, null=True, blank=True)
|
427
469
|
|
428
470
|
|
429
471
|
class EntryDates(BaseModel):
|