khoj 1.28.4.dev23__py3-none-any.whl → 1.28.4.dev77__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- khoj/configure.py +4 -6
- khoj/database/adapters/__init__.py +124 -34
- khoj/database/models/__init__.py +4 -0
- khoj/interface/compiled/404/index.html +1 -1
- khoj/interface/compiled/_next/static/chunks/1603-2418b11d8e8dacb9.js +1 -0
- khoj/interface/compiled/_next/static/chunks/1970-c78f6acc8e16e30b.js +1 -0
- khoj/interface/compiled/_next/static/chunks/3124-a4cea2eda163128d.js +1 -0
- khoj/interface/compiled/_next/static/chunks/5538-5c4f2271e9377b74.js +1 -0
- khoj/interface/compiled/_next/static/chunks/8423-db6dad6d44869097.js +1 -0
- khoj/interface/compiled/_next/static/chunks/9417-7a8a6da918d37750.js +1 -0
- khoj/interface/compiled/_next/static/chunks/app/agents/{page-36da67f03a173e52.js → page-4353b1a532795ad1.js} +1 -1
- khoj/interface/compiled/_next/static/chunks/app/automations/{page-774ae3e033f938cd.js → page-c9f13c865e739607.js} +1 -1
- khoj/interface/compiled/_next/static/chunks/app/chat/page-97876b3bd3c5e69d.js +1 -0
- khoj/interface/compiled/_next/static/chunks/app/{page-322c37514a3a613a.js → page-c33ebe19a3b7b0b2.js} +1 -1
- khoj/interface/compiled/_next/static/chunks/app/search/{page-9b64f61caa5bd7f9.js → page-8e28deacb61f75aa.js} +1 -1
- khoj/interface/compiled/_next/static/chunks/app/settings/page-2fab613a557d3cc5.js +1 -0
- khoj/interface/compiled/_next/static/chunks/app/share/chat/page-3ee3da7e8dfe3572.js +1 -0
- khoj/interface/compiled/_next/static/chunks/{webpack-c9799fdebf88abb6.js → webpack-ff5eae43b8dba1d2.js} +1 -1
- khoj/interface/compiled/_next/static/css/23f801d22927d568.css +1 -0
- khoj/interface/compiled/_next/static/css/592ca99f5122e75a.css +1 -0
- khoj/interface/compiled/_next/static/css/af0f36f71f368260.css +25 -0
- khoj/interface/compiled/agents/index.html +1 -1
- khoj/interface/compiled/agents/index.txt +2 -2
- khoj/interface/compiled/automations/index.html +1 -1
- khoj/interface/compiled/automations/index.txt +2 -2
- khoj/interface/compiled/chat/index.html +1 -1
- khoj/interface/compiled/chat/index.txt +2 -2
- khoj/interface/compiled/index.html +1 -1
- khoj/interface/compiled/index.txt +3 -3
- khoj/interface/compiled/search/index.html +1 -1
- khoj/interface/compiled/search/index.txt +2 -2
- khoj/interface/compiled/settings/index.html +1 -1
- khoj/interface/compiled/settings/index.txt +2 -2
- khoj/interface/compiled/share/chat/index.html +1 -1
- khoj/interface/compiled/share/chat/index.txt +3 -3
- khoj/processor/content/docx/docx_to_entries.py +27 -21
- khoj/processor/content/github/github_to_entries.py +2 -2
- khoj/processor/content/images/image_to_entries.py +2 -2
- khoj/processor/content/markdown/markdown_to_entries.py +2 -2
- khoj/processor/content/notion/notion_to_entries.py +2 -2
- khoj/processor/content/org_mode/org_to_entries.py +2 -2
- khoj/processor/content/pdf/pdf_to_entries.py +37 -29
- khoj/processor/content/plaintext/plaintext_to_entries.py +2 -2
- khoj/processor/content/text_to_entries.py +2 -2
- khoj/processor/conversation/anthropic/anthropic_chat.py +7 -1
- khoj/processor/conversation/google/gemini_chat.py +15 -2
- khoj/processor/conversation/offline/chat_model.py +4 -0
- khoj/processor/conversation/openai/gpt.py +6 -1
- khoj/processor/conversation/prompts.py +48 -4
- khoj/processor/conversation/utils.py +69 -11
- khoj/processor/image/generate.py +2 -0
- khoj/processor/tools/online_search.py +19 -3
- khoj/processor/tools/run_code.py +4 -0
- khoj/routers/api.py +6 -1
- khoj/routers/api_agents.py +8 -10
- khoj/routers/api_chat.py +64 -13
- khoj/routers/api_content.py +80 -8
- khoj/routers/helpers.py +105 -34
- khoj/routers/notion.py +1 -1
- khoj/routers/research.py +9 -2
- khoj/search_type/text_search.py +1 -1
- khoj/utils/fs_syncer.py +2 -1
- khoj/utils/rawconfig.py +32 -0
- {khoj-1.28.4.dev23.dist-info → khoj-1.28.4.dev77.dist-info}/METADATA +1 -1
- {khoj-1.28.4.dev23.dist-info → khoj-1.28.4.dev77.dist-info}/RECORD +70 -70
- khoj/interface/compiled/_next/static/chunks/1603-c1568f45947e9f2c.js +0 -1
- khoj/interface/compiled/_next/static/chunks/1970-d44050bf658ae5cc.js +0 -1
- khoj/interface/compiled/_next/static/chunks/5538-bf582517a8dd3faa.js +0 -1
- khoj/interface/compiled/_next/static/chunks/8423-a1f432e4a8d9a6b0.js +0 -1
- khoj/interface/compiled/_next/static/chunks/8840-b8d7b9f0923c6651.js +0 -1
- khoj/interface/compiled/_next/static/chunks/9417-0d0fc7eb49a86abb.js +0 -1
- khoj/interface/compiled/_next/static/chunks/app/chat/page-a369e2bda9897794.js +0 -1
- khoj/interface/compiled/_next/static/chunks/app/settings/page-10b288c103f19468.js +0 -1
- khoj/interface/compiled/_next/static/chunks/app/share/chat/page-959d5f097cf38c93.js +0 -1
- khoj/interface/compiled/_next/static/css/4cae6c0e5c72fb2d.css +0 -1
- khoj/interface/compiled/_next/static/css/9d45de78fba367c1.css +0 -1
- khoj/interface/compiled/_next/static/css/d2bc549245313f26.css +0 -25
- /khoj/interface/compiled/_next/static/{s_mKS5kELaw2v4a7_yWNP → sE94pAZEifEKkz4WQtTNW}/_buildManifest.js +0 -0
- /khoj/interface/compiled/_next/static/{s_mKS5kELaw2v4a7_yWNP → sE94pAZEifEKkz4WQtTNW}/_ssgManifest.js +0 -0
- {khoj-1.28.4.dev23.dist-info → khoj-1.28.4.dev77.dist-info}/WHEEL +0 -0
- {khoj-1.28.4.dev23.dist-info → khoj-1.28.4.dev77.dist-info}/entry_points.txt +0 -0
- {khoj-1.28.4.dev23.dist-info → khoj-1.28.4.dev77.dist-info}/licenses/LICENSE +0 -0
khoj/configure.py
CHANGED
@@ -253,7 +253,7 @@ def configure_server(
|
|
253
253
|
logger.info(message)
|
254
254
|
|
255
255
|
if not init:
|
256
|
-
initialize_content(regenerate, search_type
|
256
|
+
initialize_content(user, regenerate, search_type)
|
257
257
|
|
258
258
|
except Exception as e:
|
259
259
|
logger.error(f"Failed to load some search models: {e}", exc_info=True)
|
@@ -263,17 +263,17 @@ def setup_default_agent(user: KhojUser):
|
|
263
263
|
AgentAdapters.create_default_agent(user)
|
264
264
|
|
265
265
|
|
266
|
-
def initialize_content(regenerate: bool, search_type: Optional[SearchType] = None
|
266
|
+
def initialize_content(user: KhojUser, regenerate: bool, search_type: Optional[SearchType] = None):
|
267
267
|
# Initialize Content from Config
|
268
268
|
if state.search_models:
|
269
269
|
try:
|
270
270
|
logger.info("📬 Updating content index...")
|
271
271
|
all_files = collect_files(user=user)
|
272
272
|
status = configure_content(
|
273
|
+
user,
|
273
274
|
all_files,
|
274
275
|
regenerate,
|
275
276
|
search_type,
|
276
|
-
user=user,
|
277
277
|
)
|
278
278
|
if not status:
|
279
279
|
raise RuntimeError("Failed to update content index")
|
@@ -338,9 +338,7 @@ def configure_middleware(app):
|
|
338
338
|
def update_content_index():
|
339
339
|
for user in get_all_users():
|
340
340
|
all_files = collect_files(user=user)
|
341
|
-
success = configure_content(
|
342
|
-
all_files = collect_files(user=None)
|
343
|
-
success = configure_content(all_files, user=None)
|
341
|
+
success = configure_content(user, all_files)
|
344
342
|
if not success:
|
345
343
|
raise RuntimeError("Failed to update content index")
|
346
344
|
logger.info("📪 Content index updated via Scheduler")
|
@@ -8,13 +8,22 @@ import secrets
|
|
8
8
|
import sys
|
9
9
|
from datetime import date, datetime, timedelta, timezone
|
10
10
|
from enum import Enum
|
11
|
-
from
|
11
|
+
from functools import wraps
|
12
|
+
from typing import (
|
13
|
+
Any,
|
14
|
+
Callable,
|
15
|
+
Coroutine,
|
16
|
+
Iterable,
|
17
|
+
List,
|
18
|
+
Optional,
|
19
|
+
ParamSpec,
|
20
|
+
TypeVar,
|
21
|
+
)
|
12
22
|
|
13
23
|
import cron_descriptor
|
14
24
|
from apscheduler.job import Job
|
15
25
|
from asgiref.sync import sync_to_async
|
16
26
|
from django.contrib.sessions.backends.db import SessionStore
|
17
|
-
from django.db import models
|
18
27
|
from django.db.models import Prefetch, Q
|
19
28
|
from django.db.models.manager import BaseManager
|
20
29
|
from django.db.utils import IntegrityError
|
@@ -28,7 +37,6 @@ from khoj.database.models import (
|
|
28
37
|
ChatModelOptions,
|
29
38
|
ClientApplication,
|
30
39
|
Conversation,
|
31
|
-
DataStore,
|
32
40
|
Entry,
|
33
41
|
FileObject,
|
34
42
|
GithubConfig,
|
@@ -80,6 +88,45 @@ class SubscriptionState(Enum):
|
|
80
88
|
INVALID = "invalid"
|
81
89
|
|
82
90
|
|
91
|
+
P = ParamSpec("P")
|
92
|
+
T = TypeVar("T")
|
93
|
+
|
94
|
+
|
95
|
+
def require_valid_user(func: Callable[P, T]) -> Callable[P, T]:
|
96
|
+
@wraps(func)
|
97
|
+
def sync_wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
|
98
|
+
# Extract user from args/kwargs
|
99
|
+
user = next((arg for arg in args if isinstance(arg, KhojUser)), None)
|
100
|
+
if not user:
|
101
|
+
user = next((val for val in kwargs.values() if isinstance(val, KhojUser)), None)
|
102
|
+
|
103
|
+
# Throw error if user is not found
|
104
|
+
if not user:
|
105
|
+
raise ValueError("Khoj user argument required but not provided.")
|
106
|
+
|
107
|
+
return func(*args, **kwargs)
|
108
|
+
|
109
|
+
return sync_wrapper
|
110
|
+
|
111
|
+
|
112
|
+
def arequire_valid_user(func: Callable[P, Coroutine[Any, Any, T]]) -> Callable[P, Coroutine[Any, Any, T]]:
|
113
|
+
@wraps(func)
|
114
|
+
async def async_wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
|
115
|
+
# Extract user from args/kwargs
|
116
|
+
user = next((arg for arg in args if isinstance(arg, KhojUser)), None)
|
117
|
+
if not user:
|
118
|
+
user = next((v for v in kwargs.values() if isinstance(v, KhojUser)), None)
|
119
|
+
|
120
|
+
# Throw error if user is not found
|
121
|
+
if not user:
|
122
|
+
raise ValueError("Khoj user argument required but not provided.")
|
123
|
+
|
124
|
+
return await func(*args, **kwargs)
|
125
|
+
|
126
|
+
return async_wrapper
|
127
|
+
|
128
|
+
|
129
|
+
@arequire_valid_user
|
83
130
|
async def set_notion_config(token: str, user: KhojUser):
|
84
131
|
notion_config = await NotionConfig.objects.filter(user=user).afirst()
|
85
132
|
if not notion_config:
|
@@ -90,6 +137,7 @@ async def set_notion_config(token: str, user: KhojUser):
|
|
90
137
|
return notion_config
|
91
138
|
|
92
139
|
|
140
|
+
@require_valid_user
|
93
141
|
def create_khoj_token(user: KhojUser, name=None):
|
94
142
|
"Create Khoj API key for user"
|
95
143
|
token = f"kk-{secrets.token_urlsafe(32)}"
|
@@ -97,6 +145,7 @@ def create_khoj_token(user: KhojUser, name=None):
|
|
97
145
|
return KhojApiUser.objects.create(token=token, user=user, name=name)
|
98
146
|
|
99
147
|
|
148
|
+
@arequire_valid_user
|
100
149
|
async def acreate_khoj_token(user: KhojUser, name=None):
|
101
150
|
"Create Khoj API key for user"
|
102
151
|
token = f"kk-{secrets.token_urlsafe(32)}"
|
@@ -104,11 +153,13 @@ async def acreate_khoj_token(user: KhojUser, name=None):
|
|
104
153
|
return await KhojApiUser.objects.acreate(token=token, user=user, name=name)
|
105
154
|
|
106
155
|
|
156
|
+
@require_valid_user
|
107
157
|
def get_khoj_tokens(user: KhojUser):
|
108
158
|
"Get all Khoj API keys for user"
|
109
159
|
return list(KhojApiUser.objects.filter(user=user))
|
110
160
|
|
111
161
|
|
162
|
+
@arequire_valid_user
|
112
163
|
async def delete_khoj_token(user: KhojUser, token: str):
|
113
164
|
"Delete Khoj API Key for user"
|
114
165
|
await KhojApiUser.objects.filter(token=token, user=user).adelete()
|
@@ -132,6 +183,7 @@ async def aget_or_create_user_by_phone_number(phone_number: str) -> tuple[KhojUs
|
|
132
183
|
return user, is_new
|
133
184
|
|
134
185
|
|
186
|
+
@arequire_valid_user
|
135
187
|
async def aset_user_phone_number(user: KhojUser, phone_number: str) -> KhojUser:
|
136
188
|
if is_none_or_empty(phone_number):
|
137
189
|
return None
|
@@ -155,6 +207,7 @@ async def aset_user_phone_number(user: KhojUser, phone_number: str) -> KhojUser:
|
|
155
207
|
return user
|
156
208
|
|
157
209
|
|
210
|
+
@arequire_valid_user
|
158
211
|
async def aremove_phone_number(user: KhojUser) -> KhojUser:
|
159
212
|
user.phone_number = None
|
160
213
|
user.verified_phone_number = False
|
@@ -192,6 +245,7 @@ async def aget_or_create_user_by_email(email: str) -> tuple[KhojUser, bool]:
|
|
192
245
|
return user, is_new
|
193
246
|
|
194
247
|
|
248
|
+
@arequire_valid_user
|
195
249
|
async def astart_trial_subscription(user: KhojUser) -> Subscription:
|
196
250
|
subscription = await Subscription.objects.filter(user=user).afirst()
|
197
251
|
if not subscription:
|
@@ -246,6 +300,7 @@ async def create_user_by_google_token(token: dict) -> KhojUser:
|
|
246
300
|
return user
|
247
301
|
|
248
302
|
|
303
|
+
@require_valid_user
|
249
304
|
def set_user_name(user: KhojUser, first_name: str, last_name: str) -> KhojUser:
|
250
305
|
user.first_name = first_name
|
251
306
|
user.last_name = last_name
|
@@ -253,6 +308,7 @@ def set_user_name(user: KhojUser, first_name: str, last_name: str) -> KhojUser:
|
|
253
308
|
return user
|
254
309
|
|
255
310
|
|
311
|
+
@require_valid_user
|
256
312
|
def get_user_name(user: KhojUser):
|
257
313
|
full_name = user.get_full_name()
|
258
314
|
if not is_none_or_empty(full_name):
|
@@ -264,6 +320,7 @@ def get_user_name(user: KhojUser):
|
|
264
320
|
return None
|
265
321
|
|
266
322
|
|
323
|
+
@require_valid_user
|
267
324
|
def get_user_photo(user: KhojUser):
|
268
325
|
google_profile: GoogleUser = GoogleUser.objects.filter(user=user).first()
|
269
326
|
if google_profile:
|
@@ -327,6 +384,7 @@ def get_user_subscription_state(email: str) -> str:
|
|
327
384
|
return subscription_to_state(user_subscription)
|
328
385
|
|
329
386
|
|
387
|
+
@arequire_valid_user
|
330
388
|
async def aget_user_subscription_state(user: KhojUser) -> str:
|
331
389
|
"""Get subscription state of user
|
332
390
|
Valid state transitions: trial -> subscribed <-> unsubscribed OR expired
|
@@ -335,6 +393,7 @@ async def aget_user_subscription_state(user: KhojUser) -> str:
|
|
335
393
|
return await sync_to_async(subscription_to_state)(user_subscription)
|
336
394
|
|
337
395
|
|
396
|
+
@arequire_valid_user
|
338
397
|
async def ais_user_subscribed(user: KhojUser) -> bool:
|
339
398
|
"""
|
340
399
|
Get whether the user is subscribed
|
@@ -351,6 +410,7 @@ async def ais_user_subscribed(user: KhojUser) -> bool:
|
|
351
410
|
return subscribed
|
352
411
|
|
353
412
|
|
413
|
+
@require_valid_user
|
354
414
|
def is_user_subscribed(user: KhojUser) -> bool:
|
355
415
|
"""
|
356
416
|
Get whether the user is subscribed
|
@@ -416,11 +476,13 @@ def get_all_users() -> BaseManager[KhojUser]:
|
|
416
476
|
return KhojUser.objects.all()
|
417
477
|
|
418
478
|
|
479
|
+
@require_valid_user
|
419
480
|
def get_user_github_config(user: KhojUser):
|
420
481
|
config = GithubConfig.objects.filter(user=user).prefetch_related("githubrepoconfig").first()
|
421
482
|
return config
|
422
483
|
|
423
484
|
|
485
|
+
@require_valid_user
|
424
486
|
def get_user_notion_config(user: KhojUser):
|
425
487
|
config = NotionConfig.objects.filter(user=user).first()
|
426
488
|
return config
|
@@ -430,6 +492,7 @@ def delete_user_requests(window: timedelta = timedelta(days=1)):
|
|
430
492
|
return UserRequests.objects.filter(created_at__lte=datetime.now(tz=timezone.utc) - window).delete()
|
431
493
|
|
432
494
|
|
495
|
+
@arequire_valid_user
|
433
496
|
async def aget_user_name(user: KhojUser):
|
434
497
|
full_name = user.get_full_name()
|
435
498
|
if not is_none_or_empty(full_name):
|
@@ -441,18 +504,7 @@ async def aget_user_name(user: KhojUser):
|
|
441
504
|
return None
|
442
505
|
|
443
506
|
|
444
|
-
|
445
|
-
deduped_files = list(set(updated_config.input_files)) if updated_config.input_files else None
|
446
|
-
deduped_filters = list(set(updated_config.input_filter)) if updated_config.input_filter else None
|
447
|
-
await object.objects.filter(user=user).adelete()
|
448
|
-
await object.objects.acreate(
|
449
|
-
input_files=deduped_files,
|
450
|
-
input_filter=deduped_filters,
|
451
|
-
index_heading_entries=updated_config.index_heading_entries,
|
452
|
-
user=user,
|
453
|
-
)
|
454
|
-
|
455
|
-
|
507
|
+
@arequire_valid_user
|
456
508
|
async def set_user_github_config(user: KhojUser, pat_token: str, repos: list):
|
457
509
|
config = await GithubConfig.objects.filter(user=user).afirst()
|
458
510
|
|
@@ -587,8 +639,11 @@ class AgentAdapters:
|
|
587
639
|
)
|
588
640
|
|
589
641
|
@staticmethod
|
642
|
+
@arequire_valid_user
|
590
643
|
async def adelete_agent_by_slug(agent_slug: str, user: KhojUser):
|
591
644
|
agent = await AgentAdapters.aget_agent_by_slug(agent_slug, user)
|
645
|
+
if agent.creator != user:
|
646
|
+
return False
|
592
647
|
|
593
648
|
async for entry in Entry.objects.filter(agent=agent).aiterator():
|
594
649
|
await entry.adelete()
|
@@ -712,6 +767,7 @@ class AgentAdapters:
|
|
712
767
|
return await Agent.objects.filter(name=AgentAdapters.DEFAULT_AGENT_NAME).afirst()
|
713
768
|
|
714
769
|
@staticmethod
|
770
|
+
@arequire_valid_user
|
715
771
|
async def aupdate_agent(
|
716
772
|
user: KhojUser,
|
717
773
|
name: str,
|
@@ -787,19 +843,6 @@ class PublicConversationAdapters:
|
|
787
843
|
return f"/share/chat/{public_conversation.slug}/"
|
788
844
|
|
789
845
|
|
790
|
-
class DataStoreAdapters:
|
791
|
-
@staticmethod
|
792
|
-
async def astore_data(data: dict, key: str, user: KhojUser, private: bool = True):
|
793
|
-
if await DataStore.objects.filter(key=key).aexists():
|
794
|
-
return key
|
795
|
-
await DataStore.objects.acreate(value=data, key=key, owner=user, private=private)
|
796
|
-
return key
|
797
|
-
|
798
|
-
@staticmethod
|
799
|
-
async def aretrieve_public_data(key: str):
|
800
|
-
return await DataStore.objects.filter(key=key, private=False).afirst()
|
801
|
-
|
802
|
-
|
803
846
|
class ConversationAdapters:
|
804
847
|
@staticmethod
|
805
848
|
def make_public_conversation_copy(conversation: Conversation):
|
@@ -812,6 +855,7 @@ class ConversationAdapters:
|
|
812
855
|
)
|
813
856
|
|
814
857
|
@staticmethod
|
858
|
+
@require_valid_user
|
815
859
|
def get_conversation_by_user(
|
816
860
|
user: KhojUser, client_application: ClientApplication = None, conversation_id: str = None
|
817
861
|
) -> Optional[Conversation]:
|
@@ -830,6 +874,7 @@ class ConversationAdapters:
|
|
830
874
|
return conversation
|
831
875
|
|
832
876
|
@staticmethod
|
877
|
+
@require_valid_user
|
833
878
|
def get_conversation_sessions(user: KhojUser, client_application: ClientApplication = None):
|
834
879
|
return (
|
835
880
|
Conversation.objects.filter(user=user, client=client_application)
|
@@ -838,6 +883,7 @@ class ConversationAdapters:
|
|
838
883
|
)
|
839
884
|
|
840
885
|
@staticmethod
|
886
|
+
@arequire_valid_user
|
841
887
|
async def aset_conversation_title(
|
842
888
|
user: KhojUser, client_application: ClientApplication, conversation_id: str, title: str
|
843
889
|
):
|
@@ -855,6 +901,7 @@ class ConversationAdapters:
|
|
855
901
|
return Conversation.objects.filter(id=conversation_id).first()
|
856
902
|
|
857
903
|
@staticmethod
|
904
|
+
@arequire_valid_user
|
858
905
|
async def acreate_conversation_session(
|
859
906
|
user: KhojUser, client_application: ClientApplication = None, agent_slug: str = None, title: str = None
|
860
907
|
):
|
@@ -871,6 +918,7 @@ class ConversationAdapters:
|
|
871
918
|
)
|
872
919
|
|
873
920
|
@staticmethod
|
921
|
+
@require_valid_user
|
874
922
|
def create_conversation_session(
|
875
923
|
user: KhojUser, client_application: ClientApplication = None, agent_slug: str = None, title: str = None
|
876
924
|
):
|
@@ -883,6 +931,7 @@ class ConversationAdapters:
|
|
883
931
|
return Conversation.objects.create(user=user, client=client_application, agent=agent, title=title)
|
884
932
|
|
885
933
|
@staticmethod
|
934
|
+
@arequire_valid_user
|
886
935
|
async def aget_conversation_by_user(
|
887
936
|
user: KhojUser,
|
888
937
|
client_application: ClientApplication = None,
|
@@ -907,6 +956,7 @@ class ConversationAdapters:
|
|
907
956
|
)
|
908
957
|
|
909
958
|
@staticmethod
|
959
|
+
@arequire_valid_user
|
910
960
|
async def adelete_conversation_by_user(
|
911
961
|
user: KhojUser, client_application: ClientApplication = None, conversation_id: str = None
|
912
962
|
):
|
@@ -915,6 +965,7 @@ class ConversationAdapters:
|
|
915
965
|
return await Conversation.objects.filter(user=user, client=client_application).adelete()
|
916
966
|
|
917
967
|
@staticmethod
|
968
|
+
@require_valid_user
|
918
969
|
def has_any_conversation_config(user: KhojUser):
|
919
970
|
return ChatModelOptions.objects.filter(user=user).exists()
|
920
971
|
|
@@ -951,6 +1002,7 @@ class ConversationAdapters:
|
|
951
1002
|
return OpenAIProcessorConversationConfig.objects.filter().exists()
|
952
1003
|
|
953
1004
|
@staticmethod
|
1005
|
+
@arequire_valid_user
|
954
1006
|
async def aset_user_conversation_processor(user: KhojUser, conversation_processor_config_id: int):
|
955
1007
|
config = await ChatModelOptions.objects.filter(id=conversation_processor_config_id).afirst()
|
956
1008
|
if not config:
|
@@ -959,6 +1011,7 @@ class ConversationAdapters:
|
|
959
1011
|
return new_config
|
960
1012
|
|
961
1013
|
@staticmethod
|
1014
|
+
@arequire_valid_user
|
962
1015
|
async def aset_user_voice_model(user: KhojUser, model_id: str):
|
963
1016
|
config = await VoiceModelOption.objects.filter(model_id=model_id).afirst()
|
964
1017
|
if not config:
|
@@ -1143,6 +1196,7 @@ class ConversationAdapters:
|
|
1143
1196
|
return enabled_scrapers
|
1144
1197
|
|
1145
1198
|
@staticmethod
|
1199
|
+
@require_valid_user
|
1146
1200
|
def create_conversation_from_public_conversation(
|
1147
1201
|
user: KhojUser, public_conversation: PublicConversation, client_app: ClientApplication
|
1148
1202
|
):
|
@@ -1159,6 +1213,7 @@ class ConversationAdapters:
|
|
1159
1213
|
)
|
1160
1214
|
|
1161
1215
|
@staticmethod
|
1216
|
+
@require_valid_user
|
1162
1217
|
def save_conversation(
|
1163
1218
|
user: KhojUser,
|
1164
1219
|
conversation_log: dict,
|
@@ -1208,6 +1263,7 @@ class ConversationAdapters:
|
|
1208
1263
|
return await SpeechToTextModelOptions.objects.filter().afirst()
|
1209
1264
|
|
1210
1265
|
@staticmethod
|
1266
|
+
@arequire_valid_user
|
1211
1267
|
async def aget_conversation_starters(user: KhojUser, max_results=3):
|
1212
1268
|
all_questions = []
|
1213
1269
|
if await ReflectiveQuestion.objects.filter(user=user).aexists():
|
@@ -1337,6 +1393,7 @@ class ConversationAdapters:
|
|
1337
1393
|
return conversation.file_filters
|
1338
1394
|
|
1339
1395
|
@staticmethod
|
1396
|
+
@require_valid_user
|
1340
1397
|
def delete_message_by_turn_id(user: KhojUser, conversation_id: str, turn_id: str):
|
1341
1398
|
conversation = ConversationAdapters.get_conversation_by_user(user, conversation_id=conversation_id)
|
1342
1399
|
if not conversation or not conversation.conversation_log or not conversation.conversation_log.get("chat"):
|
@@ -1355,48 +1412,63 @@ class FileObjectAdapters:
|
|
1355
1412
|
file_object.save()
|
1356
1413
|
|
1357
1414
|
@staticmethod
|
1415
|
+
@require_valid_user
|
1358
1416
|
def create_file_object(user: KhojUser, file_name: str, raw_text: str):
|
1359
1417
|
return FileObject.objects.create(user=user, file_name=file_name, raw_text=raw_text)
|
1360
1418
|
|
1361
1419
|
@staticmethod
|
1420
|
+
@require_valid_user
|
1362
1421
|
def get_file_object_by_name(user: KhojUser, file_name: str):
|
1363
1422
|
return FileObject.objects.filter(user=user, file_name=file_name).first()
|
1364
1423
|
|
1365
1424
|
@staticmethod
|
1425
|
+
@require_valid_user
|
1366
1426
|
def get_all_file_objects(user: KhojUser):
|
1367
1427
|
return FileObject.objects.filter(user=user).all()
|
1368
1428
|
|
1369
1429
|
@staticmethod
|
1430
|
+
@require_valid_user
|
1370
1431
|
def delete_file_object_by_name(user: KhojUser, file_name: str):
|
1371
1432
|
return FileObject.objects.filter(user=user, file_name=file_name).delete()
|
1372
1433
|
|
1373
1434
|
@staticmethod
|
1435
|
+
@require_valid_user
|
1374
1436
|
def delete_all_file_objects(user: KhojUser):
|
1375
1437
|
return FileObject.objects.filter(user=user).delete()
|
1376
1438
|
|
1377
1439
|
@staticmethod
|
1378
|
-
async def
|
1440
|
+
async def aupdate_raw_text(file_object: FileObject, new_raw_text: str):
|
1379
1441
|
file_object.raw_text = new_raw_text
|
1380
1442
|
await file_object.asave()
|
1381
1443
|
|
1382
1444
|
@staticmethod
|
1383
|
-
|
1445
|
+
@arequire_valid_user
|
1446
|
+
async def acreate_file_object(user: KhojUser, file_name: str, raw_text: str):
|
1384
1447
|
return await FileObject.objects.acreate(user=user, file_name=file_name, raw_text=raw_text)
|
1385
1448
|
|
1386
1449
|
@staticmethod
|
1387
|
-
|
1450
|
+
@arequire_valid_user
|
1451
|
+
async def aget_file_objects_by_name(user: KhojUser, file_name: str, agent: Agent = None):
|
1388
1452
|
return await sync_to_async(list)(FileObject.objects.filter(user=user, file_name=file_name, agent=agent))
|
1389
1453
|
|
1390
1454
|
@staticmethod
|
1391
|
-
|
1455
|
+
@arequire_valid_user
|
1456
|
+
async def aget_file_objects_by_names(user: KhojUser, file_names: List[str]):
|
1457
|
+
return await sync_to_async(list)(FileObject.objects.filter(user=user, file_name__in=file_names))
|
1458
|
+
|
1459
|
+
@staticmethod
|
1460
|
+
@arequire_valid_user
|
1461
|
+
async def aget_all_file_objects(user: KhojUser):
|
1392
1462
|
return await sync_to_async(list)(FileObject.objects.filter(user=user))
|
1393
1463
|
|
1394
1464
|
@staticmethod
|
1395
|
-
|
1465
|
+
@arequire_valid_user
|
1466
|
+
async def adelete_file_object_by_name(user: KhojUser, file_name: str):
|
1396
1467
|
return await FileObject.objects.filter(user=user, file_name=file_name).adelete()
|
1397
1468
|
|
1398
1469
|
@staticmethod
|
1399
|
-
|
1470
|
+
@arequire_valid_user
|
1471
|
+
async def adelete_all_file_objects(user: KhojUser):
|
1400
1472
|
return await FileObject.objects.filter(user=user).adelete()
|
1401
1473
|
|
1402
1474
|
|
@@ -1406,15 +1478,18 @@ class EntryAdapters:
|
|
1406
1478
|
date_filter = DateFilter()
|
1407
1479
|
|
1408
1480
|
@staticmethod
|
1481
|
+
@require_valid_user
|
1409
1482
|
def does_entry_exist(user: KhojUser, hashed_value: str) -> bool:
|
1410
1483
|
return Entry.objects.filter(user=user, hashed_value=hashed_value).exists()
|
1411
1484
|
|
1412
1485
|
@staticmethod
|
1486
|
+
@require_valid_user
|
1413
1487
|
def delete_entry_by_file(user: KhojUser, file_path: str):
|
1414
1488
|
deleted_count, _ = Entry.objects.filter(user=user, file_path=file_path).delete()
|
1415
1489
|
return deleted_count
|
1416
1490
|
|
1417
1491
|
@staticmethod
|
1492
|
+
@require_valid_user
|
1418
1493
|
def get_filtered_entries(user: KhojUser, file_type: str = None, file_source: str = None):
|
1419
1494
|
queryset = Entry.objects.filter(user=user)
|
1420
1495
|
|
@@ -1427,6 +1502,7 @@ class EntryAdapters:
|
|
1427
1502
|
return queryset
|
1428
1503
|
|
1429
1504
|
@staticmethod
|
1505
|
+
@require_valid_user
|
1430
1506
|
def delete_all_entries(user: KhojUser, file_type: str = None, file_source: str = None, batch_size=1000):
|
1431
1507
|
deleted_count = 0
|
1432
1508
|
queryset = EntryAdapters.get_filtered_entries(user, file_type, file_source)
|
@@ -1438,6 +1514,7 @@ class EntryAdapters:
|
|
1438
1514
|
return deleted_count
|
1439
1515
|
|
1440
1516
|
@staticmethod
|
1517
|
+
@arequire_valid_user
|
1441
1518
|
async def adelete_all_entries(user: KhojUser, file_type: str = None, file_source: str = None, batch_size=1000):
|
1442
1519
|
deleted_count = 0
|
1443
1520
|
queryset = EntryAdapters.get_filtered_entries(user, file_type, file_source)
|
@@ -1449,10 +1526,12 @@ class EntryAdapters:
|
|
1449
1526
|
return deleted_count
|
1450
1527
|
|
1451
1528
|
@staticmethod
|
1529
|
+
@require_valid_user
|
1452
1530
|
def get_existing_entry_hashes_by_file(user: KhojUser, file_path: str):
|
1453
1531
|
return Entry.objects.filter(user=user, file_path=file_path).values_list("hashed_value", flat=True)
|
1454
1532
|
|
1455
1533
|
@staticmethod
|
1534
|
+
@require_valid_user
|
1456
1535
|
def delete_entry_by_hash(user: KhojUser, hashed_values: List[str]):
|
1457
1536
|
Entry.objects.filter(user=user, hashed_value__in=hashed_values).delete()
|
1458
1537
|
|
@@ -1464,6 +1543,7 @@ class EntryAdapters:
|
|
1464
1543
|
)
|
1465
1544
|
|
1466
1545
|
@staticmethod
|
1546
|
+
@require_valid_user
|
1467
1547
|
def user_has_entries(user: KhojUser):
|
1468
1548
|
return Entry.objects.filter(user=user).exists()
|
1469
1549
|
|
@@ -1472,6 +1552,7 @@ class EntryAdapters:
|
|
1472
1552
|
return Entry.objects.filter(agent=agent).exists()
|
1473
1553
|
|
1474
1554
|
@staticmethod
|
1555
|
+
@arequire_valid_user
|
1475
1556
|
async def auser_has_entries(user: KhojUser):
|
1476
1557
|
return await Entry.objects.filter(user=user).aexists()
|
1477
1558
|
|
@@ -1482,10 +1563,12 @@ class EntryAdapters:
|
|
1482
1563
|
return await Entry.objects.filter(agent=agent).aexists()
|
1483
1564
|
|
1484
1565
|
@staticmethod
|
1566
|
+
@arequire_valid_user
|
1485
1567
|
async def adelete_entry_by_file(user: KhojUser, file_path: str):
|
1486
1568
|
return await Entry.objects.filter(user=user, file_path=file_path).adelete()
|
1487
1569
|
|
1488
1570
|
@staticmethod
|
1571
|
+
@arequire_valid_user
|
1489
1572
|
async def adelete_entries_by_filenames(user: KhojUser, filenames: List[str], batch_size=1000):
|
1490
1573
|
deleted_count = 0
|
1491
1574
|
for i in range(0, len(filenames), batch_size):
|
@@ -1504,6 +1587,7 @@ class EntryAdapters:
|
|
1504
1587
|
)
|
1505
1588
|
|
1506
1589
|
@staticmethod
|
1590
|
+
@require_valid_user
|
1507
1591
|
def get_all_filenames_by_source(user: KhojUser, file_source: str):
|
1508
1592
|
return (
|
1509
1593
|
Entry.objects.filter(user=user, file_source=file_source)
|
@@ -1512,6 +1596,7 @@ class EntryAdapters:
|
|
1512
1596
|
)
|
1513
1597
|
|
1514
1598
|
@staticmethod
|
1599
|
+
@require_valid_user
|
1515
1600
|
def get_size_of_indexed_data_in_mb(user: KhojUser):
|
1516
1601
|
entries = Entry.objects.filter(user=user).iterator()
|
1517
1602
|
total_size = sum(sys.getsizeof(entry.compiled) for entry in entries)
|
@@ -1532,6 +1617,9 @@ class EntryAdapters:
|
|
1532
1617
|
if agent != None:
|
1533
1618
|
owner_filter |= Q(agent=agent)
|
1534
1619
|
|
1620
|
+
if owner_filter == Q():
|
1621
|
+
return Entry.objects.none()
|
1622
|
+
|
1535
1623
|
if len(word_filters) == 0 and len(file_filters) == 0 and len(date_filters) == 0:
|
1536
1624
|
return Entry.objects.filter(owner_filter)
|
1537
1625
|
|
@@ -1606,10 +1694,12 @@ class EntryAdapters:
|
|
1606
1694
|
return relevant_entries[:max_results]
|
1607
1695
|
|
1608
1696
|
@staticmethod
|
1697
|
+
@require_valid_user
|
1609
1698
|
def get_unique_file_types(user: KhojUser):
|
1610
1699
|
return Entry.objects.filter(user=user).values_list("file_type", flat=True).distinct()
|
1611
1700
|
|
1612
1701
|
@staticmethod
|
1702
|
+
@require_valid_user
|
1613
1703
|
def get_unique_file_sources(user: KhojUser):
|
1614
1704
|
return Entry.objects.filter(user=user).values_list("file_source", flat=True).distinct().all()
|
1615
1705
|
|
khoj/database/models/__init__.py
CHANGED
@@ -458,7 +458,11 @@ class Conversation(BaseModel):
|
|
458
458
|
user = models.ForeignKey(KhojUser, on_delete=models.CASCADE)
|
459
459
|
conversation_log = models.JSONField(default=dict)
|
460
460
|
client = models.ForeignKey(ClientApplication, on_delete=models.CASCADE, default=None, null=True, blank=True)
|
461
|
+
|
462
|
+
# Slug is an app-generated conversation identifier. Need not be unique. Used as display title essentially.
|
461
463
|
slug = models.CharField(max_length=200, default=None, null=True, blank=True)
|
464
|
+
|
465
|
+
# The title field is explicitly set by the user.
|
462
466
|
title = models.CharField(max_length=200, default=None, null=True, blank=True)
|
463
467
|
agent = models.ForeignKey(Agent, on_delete=models.SET_NULL, default=None, null=True, blank=True)
|
464
468
|
file_filters = models.JSONField(default=list)
|