khoj 1.23.3__py3-none-any.whl → 1.23.3.dev4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- khoj/database/adapters/__init__.py +4 -0
- khoj/database/migrations/0063_conversation_add_unique_id_field.py +19 -0
- khoj/database/migrations/0064_populate_unique_id.py +20 -0
- khoj/database/migrations/0065_add_unique_constraint_to_unique_id.py +17 -0
- khoj/database/models/__init__.py +1 -0
- khoj/interface/compiled/404/index.html +1 -1
- khoj/interface/compiled/_next/static/chunks/1603-7914d7712a47690d.js +1 -0
- khoj/interface/compiled/_next/static/chunks/app/agents/page-cd8e2ba85287e1f6.js +1 -0
- khoj/interface/compiled/_next/static/chunks/app/automations/{page-3f4b6ff0261e19b7.js → page-6ea3381528603372.js} +1 -1
- khoj/interface/compiled/_next/static/chunks/app/chat/page-5752b57c1e43e8a7.js +1 -0
- khoj/interface/compiled/_next/static/chunks/app/factchecker/{page-828cf3c5b8e3af79.js → page-04a19ab1a988976f.js} +1 -1
- khoj/interface/compiled/_next/static/chunks/app/page-cd8d5d12595676d7.js +1 -0
- khoj/interface/compiled/_next/static/chunks/app/search/{page-dcd385f03255ef36.js → page-fa15807b1ad7e30b.js} +1 -1
- khoj/interface/compiled/_next/static/chunks/app/settings/{page-ddcd51147d18c694.js → page-1a2acc46cdabaf4a.js} +1 -1
- khoj/interface/compiled/_next/static/chunks/app/share/chat/{page-a84001b4724b5463.js → page-e20f54450d3ce6c0.js} +1 -1
- khoj/interface/compiled/_next/static/chunks/{webpack-56df4667546b1c60.js → webpack-072d1cbdec7e1782.js} +1 -1
- khoj/interface/compiled/_next/static/css/43939edc2f9b2043.css +1 -0
- khoj/interface/compiled/_next/static/css/592ca99f5122e75a.css +1 -0
- khoj/interface/compiled/agents/index.html +1 -1
- khoj/interface/compiled/agents/index.txt +2 -2
- khoj/interface/compiled/automations/index.html +1 -1
- khoj/interface/compiled/automations/index.txt +2 -2
- khoj/interface/compiled/chat/index.html +1 -1
- khoj/interface/compiled/chat/index.txt +2 -2
- khoj/interface/compiled/factchecker/index.html +1 -1
- khoj/interface/compiled/factchecker/index.txt +2 -2
- khoj/interface/compiled/index.html +1 -1
- khoj/interface/compiled/index.txt +2 -2
- khoj/interface/compiled/search/index.html +1 -1
- khoj/interface/compiled/search/index.txt +2 -2
- khoj/interface/compiled/settings/index.html +1 -1
- khoj/interface/compiled/settings/index.txt +2 -2
- khoj/interface/compiled/share/chat/index.html +1 -1
- khoj/interface/compiled/share/chat/index.txt +2 -2
- khoj/interface/email/task.html +34 -31
- khoj/interface/email/welcome.html +53 -82
- khoj/main.py +1 -1
- khoj/processor/content/images/image_to_entries.py +4 -6
- khoj/processor/conversation/utils.py +7 -12
- khoj/routers/api_chat.py +31 -2
- khoj/routers/helpers.py +0 -4
- khoj/utils/cli.py +0 -6
- khoj/utils/constants.py +2 -9
- khoj/utils/initialization.py +71 -158
- {khoj-1.23.3.dist-info → khoj-1.23.3.dev4.dist-info}/METADATA +2 -2
- {khoj-1.23.3.dist-info → khoj-1.23.3.dev4.dist-info}/RECORD +54 -51
- khoj/interface/compiled/_next/static/chunks/1603-fb2d80ae73990df3.js +0 -1
- khoj/interface/compiled/_next/static/chunks/app/agents/page-922694b75f1fb67b.js +0 -1
- khoj/interface/compiled/_next/static/chunks/app/chat/page-6fac068cc1cca546.js +0 -1
- khoj/interface/compiled/_next/static/chunks/app/page-b9e0500234c59a3e.js +0 -1
- khoj/interface/compiled/_next/static/css/4cae6c0e5c72fb2d.css +0 -1
- khoj/interface/compiled/_next/static/css/92c48eece0b102b9.css +0 -1
- /khoj/interface/compiled/_next/static/chunks/{8423-ce22327cf2d2edae.js → 8423-14fc72aec9104ce9.js} +0 -0
- /khoj/interface/compiled/_next/static/chunks/{9178-3a0baad1c172d515.js → 9178-c153fc402c970365.js} +0 -0
- /khoj/interface/compiled/_next/static/chunks/{9417-2e54c6fd056982d8.js → 9417-5d14ac74aaab2c66.js} +0 -0
- /khoj/interface/compiled/_next/static/{H8WZzL_OHSsjoAIEAIo8M → d3ySlDLLYAYu1Jr-Dc35R}/_buildManifest.js +0 -0
- /khoj/interface/compiled/_next/static/{H8WZzL_OHSsjoAIEAIo8M → d3ySlDLLYAYu1Jr-Dc35R}/_ssgManifest.js +0 -0
- {khoj-1.23.3.dist-info → khoj-1.23.3.dev4.dist-info}/WHEEL +0 -0
- {khoj-1.23.3.dist-info → khoj-1.23.3.dev4.dist-info}/entry_points.txt +0 -0
- {khoj-1.23.3.dist-info → khoj-1.23.3.dev4.dist-info}/licenses/LICENSE +0 -0
|
@@ -1,90 +1,61 @@
|
|
|
1
1
|
<!DOCTYPE html>
|
|
2
|
-
<html
|
|
3
|
-
|
|
4
|
-
<
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
<
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
<
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
<
|
|
20
|
-
style="
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
completely open-source.</p>
|
|
26
|
-
|
|
27
|
-
<a href="https://app.khoj.dev" target="_blank"
|
|
28
|
-
style="display: block; width: 200px; text-align: center; padding: 10px; margin: 20px auto; background-color: #FFA07A; color: #ffffff; text-decoration: none; border-radius: 5px; font-weight: bold; font-size: 16px; text-transform: uppercase;">Get
|
|
29
|
-
Started</a>
|
|
30
|
-
|
|
31
|
-
<p style="font-size: 16px; color: #333; margin-bottom: 20px;">You're about to get a whole lot more productive.
|
|
32
|
-
</p>
|
|
33
|
-
<a href="https://docs.khoj.dev/features/online_search"
|
|
34
|
-
style="color: #FFA07A; text-decoration: none; font-weight: bold; font-size: 14px;">
|
|
35
|
-
<div style="display: grid; grid-template-columns: 1fr 1fr; grid-gap: 20px; margin-bottom: 20px;">
|
|
36
|
-
<div style="background-color: #f8f9fa; border-left: 4px solid #FFA07A; padding: 15px;">
|
|
37
|
-
<h3 style="color: #2c3e50; margin-top: 0; font-size: 18px;">Ditch the search bar</h3>
|
|
38
|
-
<p style="font-size: 14px; color: #666; margin-bottom: 0;">You don't need to click around Google
|
|
39
|
-
results
|
|
40
|
-
and sift through information yourself, because Khoj is connected to the internet.</p>
|
|
41
|
-
</div>
|
|
42
|
-
</a>
|
|
43
|
-
<a href="https://app.khoj.dev/agents"
|
|
44
|
-
style="color: #FFA07A; text-decoration: none; font-weight: bold; font-size: 14px;">
|
|
45
|
-
<div style="background-color: #f8f9fa; border-left: 4px solid #FFA07A; padding: 15px;">
|
|
46
|
-
<h3 style="color: #2c3e50; margin-top: 0; font-size: 18px;">Get a village, not just an agent</h3>
|
|
47
|
-
<p style="font-size: 14px; color: #666; margin-bottom: 0;">Khoj can fill the need for more specialized
|
|
48
|
-
assistance, such as tutoring, with its curated agents. You get a whole team, always available.</p>
|
|
2
|
+
<html>
|
|
3
|
+
<head>
|
|
4
|
+
<title>Welcome to Khoj</title>
|
|
5
|
+
</head>
|
|
6
|
+
<body>
|
|
7
|
+
<body style="font-family: 'Verdana', sans-serif; font-weight: 400; font-style: normal; padding: 0; text-align: left; width: 600px; margin: 20px auto;">
|
|
8
|
+
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no">
|
|
9
|
+
<a class="logo" href="https://khoj.dev" target="_blank" style="text-decoration: none; text-decoration: underline dotted;">
|
|
10
|
+
<img src="https://assets.khoj.dev/khoj_logo.png" alt="Khoj Logo" style="width: 100px;">
|
|
11
|
+
</a>
|
|
12
|
+
<div class="calls-to-action" style="margin-top: 20px;">
|
|
13
|
+
<div>
|
|
14
|
+
<h1 style="color: #333; font-size: large; font-weight: bold; margin: 0; line-height: 1.5; background-color: #fee285; padding: 8px; box-shadow: 6px 6px rgba(0, 0, 0, 1.5);">Merge AI with your brain</h1>
|
|
15
|
+
<p style="color: #333; font-size: medium; margin-top: 20px; padding: 0; line-height: 1.5;">Hi {{name}}! We are psyched to be part of your journey with personal AI. To better help you, we're committed to staying transparent, accessible, and completely open-source.</p>
|
|
16
|
+
<a class="button" href="https://app.khoj.dev" target="_blank" style="display: block; width: 200px; text-align: center; padding: 10px; margin-top: 20px; color: #333; background-color: #fee285; text-decoration: none; border-radius: 5px; font-weight: bold; transition: background-color 0.3s ease; box-shadow: 6px 6px rgba(0, 0, 0, 1.0); padding: 4px; font-size: large; text-transform: uppercase;">Get Started</a>
|
|
17
|
+
<p style="color: #333; font-size: medium; margin-top: 20px; padding: 0; line-height: 1.5;">You're about to get a whole lot more productive.</p>
|
|
18
|
+
|
|
19
|
+
<div style="display: grid; grid-template-columns: 1fr 1fr; grid-gap: 12px; margin-top: 20px;">
|
|
20
|
+
<div style="border: 1px solid black; border-radius: 8px; padding: 8px; box-shadow: 6px 6px rgba(0, 0, 0, 1.0); margin-top: 20px;">
|
|
21
|
+
<a href="https://docs.khoj.dev/features/online_search" style="text-decoration: none; text-decoration: underline dotted;">
|
|
22
|
+
<h3 style="color: #333; font-size: large; margin: 0; padding: 0; line-height: 2.0; background-color: #b8f1c7; padding: 8px; ">Ditch the search bar</h3>
|
|
23
|
+
</a>
|
|
24
|
+
<p style="color: #333; font-size: medium; margin-top: 20px; padding: 0; line-height: 1.5;">You don't need to click around Google results and sift through information yourself, because Khoj is connected to the internet.</p>
|
|
49
25
|
</div>
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
<
|
|
55
|
-
<p style="font-size: 14px; color: #666; margin-bottom: 0;">Build on top of your digital brain. Khoj
|
|
56
|
-
stores whatever data you share with it, so you can get answers from your personal notes and
|
|
57
|
-
documents in your native language.</p>
|
|
26
|
+
<div style="border: 1px solid black; border-radius: 8px; padding: 8px; box-shadow: 6px 6px rgba(0, 0, 0, 1.0); margin-top: 20px;">
|
|
27
|
+
<a href="https://app.khoj.dev/agents" style="text-decoration: none; text-decoration: underline dotted;">
|
|
28
|
+
<h3 style="color: #333; font-size: large; margin: 0; padding: 0; line-height: 2.0; background-color: #b8f1c7; padding: 8px;">Get a village, not just an agent</h3>
|
|
29
|
+
</a>
|
|
30
|
+
<p style="color: #333; font-size: medium; margin-top: 20px; padding: 0; line-height: 1.5;">Khoj can fill the need for more specialized assistance, <a href="https://blog.khoj.dev/posts/using-khoj-for-studying/">such as tutoring</a>, with its curated agents. You get a whole team, always available.</p>
|
|
58
31
|
</div>
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
<
|
|
64
|
-
<p style="font-size: 14px; color: #666; margin-bottom: 0;">With your shared data, Khoj can help you
|
|
65
|
-
create astoundingly personal images depicting scenes of what's important to you.</p>
|
|
32
|
+
<div style="border: 1px solid black; border-radius: 8px; padding: 8px; box-shadow: 6px 6px rgba(0, 0, 0, 1.0); margin-top: 20px;">
|
|
33
|
+
<a href="https://docs.khoj.dev/category/clients" style="text-decoration: none; text-decoration: underline dotted;">
|
|
34
|
+
<h3 style="color: #333; font-size: large; margin: 0; padding: 0; line-height: 2.0; background-color: #b8f1c7; padding: 8px;">Available where you are</h3>
|
|
35
|
+
</a>
|
|
36
|
+
<p style="color: #333; font-size: medium; margin-top: 20px; padding: 0; line-height: 1.5;">Build on top of your digital brain. Khoj stores whatever data you share with it, so you can get answers from your personal notes and documents in your native language. You can engage from your desktop, Obsidian, WhatsApp, or the web.</p>
|
|
66
37
|
</div>
|
|
67
|
-
|
|
38
|
+
<div style="border: 1px solid black; border-radius: 8px; padding: 8px; box-shadow: 6px 6px rgba(0, 0, 0, 1.0); margin-top: 20px;">
|
|
39
|
+
<a href="https://blog.khoj.dev/posts/how-khoj-generates-images/" style="text-decoration: none; text-decoration: underline dotted;">
|
|
40
|
+
<h3 style="color: #333; font-size: large; margin: 0; padding: 0; line-height: 2.0; background-color: #b8f1c7; padding: 8px;">Create rich, contextual images</h3>
|
|
41
|
+
</a>
|
|
42
|
+
<p style="color: #333; font-size: medium; margin-top: 20px; padding: 0; line-height: 1.5;">With your shared data, Khoj can help you create astoundingly personal images depicting scenes of what's important to you.</p>
|
|
43
|
+
</div>
|
|
44
|
+
</div>
|
|
68
45
|
</div>
|
|
46
|
+
</div>
|
|
47
|
+
<p style="color: #333; font-size: medium; margin-top: 20px; padding: 0; line-height: 1.5;">Like something? Dislike something? Searching for some other magical feature? Our inbox is always open for feedback! Reply to this email and say hi to introduce yourself 👋🏽.</p>
|
|
48
|
+
|
|
49
|
+
<p style="color: #333; font-size: large; margin-top: 20px; padding: 0; line-height: 1.5;">- The Khoj Team</p>
|
|
50
|
+
<table style="width: 100%; margin-top: 20px;">
|
|
51
|
+
<tr>
|
|
52
|
+
<td style="text-align: center;"><a href="https://docs.khoj.dev" target="_blank" style="padding: 8px; color: #333; background-color: #fee285; border-radius: 8px; box-shadow: 6px 6px rgba(0, 0, 0, 1.0);">Docs</a></td>
|
|
53
|
+
<td style="text-align: center;"><a href="https://github.com/khoj-ai/khoj" target="_blank" style="padding: 8px; color: #333; background-color: #fee285; border-radius: 8px; box-shadow: 6px 6px rgba(0, 0, 0, 1.0);">GitHub</a></td>
|
|
54
|
+
<td style="text-align: center;"><a href="https://twitter.com/khoj_ai" target="_blank" style="padding: 8px; color: #333; background-color: #fee285; border-radius: 8px; box-shadow: 6px 6px rgba(0, 0, 0, 1.0);">Twitter</a></td>
|
|
55
|
+
<td style="text-align: center;"><a href="https://www.linkedin.com/company/khoj-ai" target="_blank" style="padding: 8px; color: #333; background-color: #fee285; border-radius: 8px; box-shadow: 6px 6px rgba(0, 0, 0, 1.0);">LinkedIn</a></td>
|
|
56
|
+
<td style="text-align: center;"><a href="https://discord.gg/BDgyabRM6e" target="_blank" style="padding: 8px; color: #333; background-color: #fee285; border-radius: 8px; box-shadow: 6px 6px rgba(0, 0, 0, 1.0);">Discord</a></td>
|
|
57
|
+
</tr>
|
|
58
|
+
</table>
|
|
69
59
|
|
|
70
|
-
<p style="font-size: 16px; color: #333; margin-bottom: 20px;">Like something? Dislike something? Searching for
|
|
71
|
-
some other magical feature? Our inbox is always open for feedback! Reply to this email and say hi to
|
|
72
|
-
introduce yourself 👋🏽.</p>
|
|
73
|
-
<div style="font-size: 18px; font-weight: bold; margin-top: 30px; text-align: right;">- The Khoj Team</div>
|
|
74
|
-
|
|
75
|
-
<div style="margin-top: 30px; text-align: center;">
|
|
76
|
-
<a href="https://docs.khoj.dev" target="_blank"
|
|
77
|
-
style="display: inline-block; margin: 0 10px; padding: 8px 15px; background-color: #FFA07A; color: #ffffff; text-decoration: none; border-radius: 5px;">Docs</a>
|
|
78
|
-
<a href="https://github.com/khoj-ai/khoj" target="_blank"
|
|
79
|
-
style="display: inline-block; margin: 0 10px; padding: 8px 15px; background-color: #FFA07A; color: #ffffff; text-decoration: none; border-radius: 5px;">GitHub</a>
|
|
80
|
-
<a href="https://twitter.com/khoj_ai" target="_blank"
|
|
81
|
-
style="display: inline-block; margin: 0 10px; padding: 8px 15px; background-color: #FFA07A; color: #ffffff; text-decoration: none; border-radius: 5px;">Twitter</a>
|
|
82
|
-
<a href="https://www.linkedin.com/company/khoj-ai" target="_blank"
|
|
83
|
-
style="display: inline-block; margin: 0 10px; padding: 8px 15px; background-color: #FFA07A; color: #ffffff; text-decoration: none; border-radius: 5px;">LinkedIn</a>
|
|
84
|
-
<a href="https://discord.gg/BDgyabRM6e" target="_blank"
|
|
85
|
-
style="display: inline-block; margin: 0 10px; padding: 8px 15px; background-color: #FFA07A; color: #ffffff; text-decoration: none; border-radius: 5px;">Discord</a>
|
|
86
|
-
</div>
|
|
87
|
-
</div>
|
|
88
60
|
</body>
|
|
89
|
-
|
|
90
61
|
</html>
|
khoj/main.py
CHANGED
|
@@ -131,7 +131,7 @@ def run(should_start_server=True):
|
|
|
131
131
|
logger.info(f"📦 Initializing DB:\n{db_migrate_output.getvalue().strip()}")
|
|
132
132
|
logger.debug(f"🌍 Initializing Web Client:\n{collectstatic_output.getvalue().strip()}")
|
|
133
133
|
|
|
134
|
-
initialization(
|
|
134
|
+
initialization()
|
|
135
135
|
|
|
136
136
|
# Create app directory, if it doesn't exist
|
|
137
137
|
state.config_file.parent.mkdir(parents=True, exist_ok=True)
|
|
@@ -4,6 +4,8 @@ import os
|
|
|
4
4
|
from datetime import datetime
|
|
5
5
|
from typing import Dict, List, Tuple
|
|
6
6
|
|
|
7
|
+
from rapidocr_onnxruntime import RapidOCR
|
|
8
|
+
|
|
7
9
|
from khoj.database.models import Entry as DbEntry
|
|
8
10
|
from khoj.database.models import KhojUser
|
|
9
11
|
from khoj.processor.content.text_to_entries import TextToEntries
|
|
@@ -56,6 +58,7 @@ class ImageToEntries(TextToEntries):
|
|
|
56
58
|
entry_to_location_map: List[Tuple[str, str]] = []
|
|
57
59
|
for image_file in image_files:
|
|
58
60
|
try:
|
|
61
|
+
loader = RapidOCR()
|
|
59
62
|
bytes = image_files[image_file]
|
|
60
63
|
# write the image to a temporary file
|
|
61
64
|
timestamp_now = datetime.utcnow().timestamp()
|
|
@@ -68,18 +71,13 @@ class ImageToEntries(TextToEntries):
|
|
|
68
71
|
bytes = image_files[image_file]
|
|
69
72
|
f.write(bytes)
|
|
70
73
|
try:
|
|
71
|
-
from rapidocr_onnxruntime import RapidOCR
|
|
72
|
-
|
|
73
|
-
loader = RapidOCR()
|
|
74
74
|
image_entries_per_file = ""
|
|
75
75
|
result, _ = loader(tmp_file)
|
|
76
76
|
if result:
|
|
77
77
|
expanded_entries = [text[1] for text in result]
|
|
78
78
|
image_entries_per_file = " ".join(expanded_entries)
|
|
79
79
|
except ImportError:
|
|
80
|
-
logger.warning(
|
|
81
|
-
f"Unable to process image or scanned file for text: {image_file}. This file will not be indexed."
|
|
82
|
-
)
|
|
80
|
+
logger.warning(f"Unable to process file: {image_file}. This file will not be indexed.")
|
|
83
81
|
continue
|
|
84
82
|
entry_to_location_map.append((image_entries_per_file, image_file))
|
|
85
83
|
entries.extend([image_entries_per_file])
|
|
@@ -18,20 +18,13 @@ from khoj.utils.helpers import is_none_or_empty, merge_dicts
|
|
|
18
18
|
|
|
19
19
|
logger = logging.getLogger(__name__)
|
|
20
20
|
model_to_prompt_size = {
|
|
21
|
-
# OpenAI Models
|
|
22
21
|
"gpt-3.5-turbo": 12000,
|
|
22
|
+
"gpt-3.5-turbo-0125": 12000,
|
|
23
|
+
"gpt-4-0125-preview": 20000,
|
|
23
24
|
"gpt-4-turbo-preview": 20000,
|
|
24
|
-
"gpt-4o": 20000,
|
|
25
25
|
"gpt-4o-mini": 20000,
|
|
26
26
|
"o1-preview": 20000,
|
|
27
27
|
"o1-mini": 20000,
|
|
28
|
-
# Google Models
|
|
29
|
-
"gemini-1.5-flash": 20000,
|
|
30
|
-
"gemini-1.5-pro": 20000,
|
|
31
|
-
# Anthropic Models
|
|
32
|
-
"claude-3-5-sonnet-20240620": 20000,
|
|
33
|
-
"claude-3-opus-20240229": 20000,
|
|
34
|
-
# Offline Models
|
|
35
28
|
"TheBloke/Mistral-7B-Instruct-v0.2-GGUF": 3500,
|
|
36
29
|
"NousResearch/Hermes-2-Pro-Mistral-7B-GGUF": 3500,
|
|
37
30
|
"bartowski/Meta-Llama-3.1-8B-Instruct-GGUF": 20000,
|
|
@@ -170,7 +163,7 @@ def generate_chatml_messages_with_context(
|
|
|
170
163
|
if loaded_model:
|
|
171
164
|
max_prompt_size = infer_max_tokens(loaded_model.n_ctx(), model_to_prompt_size.get(model_name, math.inf))
|
|
172
165
|
else:
|
|
173
|
-
max_prompt_size = model_to_prompt_size.get(model_name,
|
|
166
|
+
max_prompt_size = model_to_prompt_size.get(model_name, 2000)
|
|
174
167
|
|
|
175
168
|
# Scale lookback turns proportional to max prompt size supported by model
|
|
176
169
|
lookback_turns = max_prompt_size // 750
|
|
@@ -298,6 +291,8 @@ def reciprocal_conversation_to_chatml(message_pair):
|
|
|
298
291
|
return [ChatMessage(content=message, role=role) for message, role in zip(message_pair, ["user", "assistant"])]
|
|
299
292
|
|
|
300
293
|
|
|
301
|
-
def remove_json_codeblock(response
|
|
294
|
+
def remove_json_codeblock(response):
|
|
302
295
|
"""Remove any markdown json codeblock formatting if present. Useful for non schema enforceable models"""
|
|
303
|
-
|
|
296
|
+
if response.startswith("```json") and response.endswith("```"):
|
|
297
|
+
response = response[7:-3]
|
|
298
|
+
return response
|
khoj/routers/api_chat.py
CHANGED
|
@@ -224,6 +224,7 @@ def chat_history(
|
|
|
224
224
|
"conversation_id": conversation.id,
|
|
225
225
|
"slug": conversation.title if conversation.title else conversation.slug,
|
|
226
226
|
"agent": agent_metadata,
|
|
227
|
+
"unique_id": conversation.unique_id,
|
|
227
228
|
}
|
|
228
229
|
)
|
|
229
230
|
|
|
@@ -245,6 +246,33 @@ def chat_history(
|
|
|
245
246
|
return {"status": "ok", "response": meta_log}
|
|
246
247
|
|
|
247
248
|
|
|
249
|
+
@api_chat.get("/metadata")
|
|
250
|
+
def get_chat_metadata(
|
|
251
|
+
request: Request,
|
|
252
|
+
common: CommonQueryParams,
|
|
253
|
+
conversation_unique_id: str,
|
|
254
|
+
):
|
|
255
|
+
user = request.user.object
|
|
256
|
+
|
|
257
|
+
# Load Conversation Metadata
|
|
258
|
+
conversation = ConversationAdapters.get_conversation_by_unique_id(user, conversation_unique_id)
|
|
259
|
+
|
|
260
|
+
if conversation is None:
|
|
261
|
+
return Response(
|
|
262
|
+
content=json.dumps({"status": "error", "message": f"Conversation: {conversation_unique_id} not found"}),
|
|
263
|
+
status_code=404,
|
|
264
|
+
)
|
|
265
|
+
|
|
266
|
+
update_telemetry_state(
|
|
267
|
+
request=request,
|
|
268
|
+
telemetry_type="api",
|
|
269
|
+
api="chat_metadata",
|
|
270
|
+
**common.__dict__,
|
|
271
|
+
)
|
|
272
|
+
|
|
273
|
+
return {"status": "ok", "conversationId": conversation.id}
|
|
274
|
+
|
|
275
|
+
|
|
248
276
|
@api_chat.get("/share/history")
|
|
249
277
|
def get_shared_chat(
|
|
250
278
|
request: Request,
|
|
@@ -418,7 +446,7 @@ def chat_sessions(
|
|
|
418
446
|
conversations = conversations[:8]
|
|
419
447
|
|
|
420
448
|
sessions = conversations.values_list(
|
|
421
|
-
"id", "slug", "title", "agent__slug", "agent__name", "agent__avatar", "created_at", "updated_at"
|
|
449
|
+
"id", "slug", "title", "agent__slug", "agent__name", "agent__avatar", "created_at", "updated_at", "unique_id"
|
|
422
450
|
)
|
|
423
451
|
|
|
424
452
|
session_values = [
|
|
@@ -429,6 +457,7 @@ def chat_sessions(
|
|
|
429
457
|
"agent_avatar": session[5],
|
|
430
458
|
"created": session[6].strftime("%Y-%m-%d %H:%M:%S"),
|
|
431
459
|
"updated": session[7].strftime("%Y-%m-%d %H:%M:%S"),
|
|
460
|
+
"unique_id": str(session[8]),
|
|
432
461
|
}
|
|
433
462
|
for session in sessions
|
|
434
463
|
]
|
|
@@ -455,7 +484,7 @@ async def create_chat_session(
|
|
|
455
484
|
# Create new Conversation Session
|
|
456
485
|
conversation = await ConversationAdapters.acreate_conversation_session(user, request.user.client_app, agent_slug)
|
|
457
486
|
|
|
458
|
-
response = {"conversation_id": conversation.id}
|
|
487
|
+
response = {"conversation_id": conversation.id, "unique_id": str(conversation.unique_id)}
|
|
459
488
|
|
|
460
489
|
conversation_metadata = {
|
|
461
490
|
"agent": agent_slug,
|
khoj/routers/helpers.py
CHANGED
|
@@ -632,7 +632,6 @@ async def send_message_to_model_wrapper(
|
|
|
632
632
|
messages=truncated_messages,
|
|
633
633
|
loaded_model=loaded_model,
|
|
634
634
|
model=chat_model,
|
|
635
|
-
max_prompt_size=max_tokens,
|
|
636
635
|
streaming=False,
|
|
637
636
|
response_type=response_type,
|
|
638
637
|
)
|
|
@@ -722,7 +721,6 @@ def send_message_to_model_wrapper_sync(
|
|
|
722
721
|
system_message=system_message,
|
|
723
722
|
model_name=chat_model,
|
|
724
723
|
loaded_model=loaded_model,
|
|
725
|
-
max_prompt_size=max_tokens,
|
|
726
724
|
vision_enabled=vision_available,
|
|
727
725
|
model_type=conversation_config.model_type,
|
|
728
726
|
)
|
|
@@ -731,7 +729,6 @@ def send_message_to_model_wrapper_sync(
|
|
|
731
729
|
messages=truncated_messages,
|
|
732
730
|
loaded_model=loaded_model,
|
|
733
731
|
model=chat_model,
|
|
734
|
-
max_prompt_size=max_tokens,
|
|
735
732
|
streaming=False,
|
|
736
733
|
response_type=response_type,
|
|
737
734
|
)
|
|
@@ -742,7 +739,6 @@ def send_message_to_model_wrapper_sync(
|
|
|
742
739
|
user_message=message,
|
|
743
740
|
system_message=system_message,
|
|
744
741
|
model_name=chat_model,
|
|
745
|
-
max_prompt_size=max_tokens,
|
|
746
742
|
vision_enabled=vision_available,
|
|
747
743
|
model_type=conversation_config.model_type,
|
|
748
744
|
)
|
khoj/utils/cli.py
CHANGED
|
@@ -50,12 +50,6 @@ def cli(args=None):
|
|
|
50
50
|
default=False,
|
|
51
51
|
help="Run Khoj in anonymous mode. This does not require any login for connecting users.",
|
|
52
52
|
)
|
|
53
|
-
parser.add_argument(
|
|
54
|
-
"--non-interactive",
|
|
55
|
-
action="store_true",
|
|
56
|
-
default=False,
|
|
57
|
-
help="Start Khoj in non-interactive mode. Assumes interactive shell unavailable for config. E.g when run via Docker.",
|
|
58
|
-
)
|
|
59
53
|
|
|
60
54
|
args, remaining_args = parser.parse_known_args(args)
|
|
61
55
|
|
khoj/utils/constants.py
CHANGED
|
@@ -8,15 +8,8 @@ empty_escape_sequences = "\n|\r|\t| "
|
|
|
8
8
|
app_env_filepath = "~/.khoj/env"
|
|
9
9
|
telemetry_server = "https://khoj.beta.haletic.com/v1/telemetry"
|
|
10
10
|
content_directory = "~/.khoj/content/"
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
"bartowski/gemma-2-9b-it-GGUF",
|
|
14
|
-
"bartowski/gemma-2-2b-it-GGUF",
|
|
15
|
-
"bartowski/Phi-3.5-mini-instruct-GGUF",
|
|
16
|
-
]
|
|
17
|
-
default_openai_chat_models = ["gpt-4o-mini", "gpt-4o"]
|
|
18
|
-
default_gemini_chat_models = ["gemini-1.5-flash", "gemini-1.5-pro"]
|
|
19
|
-
default_anthropic_chat_models = ["claude-3-5-sonnet-20240620", "claude-3-opus-20240229"]
|
|
11
|
+
default_offline_chat_model = "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF"
|
|
12
|
+
default_online_chat_model = "gpt-4o-mini"
|
|
20
13
|
|
|
21
14
|
empty_config = {
|
|
22
15
|
"search-type": {
|