khoj 1.33.3.dev32__py3-none-any.whl

khoj/database/management/commands/convert_images_png_to_webp.py

@@ -0,0 +1,99 @@
+import base64
+import io
+
+from django.core.management.base import BaseCommand
+from PIL import Image
+
+from khoj.database.models import Conversation
+from khoj.utils.helpers import ImageIntentType
+
+
+class Command(BaseCommand):
+    help = "Convert all images to WebP format or reverse."
+
+    def add_arguments(self, parser):
+        # Add a new argument 'reverse' to the command
+        parser.add_argument(
+            "--reverse",
+            action="store_true",
+            help="Convert from WebP to PNG instead of PNG to WebP",
+        )
+
+    def handle(self, *args, **options):
+        updated_count = 0
+        for conversation in Conversation.objects.all():
+            conversation_updated = False
+            for chat in conversation.conversation_log.get("chat", []):
+                if (
+                    chat.get("by", "") == "khoj"
+                    and chat.get("intent", {}).get("type", "") == ImageIntentType.TEXT_TO_IMAGE.value
+                    and not options["reverse"]
+                ):
+                    # Decode the base64 encoded PNG image
+                    print("Decode the base64 encoded PNG image")
+                    decoded_image = base64.b64decode(chat["message"])
+
+                    # Convert images from PNG to WebP format
+                    print("Convert images from PNG to WebP format")
+                    image_io = io.BytesIO(decoded_image)
+                    with Image.open(image_io) as png_image:
+                        webp_image_io = io.BytesIO()
+                        png_image.save(webp_image_io, "WEBP")
+
+                        # Encode the WebP image back to base64
+                        webp_image_bytes = webp_image_io.getvalue()
+                        chat["message"] = base64.b64encode(webp_image_bytes).decode()
+                        chat["intent"]["type"] = ImageIntentType.TEXT_TO_IMAGE_V3.value
+                        webp_image_io.close()
+                    conversation_updated = True
+                    updated_count += 1
+
+                elif (
+                    chat.get("by", "") == "khoj"
+                    and chat.get("intent", {}).get("type", "") == ImageIntentType.TEXT_TO_IMAGE_V3.value
+                    and options["reverse"]
+                ):
+                    # Decode the base64 encoded WebP image
+                    print("Decode the base64 encoded WebP image")
+                    decoded_image = base64.b64decode(chat["message"])
+
+                    # Convert images from WebP to PNG format
+                    print("Convert images from WebP to PNG format")
+                    image_io = io.BytesIO(decoded_image)
+                    with Image.open(image_io) as png_image:
+                        webp_image_io = io.BytesIO()
+                        png_image.save(webp_image_io, "PNG")
+
+                        # Encode the WebP image back to base64
+                        webp_image_bytes = webp_image_io.getvalue()
+                        chat["message"] = base64.b64encode(webp_image_bytes).decode()
+                        chat["intent"]["type"] = ImageIntentType.TEXT_TO_IMAGE.value
+                        webp_image_io.close()
+                    conversation_updated = True
+                    updated_count += 1
+
+                elif (
+                    chat.get("by", "") == "khoj"
+                    and chat.get("intent", {}).get("type", "") == ImageIntentType.TEXT_TO_IMAGE2.value
+                ):
+                    if options["reverse"] and chat.get("message", "").endswith(".webp"):
+                        # Convert WebP url to PNG url
+                        print("Convert WebP url to PNG url")
+                        chat["message"] = chat["message"].replace(".webp", ".png")
+                        conversation_updated = True
+                        updated_count += 1
+                    elif chat.get("message", "").endswith(".png"):
+                        # Convert PNG url to WebP url
+                        print("Convert PNG url to WebP url")
+                        chat["message"] = chat["message"].replace(".png", ".webp")
+                        conversation_updated = True
+                        updated_count += 1
+
+            if conversation_updated:
+                print("Save the updated conversation")
+                conversation.save()
+
+        if updated_count > 0 and options["reverse"]:
+            self.stdout.write(self.style.SUCCESS(f"Successfully converted {updated_count} WebP images to PNG format."))
+        elif updated_count > 0:
+            self.stdout.write(self.style.SUCCESS(f"Successfully converted {updated_count} PNG images to WebP format."))

khoj/database/migrations/0001_khojuser.py

@@ -0,0 +1,98 @@
+# Generated by Django 4.2.5 on 2023-09-14 19:00
+
+import django.contrib.auth.models
+import django.contrib.auth.validators
+import django.utils.timezone
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    initial = True
+
+    dependencies = [
+        ("auth", "0012_alter_user_first_name_max_length"),
+    ]
+
+    run_before = [
+        ("admin", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="KhojUser",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("password", models.CharField(max_length=128, verbose_name="password")),
+                ("last_login", models.DateTimeField(blank=True, null=True, verbose_name="last login")),
+                (
+                    "is_superuser",
+                    models.BooleanField(
+                        default=False,
+                        help_text="Designates that this user has all permissions without explicitly assigning them.",
+                        verbose_name="superuser status",
+                    ),
+                ),
+                (
+                    "username",
+                    models.CharField(
+                        error_messages={"unique": "A user with that username already exists."},
+                        help_text="Required. 150 characters or fewer. Letters, digits and @/./+/-/_ only.",
+                        max_length=150,
+                        unique=True,
+                        validators=[django.contrib.auth.validators.UnicodeUsernameValidator()],
+                        verbose_name="username",
+                    ),
+                ),
+                ("first_name", models.CharField(blank=True, max_length=150, verbose_name="first name")),
+                ("last_name", models.CharField(blank=True, max_length=150, verbose_name="last name")),
+                ("email", models.EmailField(blank=True, max_length=254, verbose_name="email address")),
+                (
+                    "is_staff",
+                    models.BooleanField(
+                        default=False,
+                        help_text="Designates whether the user can log into this admin site.",
+                        verbose_name="staff status",
+                    ),
+                ),
+                (
+                    "is_active",
+                    models.BooleanField(
+                        default=True,
+                        help_text="Designates whether this user should be treated as active. Unselect this instead of deleting accounts.",
+                        verbose_name="active",
+                    ),
+                ),
+                ("date_joined", models.DateTimeField(default=django.utils.timezone.now, verbose_name="date joined")),
+                (
+                    "groups",
+                    models.ManyToManyField(
+                        blank=True,
+                        help_text="The groups this user belongs to. A user will get all permissions granted to each of their groups.",
+                        related_name="user_set",
+                        related_query_name="user",
+                        to="auth.group",
+                        verbose_name="groups",
+                    ),
+                ),
+                (
+                    "user_permissions",
+                    models.ManyToManyField(
+                        blank=True,
+                        help_text="Specific permissions for this user.",
+                        related_name="user_set",
+                        related_query_name="user",
+                        to="auth.permission",
+                        verbose_name="user permissions",
+                    ),
+                ),
+            ],
+            options={
+                "verbose_name": "user",
+                "verbose_name_plural": "users",
+                "abstract": False,
+            },
+            managers=[
+                ("objects", django.contrib.auth.models.UserManager()),
+            ],
+        ),
+    ]

khoj/database/migrations/0002_googleuser.py

@@ -0,0 +1,32 @@
+# Generated by Django 4.2.4 on 2023-09-18 23:24
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("database", "0001_khojuser"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="GoogleUser",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("sub", models.CharField(max_length=200)),
+                ("azp", models.CharField(max_length=200)),
+                ("email", models.CharField(max_length=200)),
+                ("name", models.CharField(max_length=200)),
+                ("given_name", models.CharField(max_length=200)),
+                ("family_name", models.CharField(max_length=200)),
+                ("picture", models.CharField(max_length=200)),
+                ("locale", models.CharField(max_length=200)),
+                (
+                    "user",
+                    models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
+                ),
+            ],
+        ),
+    ]

khoj/database/migrations/0003_vector_extension.py

@@ -0,0 +1,10 @@
+from django.db import migrations
+from pgvector.django import VectorExtension
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("database", "0002_googleuser"),
+    ]
+
+    operations = [VectorExtension()]

khoj/database/migrations/0004_content_types_and_more.py

@@ -0,0 +1,181 @@
+# Generated by Django 4.2.5 on 2023-10-11 22:24
+
+import uuid
+
+import django.db.models.deletion
+import pgvector.django
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("database", "0003_vector_extension"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="GithubConfig",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                ("updated_at", models.DateTimeField(auto_now=True)),
+                ("pat_token", models.CharField(max_length=200)),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+        migrations.AddField(
+            model_name="khojuser",
+            name="uuid",
+            field=models.UUIDField(default=1234, verbose_name=models.UUIDField(default=uuid.uuid4, editable=False)),
+            preserve_default=False,
+        ),
+        migrations.CreateModel(
+            name="NotionConfig",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                ("updated_at", models.DateTimeField(auto_now=True)),
+                ("token", models.CharField(max_length=200)),
+                ("user", models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+        migrations.CreateModel(
+            name="LocalPlaintextConfig",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                ("updated_at", models.DateTimeField(auto_now=True)),
+                ("input_files", models.JSONField(default=list, null=True)),
+                ("input_filter", models.JSONField(default=list, null=True)),
+                ("index_heading_entries", models.BooleanField(default=False)),
+                ("user", models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+        migrations.CreateModel(
+            name="LocalPdfConfig",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                ("updated_at", models.DateTimeField(auto_now=True)),
+                ("input_files", models.JSONField(default=list, null=True)),
+                ("input_filter", models.JSONField(default=list, null=True)),
+                ("index_heading_entries", models.BooleanField(default=False)),
+                ("user", models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+        migrations.CreateModel(
+            name="LocalOrgConfig",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                ("updated_at", models.DateTimeField(auto_now=True)),
+                ("input_files", models.JSONField(default=list, null=True)),
+                ("input_filter", models.JSONField(default=list, null=True)),
+                ("index_heading_entries", models.BooleanField(default=False)),
+                ("user", models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+        migrations.CreateModel(
+            name="LocalMarkdownConfig",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                ("updated_at", models.DateTimeField(auto_now=True)),
+                ("input_files", models.JSONField(default=list, null=True)),
+                ("input_filter", models.JSONField(default=list, null=True)),
+                ("index_heading_entries", models.BooleanField(default=False)),
+                ("user", models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+        migrations.CreateModel(
+            name="GithubRepoConfig",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                ("updated_at", models.DateTimeField(auto_now=True)),
+                ("name", models.CharField(max_length=200)),
+                ("owner", models.CharField(max_length=200)),
+                ("branch", models.CharField(max_length=200)),
+                (
+                    "github_config",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="githubrepoconfig",
+                        to="database.githubconfig",
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+        migrations.AddField(
+            model_name="githubconfig",
+            name="user",
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.CreateModel(
+            name="Embeddings",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                ("updated_at", models.DateTimeField(auto_now=True)),
+                ("embeddings", pgvector.django.VectorField(dimensions=384)),
+                ("raw", models.TextField()),
+                ("compiled", models.TextField()),
+                ("heading", models.CharField(blank=True, default=None, max_length=1000, null=True)),
+                (
+                    "file_type",
+                    models.CharField(
+                        choices=[
+                            ("image", "Image"),
+                            ("pdf", "Pdf"),
+                            ("plaintext", "Plaintext"),
+                            ("markdown", "Markdown"),
+                            ("org", "Org"),
+                            ("notion", "Notion"),
+                            ("github", "Github"),
+                            ("conversation", "Conversation"),
+                        ],
+                        default="plaintext",
+                        max_length=30,
+                    ),
+                ),
+                ("file_path", models.CharField(blank=True, default=None, max_length=400, null=True)),
+                ("file_name", models.CharField(blank=True, default=None, max_length=400, null=True)),
+                ("url", models.URLField(blank=True, default=None, max_length=400, null=True)),
+                ("hashed_value", models.CharField(max_length=100)),
+                (
+                    "user",
+                    models.ForeignKey(
+                        blank=True,
+                        default=None,
+                        null=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+    ]

khoj/database/migrations/0005_embeddings_corpus_id.py

@@ -0,0 +1,19 @@
+# Generated by Django 4.2.5 on 2023-10-13 02:39
+
+import uuid
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("database", "0004_content_types_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="embeddings",
+            name="corpus_id",
+            field=models.UUIDField(default=uuid.uuid4, editable=False),
+        ),
+    ]

khoj/database/migrations/0006_embeddingsdates.py

@@ -0,0 +1,33 @@
+# Generated by Django 4.2.5 on 2023-10-13 19:28
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("database", "0005_embeddings_corpus_id"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="EmbeddingsDates",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                ("updated_at", models.DateTimeField(auto_now=True)),
+                ("date", models.DateField()),
+                (
+                    "embeddings",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="embeddings_dates",
+                        to="database.embeddings",
+                    ),
+                ),
+            ],
+            options={
+                "indexes": [models.Index(fields=["date"], name="database_em_date_a1ba47_idx")],
+            },
+        ),
+    ]

khoj/database/migrations/0007_add_conversation.py

@@ -0,0 +1,27 @@
+# Generated by Django 4.2.5 on 2023-10-18 05:31
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("database", "0006_embeddingsdates"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="Conversation",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                ("updated_at", models.DateTimeField(auto_now=True)),
+                ("conversation_log", models.JSONField()),
+                ("user", models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+    ]

khoj/database/migrations/0008_alter_conversation_conversation_log.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.2.5 on 2023-10-18 16:46
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("database", "0007_add_conversation"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="conversation",
+            name="conversation_log",
+            field=models.JSONField(default=dict),
+        ),
+    ]

khoj/database/migrations/0009_khojapiuser.py

@@ -0,0 +1,24 @@
+# Generated by Django 4.2.5 on 2023-10-26 17:02
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("database", "0008_alter_conversation_conversation_log"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="KhojApiUser",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("token", models.CharField(max_length=50, unique=True)),
+                ("name", models.CharField(max_length=50)),
+                ("accessed_at", models.DateTimeField(default=None, null=True)),
+                ("user", models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+    ]

khoj/database/migrations/0010_chatmodeloptions_and_more.py

@@ -0,0 +1,83 @@
+# Generated by Django 4.2.4 on 2023-11-01 17:41
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("database", "0009_khojapiuser"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="ChatModelOptions",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                ("updated_at", models.DateTimeField(auto_now=True)),
+                ("max_prompt_size", models.IntegerField(blank=True, default=None, null=True)),
+                ("tokenizer", models.CharField(blank=True, default=None, max_length=200, null=True)),
+                ("chat_model", models.CharField(blank=True, default=None, max_length=200, null=True)),
+                (
+                    "model_type",
+                    models.CharField(
+                        choices=[("openai", "Openai"), ("offline", "Offline")], default="openai", max_length=200
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+        migrations.CreateModel(
+            name="OfflineChatProcessorConversationConfig",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                ("updated_at", models.DateTimeField(auto_now=True)),
+                ("enabled", models.BooleanField(default=False)),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+        migrations.CreateModel(
+            name="OpenAIProcessorConversationConfig",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                ("updated_at", models.DateTimeField(auto_now=True)),
+                ("api_key", models.CharField(max_length=200)),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+        migrations.CreateModel(
+            name="UserConversationConfig",
+            fields=[
+                ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("created_at", models.DateTimeField(auto_now_add=True)),
+                ("updated_at", models.DateTimeField(auto_now=True)),
+                (
+                    "setting",
+                    models.ForeignKey(
+                        blank=True,
+                        default=None,
+                        null=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="database.chatmodeloptions",
+                    ),
+                ),
+                (
+                    "user",
+                    models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+    ]

khoj/database/migrations/0010_rename_embeddings_entry_and_more.py

@@ -0,0 +1,30 @@
+# Generated by Django 4.2.5 on 2023-10-26 23:52
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("database", "0009_khojapiuser"),
+    ]
+
+    operations = [
+        migrations.RenameModel(
+            old_name="Embeddings",
+            new_name="Entry",
+        ),
+        migrations.RenameModel(
+            old_name="EmbeddingsDates",
+            new_name="EntryDates",
+        ),
+        migrations.RenameField(
+            model_name="entrydates",
+            old_name="embeddings",
+            new_name="entry",
+        ),
+        migrations.RenameIndex(
+            model_name="entrydates",
+            new_name="database_en_date_8d823c_idx",
+            old_name="database_em_date_a1ba47_idx",
+        ),
+    ]