khoj 2.0.0b12.dev5__py3-none-any.whl → 2.0.0b13__py3-none-any.whl

khoj/interface/compiled/_next/static/chunks/webpack-338a5000c912cc94.js

@@ -1 +0,0 @@
-!function(){"use strict";var e,t,n,r,c,o,f,a,u,i={},d={};function b(e){var t=d[e];if(void 0!==t)return t.exports;var n=d[e]={exports:{}},r=!0;try{i[e].call(n.exports,n,n.exports,b),r=!1}finally{r&&delete d[e]}return n.exports}b.m=i,e=[],b.O=function(t,n,r,c){if(n){c=c||0;for(var o=e.length;o>0&&e[o-1][2]>c;o--)e[o]=e[o-1];e[o]=[n,r,c];return}for(var f=1/0,o=0;o<e.length;o++){for(var n=e[o][0],r=e[o][1],c=e[o][2],a=!0,u=0;u<n.length;u++)f>=c&&Object.keys(b.O).every(function(e){return b.O[e](n[u])})?n.splice(u--,1):(a=!1,c<f&&(f=c));if(a){e.splice(o--,1);var i=r();void 0!==i&&(t=i)}}return t},b.n=function(e){var t=e&&e.__esModule?function(){return e.default}:function(){return e};return b.d(t,{a:t}),t},n=Object.getPrototypeOf?function(e){return Object.getPrototypeOf(e)}:function(e){return e.__proto__},b.t=function(e,r){if(1&r&&(e=this(e)),8&r||"object"==typeof e&&e&&(4&r&&e.__esModule||16&r&&"function"==typeof e.then))return e;var c=Object.create(null);b.r(c);var o={};t=t||[null,n({}),n([]),n(n)];for(var f=2&r&&e;"object"==typeof f&&!~t.indexOf(f);f=n(f))Object.getOwnPropertyNames(f).forEach(function(t){o[t]=function(){return e[t]}});return o.default=function(){return e},b.d(c,o),c},b.d=function(e,t){for(var n in t)b.o(t,n)&&!b.o(e,n)&&Object.defineProperty(e,n,{enumerable:!0,get:t[n]})},b.f={},b.e=function(e){return Promise.all(Object.keys(b.f).reduce(function(t,n){return b.f[n](e,t),t},[]))},b.u=function(e){return 4609===e?"static/chunks/4609-33aa487dff03a9fd.js":"static/chunks/"+(({1264:"90542734",6555:"964ecbae",7293:"94ca1967"})[e]||e)+"."+({83:"48e2db193a940052",303:"fe76de943e930fbd",872:"caf84cc1a39ae59f",1191:"b547ec13349b4aed",1264:"2c21f16f18b22411",1588:"f0558a0bdffc4761",1918:"925cb4a35518d258",2242:"9a07e19f1a3a8b16",2612:"bcf5a623b3da209e",2849:"dc00ae5ba7219cfc",3265:"924139c4146ee344",3489:"c523fe96a2eee74f",4327:"8d2a1b8f1ea78208",4533:"586e74b45a2bde25",4551:"82ce1476b5516bc2",4610:"196691887afb7fea",4748:"0edd37cba3ea2809",4980:"63500d68b3bb1222",5210:"cd35a1c1ec594a20",5329:"f8b3c5b3d16159cd",5830:"8876eccb82da9b7d",6230:"88a71d8145347b3f",6434:"e6cd986d690f2cef",6555:"d5be7c49c320d695",7161:"77e0530a40ad5ca8",7293:"1b3402358e0e1255",7303:"d0612f812a967a08",7505:"c31027a3695bdebb",7760:"35649cc21d9585bd",8427:"844694e06133fb51",8665:"4db7e6b2e8933497",8694:"2bd9c2f65d8c5847",8888:"ebe0e552b59e7fed",8890:"6e8a59e4de6978bc",8950:"5f2272e0ac923f9e",9202:"c703864fcedc8d1f",9245:"a04e92d034540234",9320:"6aca4885d541aa44",9535:"f78cd92d03331e55",9968:"b111fc002796da81"})[e]+".js"},b.miniCssF=function(e){},b.g=function(){if("object"==typeof globalThis)return globalThis;try{return this||Function("return this")()}catch(e){if("object"==typeof window)return window}}(),b.o=function(e,t){return Object.prototype.hasOwnProperty.call(e,t)},r={},c="_N_E:",b.l=function(e,t,n,o){if(r[e]){r[e].push(t);return}if(void 0!==n)for(var f,a,u=document.getElementsByTagName("script"),i=0;i<u.length;i++){var d=u[i];if(d.getAttribute("src")==e||d.getAttribute("data-webpack")==c+n){f=d;break}}f||(a=!0,(f=document.createElement("script")).charset="utf-8",f.timeout=120,b.nc&&f.setAttribute("nonce",b.nc),f.setAttribute("data-webpack",c+n),f.src=b.tu(e)),r[e]=[t];var l=function(t,n){f.onerror=f.onload=null,clearTimeout(s);var c=r[e];if(delete r[e],f.parentNode&&f.parentNode.removeChild(f),c&&c.forEach(function(e){return e(n)}),t)return t(n)},s=setTimeout(l.bind(null,void 0,{type:"timeout",target:f}),12e4);f.onerror=l.bind(null,f.onerror),f.onload=l.bind(null,f.onload),a&&document.head.appendChild(f)},b.r=function(e){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},b.tt=function(){return void 0===o&&(o={createScriptURL:function(e){return e}},"undefined"!=typeof trustedTypes&&trustedTypes.createPolicy&&(o=trustedTypes.createPolicy("nextjs#bundler",o))),o},b.tu=function(e){return b.tt().createScriptURL(e)},b.p="/_next/",f={2272:0,3254:0,3587:0,7400:0,6250:0,4842:0,5638:0,2026:0,9346:0,6008:0},b.f.j=function(e,t){var n=b.o(f,e)?f[e]:void 0;if(0!==n){if(n)t.push(n[2]);else if(/^(2026|2272|3254|3587|4842|5638|6008|6250|7400|9346)$/.test(e))f[e]=0;else{var r=new Promise(function(t,r){n=f[e]=[t,r]});t.push(n[2]=r);var c=b.p+b.u(e),o=Error();b.l(c,function(t){if(b.o(f,e)&&(0!==(n=f[e])&&(f[e]=void 0),n)){var r=t&&("load"===t.type?"missing":t.type),c=t&&t.target&&t.target.src;o.message="Loading chunk "+e+" failed.\n("+r+": "+c+")",o.name="ChunkLoadError",o.type=r,o.request=c,n[1](o)}},"chunk-"+e,e)}}},b.O.j=function(e){return 0===f[e]},a=function(e,t){var n,r,c=t[0],o=t[1],a=t[2],u=0;if(c.some(function(e){return 0!==f[e]})){for(n in o)b.o(o,n)&&(b.m[n]=o[n]);if(a)var i=a(b)}for(e&&e(t);u<c.length;u++)r=c[u],b.o(f,r)&&f[r]&&f[r][0](),f[r]=0;return b.O(i)},(u=self.webpackChunk_N_E=self.webpackChunk_N_E||[]).forEach(a.bind(null,0)),u.push=a.bind(null,u.push.bind(u)),b.nc=void 0}();

khoj/migrations/migrate_offline_chat_default_model.py

@@ -1,69 +0,0 @@
-"""
-Current format of khoj.yml
----
-app:
-    ...
-content-type:
-    ...
-processor:
-  conversation:
-    offline-chat:
-        enable-offline-chat: false
-        chat-model: llama-2-7b-chat.ggmlv3.q4_0.bin
-    ...
-search-type:
-    ...
-
-New format of khoj.yml
----
-app:
-    ...
-content-type:
-    ...
-processor:
-  conversation:
-    offline-chat:
-        enable-offline-chat: false
-        chat-model: mistral-7b-instruct-v0.1.Q4_0.gguf
-    ...
-search-type:
-    ...
-"""
-import logging
-
-from packaging import version
-
-from khoj.utils.yaml import load_config_from_file, save_config_to_file
-
-logger = logging.getLogger(__name__)
-
-
-def migrate_offline_chat_default_model(args):
-    schema_version = "0.12.4"
-    raw_config = load_config_from_file(args.config_file)
-    previous_version = raw_config.get("version")
-
-    if "processor" not in raw_config:
-        return args
-    if raw_config["processor"] is None:
-        return args
-    if "conversation" not in raw_config["processor"]:
-        return args
-    if "offline-chat" not in raw_config["processor"]["conversation"]:
-        return args
-    if "chat-model" not in raw_config["processor"]["conversation"]["offline-chat"]:
-        return args
-
-    if previous_version is None or version.parse(previous_version) < version.parse("0.12.4"):
-        logger.info(
-            f"Upgrading config schema to {schema_version} from {previous_version} to change default (offline) chat model to mistral GGUF"
-        )
-        raw_config["version"] = schema_version
-
-        # Update offline chat model to mistral in GGUF format to use latest GPT4All
-        offline_chat_model = raw_config["processor"]["conversation"]["offline-chat"]["chat-model"]
-        if offline_chat_model.endswith(".bin"):
-            raw_config["processor"]["conversation"]["offline-chat"]["chat-model"] = "mistral-7b-instruct-v0.1.Q4_0.gguf"
-
-        save_config_to_file(raw_config, args.config_file)
-    return args

khoj/migrations/migrate_offline_chat_default_model_2.py

@@ -1,71 +0,0 @@
-"""
-Current format of khoj.yml
----
-app:
-    ...
-content-type:
-    ...
-processor:
-  conversation:
-    offline-chat:
-        enable-offline-chat: false
-        chat-model: mistral-7b-instruct-v0.1.Q4_0.gguf
-    ...
-search-type:
-    ...
-
-New format of khoj.yml
----
-app:
-    ...
-content-type:
-    ...
-processor:
-  conversation:
-    offline-chat:
-        enable-offline-chat: false
-        chat-model: NousResearch/Hermes-2-Pro-Mistral-7B-GGUF
-    ...
-search-type:
-    ...
-"""
-import logging
-
-from packaging import version
-
-from khoj.utils.yaml import load_config_from_file, save_config_to_file
-
-logger = logging.getLogger(__name__)
-
-
-def migrate_offline_chat_default_model(args):
-    schema_version = "1.7.0"
-    raw_config = load_config_from_file(args.config_file)
-    previous_version = raw_config.get("version")
-
-    if "processor" not in raw_config:
-        return args
-    if raw_config["processor"] is None:
-        return args
-    if "conversation" not in raw_config["processor"]:
-        return args
-    if "offline-chat" not in raw_config["processor"]["conversation"]:
-        return args
-    if "chat-model" not in raw_config["processor"]["conversation"]["offline-chat"]:
-        return args
-
-    if previous_version is None or version.parse(previous_version) < version.parse(schema_version):
-        logger.info(
-            f"Upgrading config schema to {schema_version} from {previous_version} to change default (offline) chat model to mistral GGUF"
-        )
-        raw_config["version"] = schema_version
-
-        # Update offline chat model to use Nous Research's Hermes-2-Pro GGUF in path format suitable for llama-cpp
-        offline_chat_model = raw_config["processor"]["conversation"]["offline-chat"]["chat-model"]
-        if offline_chat_model == "mistral-7b-instruct-v0.1.Q4_0.gguf":
-            raw_config["processor"]["conversation"]["offline-chat"][
-                "chat-model"
-            ] = "NousResearch/Hermes-2-Pro-Mistral-7B-GGUF"
-
-        save_config_to_file(raw_config, args.config_file)
-    return args

khoj/migrations/migrate_offline_chat_schema.py

@@ -1,83 +0,0 @@
-"""
-Current format of khoj.yml
----
-app:
-    ...
-content-type:
-    ...
-processor:
-  conversation:
-    enable-offline-chat: false
-    conversation-logfile: ~/.khoj/processor/conversation/conversation_logs.json
-    openai:
-        ...
-search-type:
-    ...
-
-New format of khoj.yml
----
-app:
-    ...
-content-type:
-    ...
-processor:
-  conversation:
-    offline-chat:
-        enable-offline-chat: false
-        chat-model: llama-2-7b-chat.ggmlv3.q4_0.bin
-    tokenizer: null
-    max_prompt_size: null
-    conversation-logfile: ~/.khoj/processor/conversation/conversation_logs.json
-    openai:
-        ...
-search-type:
-    ...
-"""
-import logging
-
-from packaging import version
-
-from khoj.utils.yaml import load_config_from_file, save_config_to_file
-
-logger = logging.getLogger(__name__)
-
-
-def migrate_offline_chat_schema(args):
-    schema_version = "0.12.3"
-    raw_config = load_config_from_file(args.config_file)
-    previous_version = raw_config.get("version")
-
-    if "processor" not in raw_config:
-        return args
-    if raw_config["processor"] is None:
-        return args
-    if "conversation" not in raw_config["processor"]:
-        return args
-
-    if previous_version is None or version.parse(previous_version) < version.parse("0.12.3"):
-        logger.info(
-            f"Upgrading config schema to {schema_version} from {previous_version} to make (offline) chat more configuration"
-        )
-        raw_config["version"] = schema_version
-
-        # Create max-prompt-size field in conversation processor schema
-        raw_config["processor"]["conversation"]["max-prompt-size"] = None
-        raw_config["processor"]["conversation"]["tokenizer"] = None
-
-        # Create offline chat schema based on existing enable_offline_chat field in khoj config schema
-        offline_chat_model = (
-            raw_config["processor"]["conversation"]
-            .get("offline-chat", {})
-            .get("chat-model", "llama-2-7b-chat.ggmlv3.q4_0.bin")
-        )
-        raw_config["processor"]["conversation"]["offline-chat"] = {
-            "enable-offline-chat": raw_config["processor"]["conversation"].get("enable-offline-chat", False),
-            "chat-model": offline_chat_model,
-        }
-
-        # Delete old enable-offline-chat field from conversation processor schema
-        if "enable-offline-chat" in raw_config["processor"]["conversation"]:
-            del raw_config["processor"]["conversation"]["enable-offline-chat"]
-
-        save_config_to_file(raw_config, args.config_file)
-    return args

khoj/migrations/migrate_offline_model.py

@@ -1,29 +0,0 @@
-import logging
-import os
-
-from packaging import version
-
-from khoj.utils.yaml import load_config_from_file, save_config_to_file
-
-logger = logging.getLogger(__name__)
-
-
-def migrate_offline_model(args):
-    schema_version = "0.10.1"
-    raw_config = load_config_from_file(args.config_file)
-    previous_version = raw_config.get("version")
-
-    if previous_version is None or version.parse(previous_version) < version.parse("0.10.1"):
-        logger.info(
-            f"Migrating offline model used for version {previous_version} to latest version for {args.version_no}"
-        )
-        raw_config["version"] = schema_version
-
-        # If the user has downloaded the offline model, remove it from the cache.
-        offline_model_path = os.path.expanduser("~/.cache/gpt4all/llama-2-7b-chat.ggmlv3.q4_K_S.bin")
-        if os.path.exists(offline_model_path):
-            os.remove(offline_model_path)
-
-        save_config_to_file(raw_config, args.config_file)
-
-    return args

khoj/migrations/migrate_processor_config_openai.py

@@ -1,67 +0,0 @@
-"""
-Current format of khoj.yml
----
-app:
-  should-log-telemetry: true
-content-type:
-    ...
-processor:
-  conversation:
-    chat-model: gpt-3.5-turbo
-    conversation-logfile: ~/.khoj/processor/conversation/conversation_logs.json
-    model: text-davinci-003
-    openai-api-key: sk-secret-key
-search-type:
-    ...
-
-New format of khoj.yml
----
-app:
-  should-log-telemetry: true
-content-type:
-    ...
-processor:
-  conversation:
-    openai:
-        chat-model: gpt-3.5-turbo
-        openai-api-key: sk-secret-key
-    conversation-logfile: ~/.khoj/processor/conversation/conversation_logs.json
-    enable-offline-chat: false
-search-type:
-    ...
-"""
-from khoj.utils.yaml import load_config_from_file, save_config_to_file
-
-
-def migrate_processor_conversation_schema(args):
-    schema_version = "0.10.0"
-    raw_config = load_config_from_file(args.config_file)
-
-    if "processor" not in raw_config:
-        return args
-    if raw_config["processor"] is None:
-        return args
-    if "conversation" not in raw_config["processor"]:
-        return args
-
-    current_openai_api_key = raw_config["processor"]["conversation"].get("openai-api-key", None)
-    current_chat_model = raw_config["processor"]["conversation"].get("chat-model", None)
-    if current_openai_api_key is None and current_chat_model is None:
-        return args
-
-    raw_config["version"] = schema_version
-
-    # Add enable_offline_chat to khoj config schema
-    if "enable-offline-chat" not in raw_config["processor"]["conversation"]:
-        raw_config["processor"]["conversation"]["enable-offline-chat"] = False
-
-    # Update conversation processor schema
-    conversation_logfile = raw_config["processor"]["conversation"].get("conversation-logfile", None)
-    raw_config["processor"]["conversation"] = {
-        "openai": {"chat-model": current_chat_model, "api-key": current_openai_api_key},
-        "conversation-logfile": conversation_logfile,
-        "enable-offline-chat": False,
-    }
-
-    save_config_to_file(raw_config, args.config_file)
-    return args

khoj/migrations/migrate_server_pg.py

@@ -1,132 +0,0 @@
-"""
-The application config currently looks like this:
-app:
-  should-log-telemetry: true
-content-type:
-    ...
-processor:
-  conversation:
-    conversation-logfile: ~/.khoj/processor/conversation/conversation_logs.json
-    max-prompt-size: null
-    offline-chat:
-      chat-model: mistral-7b-instruct-v0.1.Q4_0.gguf
-      enable-offline-chat: false
-    openai:
-      api-key: sk-blah
-      chat-model: gpt-3.5-turbo
-    tokenizer: null
-search-type:
-  asymmetric:
-    cross-encoder: cross-encoder/ms-marco-MiniLM-L-6-v2
-    encoder: sentence-transformers/multi-qa-MiniLM-L6-cos-v1
-    encoder-type: null
-    model-directory: /Users/si/.khoj/search/asymmetric
-  image:
-    encoder: sentence-transformers/clip-ViT-B-32
-    encoder-type: null
-    model-directory: /Users/si/.khoj/search/image
-  symmetric:
-    cross-encoder: cross-encoder/ms-marco-MiniLM-L-6-v2
-    encoder: sentence-transformers/all-MiniLM-L6-v2
-    encoder-type: null
-    model-directory: ~/.khoj/search/symmetric
-version: 0.14.0
-
-
-The new version will looks like this:
-app:
-  should-log-telemetry: true
-processor:
-  conversation:
-    offline-chat:
-      enabled: false
-    openai:
-      api-key: sk-blah
-    chat-model-options:
-      - chat-model: gpt-3.5-turbo
-        tokenizer: null
-        type: openai
-      - chat-model: mistral-7b-instruct-v0.1.Q4_0.gguf
-        tokenizer: null
-        type: offline
-search-type:
-  asymmetric:
-    cross-encoder: cross-encoder/ms-marco-MiniLM-L-6-v2
-    encoder: sentence-transformers/multi-qa-MiniLM-L6-cos-v1
-version: 0.15.0
-"""
-
-import logging
-
-from packaging import version
-
-from khoj.database.models import AiModelApi, ChatModel, SearchModelConfig
-from khoj.utils.yaml import load_config_from_file, save_config_to_file
-
-logger = logging.getLogger(__name__)
-
-
-def migrate_server_pg(args):
-    schema_version = "0.15.0"
-    raw_config = load_config_from_file(args.config_file)
-    previous_version = raw_config.get("version")
-
-    if previous_version is None or version.parse(previous_version) < version.parse(schema_version):
-        logger.info(
-            f"Migrating configuration used for version {previous_version} to latest version for server with postgres in {args.version_no}"
-        )
-        raw_config["version"] = schema_version
-
-        if raw_config is None:
-            return args
-
-        if "search-type" in raw_config and raw_config["search-type"]:
-            if "asymmetric" in raw_config["search-type"]:
-                # Delete all existing search models
-                SearchModelConfig.objects.filter(model_type=SearchModelConfig.ModelType.TEXT).delete()
-                # Create new search model from existing Khoj YAML config
-                asymmetric_search = raw_config["search-type"]["asymmetric"]
-                SearchModelConfig.objects.create(
-                    name="default",
-                    model_type=SearchModelConfig.ModelType.TEXT,
-                    bi_encoder=asymmetric_search.get("encoder"),
-                    cross_encoder=asymmetric_search.get("cross-encoder"),
-                )
-
-        if "processor" in raw_config and raw_config["processor"] and "conversation" in raw_config["processor"]:
-            processor_conversation = raw_config["processor"]["conversation"]
-
-            if "offline-chat" in raw_config["processor"]["conversation"]:
-                offline_chat = raw_config["processor"]["conversation"]["offline-chat"]
-                ChatModel.objects.create(
-                    name=offline_chat.get("chat-model"),
-                    tokenizer=processor_conversation.get("tokenizer"),
-                    max_prompt_size=processor_conversation.get("max-prompt-size"),
-                    model_type=ChatModel.ModelType.OFFLINE,
-                )
-
-            if (
-                "openai" in raw_config["processor"]["conversation"]
-                and raw_config["processor"]["conversation"]["openai"]
-            ):
-                openai = raw_config["processor"]["conversation"]["openai"]
-
-                if openai.get("api-key") is None:
-                    logger.error("OpenAI API Key is not set. Will not be migrating OpenAI config.")
-                else:
-                    if openai.get("chat-model") is None:
-                        openai["chat-model"] = "gpt-3.5-turbo"
-
-                    openai_model_api = AiModelApi.objects.create(api_key=openai.get("api-key"), name="default")
-
-                    ChatModel.objects.create(
-                        name=openai.get("chat-model"),
-                        tokenizer=processor_conversation.get("tokenizer"),
-                        max_prompt_size=processor_conversation.get("max-prompt-size"),
-                        model_type=ChatModel.ModelType.OPENAI,
-                        ai_model_api=openai_model_api,
-                    )
-
-        save_config_to_file(raw_config, args.config_file)
-
-    return args

khoj/migrations/migrate_version.py

@@ -1,17 +0,0 @@
-from khoj.utils.yaml import load_config_from_file, save_config_to_file
-
-
-def migrate_config_to_version(args):
-    schema_version = "0.9.0"
-    raw_config = load_config_from_file(args.config_file)
-
-    # Add version to khoj config schema
-    if "version" not in raw_config:
-        raw_config["version"] = schema_version
-        save_config_to_file(raw_config, args.config_file)
-
-        # regenerate khoj index on first start of this version
-        # this should refresh index and apply index corruption fixes from #325
-        args.regenerate = True
-
-    return args