khoj 1.16.1.dev25__py3-none-any.whl → 1.17.1.dev216__py3-none-any.whl

khoj/interface/web/content_source_github_input.html

@@ -165,7 +165,7 @@
 
         // Save Github config on server
         const csrfToken = document.cookie.split('; ').find(row => row.startsWith('csrftoken'))?.split('=')[1];
-        fetch('/api/config/data/content-source/github', {
+        fetch('/api/content/github', {
             method: 'POST',
             headers: {
                 'Content-Type': 'application/json',

khoj/interface/web/content_source_notion_input.html

@@ -45,7 +45,7 @@
 
         // Save Notion config on server
         const csrfToken = document.cookie.split('; ').find(row => row.startsWith('csrftoken'))?.split('=')[1];
-        fetch('/api/config/data/content-source/notion', {
+        fetch('/api/content/notion', {
             method: 'POST',
             headers: {
                 'Content-Type': 'application/json',

khoj/interface/web/public_conversation.html

@@ -34,7 +34,7 @@ Hi, I am Khoj, your open, personal AI 👋🏽. I can:
 - 📚 Understand files you drag & drop here
 - 👩🏾‍🚀 Be tuned to your conversation needs via [agents](./agents)
 
-Get the Khoj [Desktop](https://khoj.dev/downloads), [Obsidian](https://docs.khoj.dev/clients/obsidian#setup), [Emacs](https://docs.khoj.dev/clients/emacs#setup) apps to search, chat with your 🖥️ computer docs. You can manage all the files you've shared with me at any time by going to [your settings](/config/content-source/computer/).
+Get the Khoj [Desktop](https://khoj.dev/downloads), [Obsidian](https://docs.khoj.dev/clients/obsidian#setup), [Emacs](https://docs.khoj.dev/clients/emacs#setup) apps to search, chat with your 🖥️ computer docs. You can manage all the files you've shared with me at any time by going to [your settings](/settings/content/computer/).
 
 To get started, just start typing below. You can also type / to see a list of commands.
 `.trim()

khoj/interface/web/search.html

@@ -209,12 +209,12 @@
 
         function populate_type_dropdown() {
             // Populate type dropdown field with enabled content types only
-            fetch("/api/config/types")
+            fetch("/api/content/types")
                 .then(response => response.json())
                 .then(enabled_types => {
                     // Show warning if no content types are enabled, or just one ("all")
                     if (enabled_types[0] === "all" && enabled_types.length === 1) {
-                        document.getElementById("results").innerHTML = "<div id='results-error'>To use Khoj search, setup your content plugins on the Khoj <a class='inline-chat-link' href='/config'>settings page</a>.</div>";
+                        document.getElementById("results").innerHTML = "<div id='results-error'>To use Khoj search, setup your content plugins on the Khoj <a class='inline-chat-link' href='/settings'>settings page</a>.</div>";
                         document.getElementById("query").setAttribute("disabled", "disabled");
                         document.getElementById("query").setAttribute("placeholder", "Configure Khoj to enable search");
                         return [];

khoj/interface/web/{config.html → settings.html}

@@ -34,7 +34,7 @@
                     <h3 id="card-title-computer" class="card-title">
                         <span>Files</span>
                         <img id="configured-icon-computer"
-                            style="display: {% if not current_model_state.computer %}none{% endif %}"
+                            style="display: {% if not enabled_content_source.computer %}none{% endif %}"
                             class="configured-icon"
                             src="/static/assets/icons/confirm-icon.svg"
                             alt="Configured">
@@ -44,8 +44,8 @@
                 <p class="card-description">Manage files from your computer</p>
                 </div>
                 <div class="card-action-row">
-                    <a class="card-button" href="/config/content-source/computer">
-                        {% if current_model_state.computer %}
+                    <a class="card-button" href="/settings/content/computer">
+                        {% if enabled_content_source.computer %}
                             Update
                         {% else %}
                             Setup
@@ -53,7 +53,7 @@
                         <svg xmlns="http://www.w3.org/2000/svg" width="1em" height="1em" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M5 12h14M12 5l7 7-7 7"></path></svg>
                     </a>
                     <div id="clear-computer" class="card-action-row"
-                        style="display: {% if not current_model_state.computer %}none{% endif %}">
+                        style="display: {% if not enabled_content_source.computer %}none{% endif %}">
                         <button class="card-button" onclick="clearContentType('computer')">
                             Disable
                         </button>
@@ -69,15 +69,15 @@
                             class="configured-icon"
                             src="/static/assets/icons/confirm-icon.svg"
                             alt="Configured"
-                            style="display: {% if not current_model_state.github %}none{% endif %}">
+                            style="display: {% if not enabled_content_source.github %}none{% endif %}">
                     </h3>
                 </div>
                 <div class="card-description-row">
                 <p class="card-description">Set repositories to index</p>
                 </div>
                 <div class="card-action-row">
-                    <a class="card-button" href="/config/content-source/github">
-                        {% if current_model_state.github %}
+                    <a class="card-button" href="/settings/content/github">
+                        {% if enabled_content_source.github %}
                             Update
                         {% else %}
                             Setup
@@ -86,7 +86,7 @@
                     </a>
                     <div id="clear-github"
                         class="card-action-row"
-                        style="display: {% if not current_model_state.github %}none{% endif %}">
+                        style="display: {% if not enabled_content_source.github %}none{% endif %}">
                         <button class="card-button" onclick="clearContentType('github')">
                             Disable
                         </button>
@@ -102,15 +102,15 @@
                             class="configured-icon"
                             src="/static/assets/icons/confirm-icon.svg"
                             alt="Configured"
-                            style="display: {% if not current_model_state.notion %}none{% endif %}">
+                            style="display: {% if not enabled_content_source.notion %}none{% endif %}">
                     </h3>
                 </div>
                 <div class="card-description-row">
                     <p class="card-description">Sync your Notion pages</p>
                 </div>
                 <div class="card-action-row">
-                    {% if current_model_state.notion %}
-                    <a class="card-button" href="/config/content-source/notion">
+                    {% if enabled_content_source.notion %}
+                    <a class="card-button" href="/settings/content/notion">
                             Update
                         <svg xmlns="http://www.w3.org/2000/svg" width="1em" height="1em" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M5 12h14M12 5l7 7-7 7"></path></svg>
                     </a>
@@ -120,7 +120,7 @@
                         <svg xmlns="http://www.w3.org/2000/svg" width="1em" height="1em" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M5 12h14M12 5l7 7-7 7"></path></svg>
                     </a>
                     {% else %}
-                    <a class="card-button" href="/config/content-source/notion">
+                    <a class="card-button" href="/settings/content/notion">
                             Setup
                         <svg xmlns="http://www.w3.org/2000/svg" width="1em" height="1em" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M5 12h14M12 5l7 7-7 7"></path></svg>
                     </a>
@@ -128,7 +128,7 @@
 
                     <div id="clear-notion"
                         class="card-action-row"
-                        style="display: {% if not current_model_state.notion %}none{% endif %}">
+                        style="display: {% if not enabled_content_source.notion %}none{% endif %}">
                         <button class="card-button" onclick="clearContentType('notion')">
                             Disable
                         </button>
@@ -181,8 +181,8 @@
                 </div>
                 <div class="card-description-row">
                     <select id="chat-models">
-                        {% for option in conversation_options %}
-                            <option value="{{ option.id }}" {% if option.id == selected_conversation_config %}selected{% endif %}>{{ option.chat_model }}</option>
+                        {% for option in chat_model_options %}
+                            <option value="{{ option.id }}" {% if option.id == selected_chat_model_config %}selected{% endif %}>{{ option.name }}</option>
                         {% endfor %}
                     </select>
                 </div>
@@ -208,7 +208,7 @@
                 <div class="card-description-row">
                     <select id="paint-models">
                         {% for option in paint_model_options %}
-                            <option value="{{ option.id }}" {% if option.id == selected_paint_model_config %}selected{% endif %}>{{ option.model_name }}</option>
+                            <option value="{{ option.id }}" {% if option.id == selected_paint_model_config %}selected{% endif %}>{{ option.name }}</option>
                         {% endfor %}
                     </select>
                 </div>
@@ -235,7 +235,7 @@
                     <div class="card-description-row">
                         <select id="voice-models">
                             {% for option in voice_model_options %}
-                                <option value="{{ option.id }}" {% if option.id == selected_voice_config %}selected{% endif %}>{{ option.name }}</option>
+                                <option value="{{ option.id }}" {% if option.id == selected_voice_model_config %}selected{% endif %}>{{ option.name }}</option>
                             {% endfor %}
                         </select>
                     </div>
@@ -394,8 +394,8 @@
 
     function saveProfileGivenName() {
         const givenName = document.getElementById("profile_given_name").value;
-        fetch('/api/config/user/name?name=' + givenName, {
-            method: 'POST',
+        fetch('/api/user/name?name=' + givenName, {
+            method: 'PATCH',
             headers: {
                 'Content-Type': 'application/json',
             }
@@ -421,7 +421,7 @@
         saveVoiceModelButton.disabled = true;
         saveVoiceModelButton.textContent = "Saving...";
 
-        fetch('/api/config/data/voice/model?id=' + voiceModel, {
+        fetch('/api/model/voice?id=' + voiceModel, {
             method: 'POST',
             headers: {
                 'Content-Type': 'application/json',
@@ -455,7 +455,7 @@
         saveModelButton.innerHTML = "";
         saveModelButton.textContent = "Saving...";
 
-        fetch('/api/config/data/conversation/model?id=' + chatModel, {
+        fetch('/api/model/chat?id=' + chatModel, {
             method: 'POST',
             headers: {
                 'Content-Type': 'application/json',
@@ -494,7 +494,7 @@
         saveSearchModelButton.disabled = true;
         saveSearchModelButton.textContent = "Saving...";
 
-        fetch('/api/config/data/search/model?id=' + searchModel, {
+        fetch('/api/model/search?id=' + searchModel, {
             method: 'POST',
             headers: {
                 'Content-Type': 'application/json',
@@ -526,7 +526,7 @@
         saveModelButton.disabled = true;
         saveModelButton.innerHTML = "Saving...";
 
-        fetch('/api/config/data/paint/model?id=' + paintModel, {
+        fetch('/api/model/paint?id=' + paintModel, {
             method: 'POST',
             headers: {
                 'Content-Type': 'application/json',
@@ -553,7 +553,7 @@
     };
 
     function clearContentType(content_source) {
-        fetch('/api/config/data/content-source/' + content_source, {
+        fetch('/api/content/' + content_source, {
             method: 'DELETE',
             headers: {
                 'Content-Type': 'application/json',
@@ -676,7 +676,7 @@
 
         content_sources = ["computer", "github", "notion"];
         content_sources.forEach(content_source => {
-            fetch(`/api/config/data/${content_source}`, {
+            fetch(`/api/content/${content_source}`, {
                 method: 'GET',
                 headers: {
                     'Content-Type': 'application/json',
@@ -807,7 +807,7 @@
 
     function getIndexedDataSize() {
         document.getElementById("indexed-data-size").textContent = "Calculating...";
-        fetch('/api/config/index/size')
+        fetch('/api/content/size')
             .then(response => response.json())
             .then(data => {
                 document.getElementById("indexed-data-size").textContent = data.indexed_data_size_in_mb + " MB used";
@@ -815,7 +815,7 @@
     }
 
     function removeFile(path) {
-        fetch('/api/config/data/file?filename=' + path, {
+        fetch('/api/content/file?filename=' + path, {
             method: 'DELETE',
             headers: {
                 'Content-Type': 'application/json',
@@ -890,7 +890,7 @@
     })
 
     phonenumberRemoveButton.addEventListener("click", () => {
-        fetch('/api/config/phone', {
+        fetch('/api/phone', {
             method: 'DELETE',
             headers: {
                 'Content-Type': 'application/json',
@@ -917,7 +917,7 @@
             }, 5000);
         } else {
             const mobileNumber = iti.getNumber();
-            fetch('/api/config/phone?phone_number=' + mobileNumber, {
+            fetch('/api/phone?phone_number=' + mobileNumber, {
                 method: 'POST',
                 headers: {
                     'Content-Type': 'application/json',
@@ -970,7 +970,7 @@
             return;
         }
 
-        fetch('/api/config/phone/verify?code=' + otp, {
+        fetch('/api/phone/verify?code=' + otp, {
             method: 'POST',
             headers: {
                 'Content-Type': 'application/json',

khoj/interface/web/utils.html

@@ -36,7 +36,7 @@
                     {% endif %}
                     <div id="khoj-nav-menu" class="khoj-nav-dropdown-content">
                         <div class="khoj-nav-username"> {{ username }} </div>
-                        <a id="settings-nav" class="khoj-nav" href="/config">Settings</a>
+                        <a id="settings-nav" class="khoj-nav" href="/settings">Settings</a>
                         <a id="github-nav" class="khoj-nav" href="https://github.com/khoj-ai/khoj">GitHub</a>
                         <a id="help-nav" class="khoj-nav" href="https://docs.khoj.dev" target="_blank">Help</a>
                         <a class="khoj-nav" href="/auth/logout">Logout</a>

khoj/processor/content/docx/docx_to_entries.py

@@ -19,16 +19,11 @@ class DocxToEntries(TextToEntries):
         super().__init__()
 
     # Define Functions
-    def process(
-        self, files: dict[str, str] = None, full_corpus: bool = True, user: KhojUser = None, regenerate: bool = False
-    ) -> Tuple[int, int]:
+    def process(self, files: dict[str, str] = None, user: KhojUser = None, regenerate: bool = False) -> Tuple[int, int]:
         # Extract required fields from config
-        if not full_corpus:
-            deletion_file_names = set([file for file in files if files[file] == b""])
-            files_to_process = set(files) - deletion_file_names
-            files = {file: files[file] for file in files_to_process}
-        else:
-            deletion_file_names = None
+        deletion_file_names = set([file for file in files if files[file] == b""])
+        files_to_process = set(files) - deletion_file_names
+        files = {file: files[file] for file in files_to_process}
 
         # Extract Entries from specified Docx files
         with timer("Extract entries from specified DOCX files", logger):

khoj/processor/content/github/github_to_entries.py

@@ -48,9 +48,7 @@ class GithubToEntries(TextToEntries):
         else:
             return
 
-    def process(
-        self, files: dict[str, str] = None, full_corpus: bool = True, user: KhojUser = None, regenerate: bool = False
-    ) -> Tuple[int, int]:
+    def process(self, files: dict[str, str] = None, user: KhojUser = None, regenerate: bool = False) -> Tuple[int, int]:
         if self.config.pat_token is None or self.config.pat_token == "":
             logger.error(f"Github PAT token is not set. Skipping github content")
             raise ValueError("Github PAT token is not set. Skipping github content")

khoj/processor/content/images/image_to_entries.py

@@ -20,16 +20,11 @@ class ImageToEntries(TextToEntries):
         super().__init__()
 
     # Define Functions
-    def process(
-        self, files: dict[str, str] = None, full_corpus: bool = True, user: KhojUser = None, regenerate: bool = False
-    ) -> Tuple[int, int]:
+    def process(self, files: dict[str, str] = None, user: KhojUser = None, regenerate: bool = False) -> Tuple[int, int]:
         # Extract required fields from config
-        if not full_corpus:
-            deletion_file_names = set([file for file in files if files[file] == b""])
-            files_to_process = set(files) - deletion_file_names
-            files = {file: files[file] for file in files_to_process}
-        else:
-            deletion_file_names = None
+        deletion_file_names = set([file for file in files if files[file] == b""])
+        files_to_process = set(files) - deletion_file_names
+        files = {file: files[file] for file in files_to_process}
 
         # Extract Entries from specified image files
         with timer("Extract entries from specified Image files", logger):

khoj/processor/content/markdown/markdown_to_entries.py

@@ -19,16 +19,11 @@ class MarkdownToEntries(TextToEntries):
         super().__init__()
 
     # Define Functions
-    def process(
-        self, files: dict[str, str] = None, full_corpus: bool = True, user: KhojUser = None, regenerate: bool = False
-    ) -> Tuple[int, int]:
+    def process(self, files: dict[str, str] = None, user: KhojUser = None, regenerate: bool = False) -> Tuple[int, int]:
         # Extract required fields from config
-        if not full_corpus:
-            deletion_file_names = set([file for file in files if files[file] == ""])
-            files_to_process = set(files) - deletion_file_names
-            files = {file: files[file] for file in files_to_process}
-        else:
-            deletion_file_names = None
+        deletion_file_names = set([file for file in files if files[file] == ""])
+        files_to_process = set(files) - deletion_file_names
+        files = {file: files[file] for file in files_to_process}
 
         max_tokens = 256
         # Extract Entries from specified Markdown files

khoj/processor/content/notion/notion_to_entries.py

@@ -78,9 +78,7 @@ class NotionToEntries(TextToEntries):
 
         self.body_params = {"page_size": 100}
 
-    def process(
-        self, files: dict[str, str] = None, full_corpus: bool = True, user: KhojUser = None, regenerate: bool = False
-    ) -> Tuple[int, int]:
+    def process(self, files: dict[str, str] = None, user: KhojUser = None, regenerate: bool = False) -> Tuple[int, int]:
         current_entries = []
 
         # Get all pages

khoj/processor/content/org_mode/org_to_entries.py

@@ -20,15 +20,10 @@ class OrgToEntries(TextToEntries):
         super().__init__()
 
     # Define Functions
-    def process(
-        self, files: dict[str, str] = None, full_corpus: bool = True, user: KhojUser = None, regenerate: bool = False
-    ) -> Tuple[int, int]:
-        if not full_corpus:
-            deletion_file_names = set([file for file in files if files[file] == ""])
-            files_to_process = set(files) - deletion_file_names
-            files = {file: files[file] for file in files_to_process}
-        else:
-            deletion_file_names = None
+    def process(self, files: dict[str, str] = None, user: KhojUser = None, regenerate: bool = False) -> Tuple[int, int]:
+        deletion_file_names = set([file for file in files if files[file] == ""])
+        files_to_process = set(files) - deletion_file_names
+        files = {file: files[file] for file in files_to_process}
 
         # Extract Entries from specified Org files
         max_tokens = 256

khoj/processor/content/pdf/pdf_to_entries.py

@@ -22,16 +22,11 @@ class PdfToEntries(TextToEntries):
         super().__init__()
 
     # Define Functions
-    def process(
-        self, files: dict[str, str] = None, full_corpus: bool = True, user: KhojUser = None, regenerate: bool = False
-    ) -> Tuple[int, int]:
+    def process(self, files: dict[str, str] = None, user: KhojUser = None, regenerate: bool = False) -> Tuple[int, int]:
         # Extract required fields from config
-        if not full_corpus:
-            deletion_file_names = set([file for file in files if files[file] == b""])
-            files_to_process = set(files) - deletion_file_names
-            files = {file: files[file] for file in files_to_process}
-        else:
-            deletion_file_names = None
+        deletion_file_names = set([file for file in files if files[file] == b""])
+        files_to_process = set(files) - deletion_file_names
+        files = {file: files[file] for file in files_to_process}
 
         # Extract Entries from specified Pdf files
         with timer("Extract entries from specified PDF files", logger):

khoj/processor/content/plaintext/plaintext_to_entries.py

@@ -20,15 +20,10 @@ class PlaintextToEntries(TextToEntries):
         super().__init__()
 
     # Define Functions
-    def process(
-        self, files: dict[str, str] = None, full_corpus: bool = True, user: KhojUser = None, regenerate: bool = False
-    ) -> Tuple[int, int]:
-        if not full_corpus:
-            deletion_file_names = set([file for file in files if files[file] == ""])
-            files_to_process = set(files) - deletion_file_names
-            files = {file: files[file] for file in files_to_process}
-        else:
-            deletion_file_names = None
+    def process(self, files: dict[str, str] = None, user: KhojUser = None, regenerate: bool = False) -> Tuple[int, int]:
+        deletion_file_names = set([file for file in files if files[file] == ""])
+        files_to_process = set(files) - deletion_file_names
+        files = {file: files[file] for file in files_to_process}
 
         # Extract Entries from specified plaintext files
         with timer("Extract entries from specified Plaintext files", logger):

khoj/processor/content/text_to_entries.py

@@ -31,9 +31,7 @@ class TextToEntries(ABC):
         self.date_filter = DateFilter()
 
     @abstractmethod
-    def process(
-        self, files: dict[str, str] = None, full_corpus: bool = True, user: KhojUser = None, regenerate: bool = False
-    ) -> Tuple[int, int]:
+    def process(self, files: dict[str, str] = None, user: KhojUser = None, regenerate: bool = False) -> Tuple[int, int]:
         ...
 
     @staticmethod

khoj/processor/conversation/utils.py

@@ -62,10 +62,6 @@ class ThreadedGenerator:
         self.queue.put(data)
 
     def close(self):
-        if self.compiled_references and len(self.compiled_references) > 0:
-            self.queue.put(f"### compiled references:{json.dumps(self.compiled_references)}")
-        if self.online_results and len(self.online_results) > 0:
-            self.queue.put(f"### compiled references:{json.dumps(self.online_results)}")
         self.queue.put(StopIteration)
 
 

khoj/processor/tools/online_search.py

@@ -11,6 +11,7 @@ from bs4 import BeautifulSoup
 from markdownify import markdownify
 
 from khoj.routers.helpers import (
+    ChatEvent,
     extract_relevant_info,
     generate_online_subqueries,
     infer_webpage_urls,
@@ -56,7 +57,8 @@ async def search_online(
     query += " ".join(custom_filters)
     if not is_internet_connected():
         logger.warn("Cannot search online as not connected to internet")
-        return {}
+        yield {}
+        return
 
     # Breakdown the query into subqueries to get the correct answer
     subqueries = await generate_online_subqueries(query, conversation_history, location)
@@ -66,7 +68,8 @@ async def search_online(
         logger.info(f"🌐 Searching the Internet for {list(subqueries)}")
         if send_status_func:
             subqueries_str = "\n- " + "\n- ".join(list(subqueries))
-            await send_status_func(f"**🌐 Searching the Internet for**: {subqueries_str}")
+            async for event in send_status_func(f"**Searching the Internet for**: {subqueries_str}"):
+                yield {ChatEvent.STATUS: event}
 
     with timer(f"Internet searches for {list(subqueries)} took", logger):
         search_func = search_with_google if SERPER_DEV_API_KEY else search_with_jina
@@ -89,7 +92,8 @@ async def search_online(
         logger.info(f"🌐👀 Reading web pages at: {list(webpage_links)}")
         if send_status_func:
             webpage_links_str = "\n- " + "\n- ".join(list(webpage_links))
-            await send_status_func(f"**📖 Reading web pages**: {webpage_links_str}")
+            async for event in send_status_func(f"**Reading web pages**: {webpage_links_str}"):
+                yield {ChatEvent.STATUS: event}
     tasks = [read_webpage_and_extract_content(subquery, link, content) for link, subquery, content in webpages]
     results = await asyncio.gather(*tasks)
 
@@ -98,7 +102,7 @@ async def search_online(
         if webpage_extract is not None:
             response_dict[subquery]["webpages"] = {"link": url, "snippet": webpage_extract}
 
-    return response_dict
+    yield response_dict
 
 
 async def search_with_google(query: str) -> Tuple[str, Dict[str, List[Dict]]]:
@@ -127,13 +131,15 @@ async def read_webpages(
     "Infer web pages to read from the query and extract relevant information from them"
     logger.info(f"Inferring web pages to read")
     if send_status_func:
-        await send_status_func(f"**🧐 Inferring web pages to read**")
+        async for event in send_status_func(f"**Inferring web pages to read**"):
+            yield {ChatEvent.STATUS: event}
     urls = await infer_webpage_urls(query, conversation_history, location)
 
     logger.info(f"Reading web pages at: {urls}")
     if send_status_func:
         webpage_links_str = "\n- " + "\n- ".join(list(urls))
-        await send_status_func(f"**📖 Reading web pages**: {webpage_links_str}")
+        async for event in send_status_func(f"**Reading web pages**: {webpage_links_str}"):
+            yield {ChatEvent.STATUS: event}
     tasks = [read_webpage_and_extract_content(query, url) for url in urls]
     results = await asyncio.gather(*tasks)
 
@@ -141,7 +147,7 @@ async def read_webpages(
     response[query]["webpages"] = [
         {"query": q, "link": url, "snippet": web_extract} for q, web_extract, url in results if web_extract is not None
     ]
-    return response
+    yield response
 
 
 async def read_webpage_and_extract_content(