khoj 1.16.1.dev15__py3-none-any.whl

khoj/processor/conversation/utils.py

@@ -0,0 +1,262 @@
+import json
+import logging
+import math
+import queue
+from datetime import datetime
+from time import perf_counter
+from typing import Any, Dict, List, Optional
+
+import tiktoken
+from langchain.schema import ChatMessage
+from llama_cpp.llama import Llama
+from transformers import AutoTokenizer
+
+from khoj.database.adapters import ConversationAdapters
+from khoj.database.models import ClientApplication, KhojUser
+from khoj.processor.conversation.offline.utils import download_model, infer_max_tokens
+from khoj.utils import state
+from khoj.utils.helpers import is_none_or_empty, merge_dicts
+
+logger = logging.getLogger(__name__)
+model_to_prompt_size = {
+    "gpt-3.5-turbo": 12000,
+    "gpt-3.5-turbo-0125": 12000,
+    "gpt-4-0125-preview": 20000,
+    "gpt-4-turbo-preview": 20000,
+    "TheBloke/Mistral-7B-Instruct-v0.2-GGUF": 3500,
+    "NousResearch/Hermes-2-Pro-Mistral-7B-GGUF": 3500,
+}
+model_to_tokenizer: Dict[str, str] = {}
+
+
+class ThreadedGenerator:
+    def __init__(self, compiled_references, online_results, completion_func=None):
+        self.queue = queue.Queue()
+        self.compiled_references = compiled_references
+        self.online_results = online_results
+        self.completion_func = completion_func
+        self.response = ""
+        self.start_time = perf_counter()
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        item = self.queue.get()
+        if item is StopIteration:
+            time_to_response = perf_counter() - self.start_time
+            logger.info(f"Chat streaming took: {time_to_response:.3f} seconds")
+            if self.completion_func:
+                # The completion func effectively acts as a callback.
+                # It adds the aggregated response to the conversation history.
+                self.completion_func(chat_response=self.response)
+            raise StopIteration
+        return item
+
+    def send(self, data):
+        if self.response == "":
+            time_to_first_response = perf_counter() - self.start_time
+            logger.info(f"First response took: {time_to_first_response:.3f} seconds")
+
+        self.response += data
+        self.queue.put(data)
+
+    def close(self):
+        if self.compiled_references and len(self.compiled_references) > 0:
+            self.queue.put(f"### compiled references:{json.dumps(self.compiled_references)}")
+        if self.online_results and len(self.online_results) > 0:
+            self.queue.put(f"### compiled references:{json.dumps(self.online_results)}")
+        self.queue.put(StopIteration)
+
+
+def message_to_log(
+    user_message, chat_response, user_message_metadata={}, khoj_message_metadata={}, conversation_log=[]
+):
+    """Create json logs from messages, metadata for conversation log"""
+    default_khoj_message_metadata = {
+        "intent": {"type": "remember", "memory-type": "notes", "query": user_message},
+        "trigger-emotion": "calm",
+    }
+    khoj_response_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+
+    # Create json log from Human's message
+    human_log = merge_dicts({"message": user_message, "by": "you"}, user_message_metadata)
+
+    # Create json log from GPT's response
+    khoj_log = merge_dicts(khoj_message_metadata, default_khoj_message_metadata)
+    khoj_log = merge_dicts({"message": chat_response, "by": "khoj", "created": khoj_response_time}, khoj_log)
+
+    conversation_log.extend([human_log, khoj_log])
+    return conversation_log
+
+
+def save_to_conversation_log(
+    q: str,
+    chat_response: str,
+    user: KhojUser,
+    meta_log: Dict,
+    user_message_time: str = None,
+    compiled_references: List[Dict[str, Any]] = [],
+    online_results: Dict[str, Any] = {},
+    inferred_queries: List[str] = [],
+    intent_type: str = "remember",
+    client_application: ClientApplication = None,
+    conversation_id: int = None,
+    automation_id: str = None,
+):
+    user_message_time = user_message_time or datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    updated_conversation = message_to_log(
+        user_message=q,
+        chat_response=chat_response,
+        user_message_metadata={"created": user_message_time},
+        khoj_message_metadata={
+            "context": compiled_references,
+            "intent": {"inferred-queries": inferred_queries, "type": intent_type},
+            "onlineContext": online_results,
+            "automationId": automation_id,
+        },
+        conversation_log=meta_log.get("chat", []),
+    )
+    ConversationAdapters.save_conversation(
+        user,
+        {"chat": updated_conversation},
+        client_application=client_application,
+        conversation_id=conversation_id,
+        user_message=q,
+    )
+
+    logger.info(
+        f"""
+Saved Conversation Turn
+You ({user.username}): "{q}"
+
+Khoj: "{inferred_queries if ("text-to-image" in intent_type) else chat_response}"
+""".strip()
+    )
+
+
+def generate_chatml_messages_with_context(
+    user_message,
+    system_message=None,
+    conversation_log={},
+    model_name="gpt-3.5-turbo",
+    loaded_model: Optional[Llama] = None,
+    max_prompt_size=None,
+    tokenizer_name=None,
+):
+    """Generate messages for ChatGPT with context from previous conversation"""
+    # Set max prompt size from user config or based on pre-configured for model and machine specs
+    if not max_prompt_size:
+        if loaded_model:
+            max_prompt_size = infer_max_tokens(loaded_model.n_ctx(), model_to_prompt_size.get(model_name, math.inf))
+        else:
+            max_prompt_size = model_to_prompt_size.get(model_name, 2000)
+
+    # Scale lookback turns proportional to max prompt size supported by model
+    lookback_turns = max_prompt_size // 750
+
+    # Extract Chat History for Context
+    chat_logs = []
+    for chat in conversation_log.get("chat", []):
+        chat_notes = f'\n\n Notes:\n{chat.get("context")}' if chat.get("context") else "\n"
+        chat_logs += [chat["message"] + chat_notes]
+
+    rest_backnforths: List[ChatMessage] = []
+    # Extract in reverse chronological order
+    for user_msg, assistant_msg in zip(chat_logs[-2::-2], chat_logs[::-2]):
+        if len(rest_backnforths) >= 2 * lookback_turns:
+            break
+        rest_backnforths += reciprocal_conversation_to_chatml([user_msg, assistant_msg])[::-1]
+
+    # Format user and system messages to chatml format
+    messages = []
+    if not is_none_or_empty(user_message):
+        messages.append(ChatMessage(content=user_message, role="user"))
+    if len(rest_backnforths) > 0:
+        messages += rest_backnforths
+    if not is_none_or_empty(system_message):
+        messages.append(ChatMessage(content=system_message, role="system"))
+
+    # Truncate oldest messages from conversation history until under max supported prompt size by model
+    messages = truncate_messages(messages, max_prompt_size, model_name, loaded_model, tokenizer_name)
+
+    # Return message in chronological order
+    return messages[::-1]
+
+
+def truncate_messages(
+    messages: list[ChatMessage],
+    max_prompt_size,
+    model_name: str,
+    loaded_model: Optional[Llama] = None,
+    tokenizer_name=None,
+) -> list[ChatMessage]:
+    """Truncate messages to fit within max prompt size supported by model"""
+
+    default_tokenizer = "hf-internal-testing/llama-tokenizer"
+
+    try:
+        if loaded_model:
+            encoder = loaded_model.tokenizer()
+        elif model_name.startswith("gpt-"):
+            encoder = tiktoken.encoding_for_model(model_name)
+        elif tokenizer_name:
+            if tokenizer_name in state.pretrained_tokenizers:
+                encoder = state.pretrained_tokenizers[tokenizer_name]
+            else:
+                encoder = AutoTokenizer.from_pretrained(tokenizer_name)
+                state.pretrained_tokenizers[tokenizer_name] = encoder
+        else:
+            encoder = download_model(model_name).tokenizer()
+    except:
+        if default_tokenizer in state.pretrained_tokenizers:
+            encoder = state.pretrained_tokenizers[default_tokenizer]
+        else:
+            encoder = AutoTokenizer.from_pretrained(default_tokenizer)
+            state.pretrained_tokenizers[default_tokenizer] = encoder
+        logger.warning(
+            f"Fallback to default chat model tokenizer: {tokenizer_name}.\nConfigure tokenizer for unsupported model: {model_name} in Khoj settings to improve context stuffing."
+        )
+
+    # Extract system message from messages
+    system_message = None
+    for idx, message in enumerate(messages):
+        if message.role == "system":
+            system_message = messages.pop(idx)
+            break
+
+    system_message_tokens = (
+        len(encoder.encode(system_message.content)) if system_message and type(system_message.content) == str else 0
+    )
+
+    tokens = sum([len(encoder.encode(message.content)) for message in messages if type(message.content) == str])
+
+    # Drop older messages until under max supported prompt size by model
+    while (tokens + system_message_tokens) > max_prompt_size and len(messages) > 1:
+        messages.pop()
+        tokens = sum([len(encoder.encode(message.content)) for message in messages if type(message.content) == str])
+
+    # Truncate current message if still over max supported prompt size by model
+    if (tokens + system_message_tokens) > max_prompt_size:
+        current_message = "\n".join(messages[0].content.split("\n")[:-1]) if type(messages[0].content) == str else ""
+        original_question = "\n".join(messages[0].content.split("\n")[-1:]) if type(messages[0].content) == str else ""
+        original_question = f"\n{original_question}"
+        original_question_tokens = len(encoder.encode(original_question))
+        remaining_tokens = max_prompt_size - system_message_tokens
+        if remaining_tokens > original_question_tokens:
+            remaining_tokens -= original_question_tokens
+            truncated_message = encoder.decode(encoder.encode(current_message)[:remaining_tokens]).strip()
+            messages = [ChatMessage(content=truncated_message + original_question, role=messages[0].role)]
+        else:
+            truncated_message = encoder.decode(encoder.encode(original_question)[:remaining_tokens]).strip()
+            messages = [ChatMessage(content=truncated_message, role=messages[0].role)]
+        logger.debug(
+            f"Truncate current message to fit within max prompt size of {max_prompt_size} supported by {model_name} model:\n {truncated_message}"
+        )
+
+    return messages + [system_message] if system_message else messages
+
+
+def reciprocal_conversation_to_chatml(message_pair):
+    """Convert a single back and forth between user and assistant to chatml format"""
+    return [ChatMessage(content=message, role=role) for message, role in zip(message_pair, ["user", "assistant"])]

khoj/processor/embeddings.py

@@ -0,0 +1,117 @@
+import logging
+from typing import List
+
+import requests
+import tqdm
+from sentence_transformers import CrossEncoder, SentenceTransformer
+from tenacity import (
+    before_sleep_log,
+    retry,
+    retry_if_exception_type,
+    stop_after_attempt,
+    wait_random_exponential,
+)
+from torch import nn
+
+from khoj.utils.helpers import get_device, merge_dicts
+from khoj.utils.rawconfig import SearchResponse
+
+logger = logging.getLogger(__name__)
+
+
+class EmbeddingsModel:
+    def __init__(
+        self,
+        model_name: str = "thenlper/gte-small",
+        embeddings_inference_endpoint: str = None,
+        embeddings_inference_endpoint_api_key: str = None,
+        query_encode_kwargs: dict = {},
+        docs_encode_kwargs: dict = {},
+        model_kwargs: dict = {},
+    ):
+        default_query_encode_kwargs = {"show_progress_bar": False, "normalize_embeddings": True}
+        default_docs_encode_kwargs = {"show_progress_bar": True, "normalize_embeddings": True}
+        self.query_encode_kwargs = merge_dicts(query_encode_kwargs, default_query_encode_kwargs)
+        self.docs_encode_kwargs = merge_dicts(docs_encode_kwargs, default_docs_encode_kwargs)
+        self.model_kwargs = merge_dicts(model_kwargs, {"device": get_device()})
+        self.model_name = model_name
+        self.inference_endpoint = embeddings_inference_endpoint
+        self.api_key = embeddings_inference_endpoint_api_key
+        self.embeddings_model = SentenceTransformer(self.model_name, **self.model_kwargs)
+
+    def inference_server_enabled(self) -> bool:
+        return self.api_key is not None and self.inference_endpoint is not None
+
+    def embed_query(self, query):
+        if self.inference_server_enabled():
+            return self.embed_with_api([query])[0]
+        return self.embeddings_model.encode([query], **self.query_encode_kwargs)[0]
+
+    @retry(
+        retry=retry_if_exception_type(requests.exceptions.HTTPError),
+        wait=wait_random_exponential(multiplier=1, max=10),
+        stop=stop_after_attempt(5),
+        before_sleep=before_sleep_log(logger, logging.DEBUG),
+    )
+    def embed_with_api(self, docs):
+        payload = {"inputs": docs}
+        headers = {
+            "Authorization": f"Bearer {self.api_key}",
+            "Content-Type": "application/json",
+        }
+        try:
+            response = requests.post(self.inference_endpoint, json=payload, headers=headers)
+            response.raise_for_status()
+        except requests.exceptions.HTTPError as e:
+            logger.error(
+                f" Error while calling inference endpoint {self.inference_endpoint} with error {e}, response {response.json()} ",
+                exc_info=True,
+            )
+            raise e
+        return response.json()["embeddings"]
+
+    def embed_documents(self, docs):
+        if self.inference_server_enabled():
+            if "huggingface" not in self.inference_endpoint:
+                logger.warning(
+                    f"Unsupported inference endpoint: {self.inference_endpoint}. Only HuggingFace supported. Generating embeddings on device instead."
+                )
+                return self.embeddings_model.encode(docs, **self.docs_encode_kwargs).tolist()
+            # break up the docs payload in chunks of 1000 to avoid hitting rate limits
+            embeddings = []
+            with tqdm.tqdm(total=len(docs)) as pbar:
+                for i in range(0, len(docs), 1000):
+                    docs_to_embed = docs[i : i + 1000]
+                    generated_embeddings = self.embed_with_api(docs_to_embed)
+                    embeddings += generated_embeddings
+                    pbar.update(1000)
+            return embeddings
+        return self.embeddings_model.encode(docs, **self.docs_encode_kwargs).tolist() if docs else []
+
+
+class CrossEncoderModel:
+    def __init__(
+        self,
+        model_name: str = "mixedbread-ai/mxbai-rerank-xsmall-v1",
+        cross_encoder_inference_endpoint: str = None,
+        cross_encoder_inference_endpoint_api_key: str = None,
+    ):
+        self.model_name = model_name
+        self.cross_encoder_model = CrossEncoder(model_name=self.model_name, device=get_device())
+        self.inference_endpoint = cross_encoder_inference_endpoint
+        self.api_key = cross_encoder_inference_endpoint_api_key
+
+    def inference_server_enabled(self) -> bool:
+        return self.api_key is not None and self.inference_endpoint is not None
+
+    def predict(self, query, hits: List[SearchResponse], key: str = "compiled"):
+        if self.inference_server_enabled() and "huggingface" in self.inference_endpoint:
+            target_url = f"{self.inference_endpoint}"
+            payload = {"inputs": {"query": query, "passages": [hit.additional[key] for hit in hits]}}
+            headers = {"Authorization": f"Bearer {self.api_key}", "Content-Type": "application/json"}
+            response = requests.post(target_url, json=payload, headers=headers)
+            return response.json()["scores"]
+
+        cross_inp = [[query, hit.additional[key]] for hit in hits]
+        cross_scores = self.cross_encoder_model.predict(cross_inp, activation_fct=nn.Sigmoid())
+        return cross_scores

khoj/processor/speech/text_to_speech.py

@@ -0,0 +1,51 @@
+import json  # Used for working with JSON data
+import os
+
+import requests  # Used for making HTTP requests
+from bs4 import BeautifulSoup
+from markdown_it import MarkdownIt
+
+# Define constants for the script
+CHUNK_SIZE = 1024  # Size of chunks to read/write at a time
+ELEVEN_LABS_API_KEY = os.getenv("ELEVEN_LABS_API_KEY", None)  # Your API key for authentication
+VOICE_ID = "RPEIZnKMqlQiZyZd1Dae"  # ID of the voice model to use. MALE - Christopher - friendly guy next door.
+ELEVEN_API_URL = "https://api.elevenlabs.io/v1/text-to-speech"  # Base URL for the Text-to-Speech API
+
+markdown_renderer = MarkdownIt()
+
+
+def is_eleven_labs_enabled():
+    return ELEVEN_LABS_API_KEY is not None
+
+
+def generate_text_to_speech(
+    text_to_speak: str,
+    voice_id: str = VOICE_ID,
+):
+    if not is_eleven_labs_enabled():
+        return "Eleven Labs API key is not set"
+
+    # Convert the incoming text from markdown format to plain text
+    html = markdown_renderer.render(text_to_speak)
+    text = "".join(BeautifulSoup(html, features="lxml").findAll(text=True))
+
+    # Construct the URL for the Text-to-Speech API request
+    tts_url = f"{ELEVEN_API_URL}/{voice_id}/stream"
+
+    # Set up headers for the API request, including the API key for authentication
+    headers = {"Accept": "application/json", "xi-api-key": ELEVEN_LABS_API_KEY}
+
+    # Set up the data payload for the API request, including the text and voice settings
+    data = {
+        "text": text,
+        # "model_id": "eleven_multilingual_v2",
+        "voice_settings": {"stability": 0.5, "similarity_boost": 0.8, "style": 0.0, "use_speaker_boost": True},
+    }
+
+    # Make the POST request to the TTS API with headers and data, enabling streaming response
+    response = requests.post(tts_url, headers=headers, json=data, stream=True)
+
+    if response.ok:
+        return response
+    else:
+        raise Exception(f"Failed to generate text-to-speech: {response.text}")

khoj/processor/tools/online_search.py

@@ -0,0 +1,225 @@
+import asyncio
+import json
+import logging
+import os
+import urllib.parse
+from collections import defaultdict
+from typing import Callable, Dict, List, Optional, Tuple, Union
+
+import aiohttp
+from bs4 import BeautifulSoup
+from markdownify import markdownify
+
+from khoj.routers.helpers import (
+    extract_relevant_info,
+    generate_online_subqueries,
+    infer_webpage_urls,
+)
+from khoj.utils.helpers import is_internet_connected, is_none_or_empty, timer
+from khoj.utils.rawconfig import LocationData
+
+logger = logging.getLogger(__name__)
+
+SERPER_DEV_API_KEY = os.getenv("SERPER_DEV_API_KEY")
+SERPER_DEV_URL = "https://google.serper.dev/search"
+
+JINA_READER_API_URL = "https://r.jina.ai/"
+JINA_SEARCH_API_URL = "https://s.jina.ai/"
+JINA_API_KEY = os.getenv("JINA_API_KEY")
+
+OLOSTEP_API_KEY = os.getenv("OLOSTEP_API_KEY")
+OLOSTEP_API_URL = "https://agent.olostep.com/olostep-p2p-incomingAPI"
+OLOSTEP_QUERY_PARAMS = {
+    "timeout": 35,  # seconds
+    "waitBeforeScraping": 1,  # seconds
+    "saveHtml": "False",
+    "saveMarkdown": "True",
+    "removeCSSselectors": "default",
+    "htmlTransformer": "none",
+    "removeImages": "True",
+    "fastLane": "True",
+    # Similar to Stripe's API, the expand parameters avoid the need to make a second API call
+    # to retrieve the dataset (from the dataset API) if you only need the markdown or html.
+    "expandMarkdown": "True",
+    "expandHtml": "False",
+}
+MAX_WEBPAGES_TO_READ = 1
+
+
+async def search_online(
+    query: str,
+    conversation_history: dict,
+    location: LocationData,
+    send_status_func: Optional[Callable] = None,
+    custom_filters: List[str] = [],
+):
+    query += " ".join(custom_filters)
+    if not is_internet_connected():
+        logger.warn("Cannot search online as not connected to internet")
+        return {}
+
+    # Breakdown the query into subqueries to get the correct answer
+    subqueries = await generate_online_subqueries(query, conversation_history, location)
+    response_dict = {}
+
+    if subqueries:
+        logger.info(f"🌐 Searching the Internet for {list(subqueries)}")
+        if send_status_func:
+            subqueries_str = "\n- " + "\n- ".join(list(subqueries))
+            await send_status_func(f"**🌐 Searching the Internet for**: {subqueries_str}")
+
+    with timer(f"Internet searches for {list(subqueries)} took", logger):
+        search_func = search_with_google if SERPER_DEV_API_KEY else search_with_jina
+        search_tasks = [search_func(subquery) for subquery in subqueries]
+        search_results = await asyncio.gather(*search_tasks)
+        response_dict = {subquery: search_result for subquery, search_result in search_results}
+
+    # Gather distinct web page data from organic results of each subquery without an instant answer.
+    # Content of web pages is directly available when Jina is used for search.
+    webpages = {
+        (organic.get("link"), subquery, organic.get("content"))
+        for subquery in response_dict
+        for organic in response_dict[subquery].get("organic", [])[:MAX_WEBPAGES_TO_READ]
+        if "answerBox" not in response_dict[subquery]
+    }
+
+    # Read, extract relevant info from the retrieved web pages
+    if webpages:
+        webpage_links = [link for link, _, _ in webpages]
+        logger.info(f"🌐👀 Reading web pages at: {list(webpage_links)}")
+        if send_status_func:
+            webpage_links_str = "\n- " + "\n- ".join(list(webpage_links))
+            await send_status_func(f"**📖 Reading web pages**: {webpage_links_str}")
+    tasks = [read_webpage_and_extract_content(subquery, link, content) for link, subquery, content in webpages]
+    results = await asyncio.gather(*tasks)
+
+    # Collect extracted info from the retrieved web pages
+    for subquery, webpage_extract, url in results:
+        if webpage_extract is not None:
+            response_dict[subquery]["webpages"] = {"link": url, "snippet": webpage_extract}
+
+    return response_dict
+
+
+async def search_with_google(query: str) -> Tuple[str, Dict[str, List[Dict]]]:
+    payload = json.dumps({"q": query})
+    headers = {"X-API-KEY": SERPER_DEV_API_KEY, "Content-Type": "application/json"}
+
+    async with aiohttp.ClientSession() as session:
+        async with session.post(SERPER_DEV_URL, headers=headers, data=payload) as response:
+            if response.status != 200:
+                logger.error(await response.text())
+                return query, {}
+            json_response = await response.json()
+            extraction_fields = ["organic", "answerBox", "peopleAlsoAsk", "knowledgeGraph"]
+            extracted_search_result = {
+                field: json_response[field]
+                for field in extraction_fields
+                if not is_none_or_empty(json_response.get(field))
+            }
+
+            return query, extracted_search_result
+
+
+async def read_webpages(
+    query: str, conversation_history: dict, location: LocationData, send_status_func: Optional[Callable] = None
+):
+    "Infer web pages to read from the query and extract relevant information from them"
+    logger.info(f"Inferring web pages to read")
+    if send_status_func:
+        await send_status_func(f"**🧐 Inferring web pages to read**")
+    urls = await infer_webpage_urls(query, conversation_history, location)
+
+    logger.info(f"Reading web pages at: {urls}")
+    if send_status_func:
+        webpage_links_str = "\n- " + "\n- ".join(list(urls))
+        await send_status_func(f"**📖 Reading web pages**: {webpage_links_str}")
+    tasks = [read_webpage_and_extract_content(query, url) for url in urls]
+    results = await asyncio.gather(*tasks)
+
+    response: Dict[str, Dict] = defaultdict(dict)
+    response[query]["webpages"] = [
+        {"query": q, "link": url, "snippet": web_extract} for q, web_extract, url in results if web_extract is not None
+    ]
+    return response
+
+
+async def read_webpage_and_extract_content(
+    subquery: str, url: str, content: str = None
+) -> Tuple[str, Union[None, str], str]:
+    try:
+        if is_none_or_empty(content):
+            with timer(f"Reading web page at '{url}' took", logger):
+                content = await read_webpage_with_olostep(url) if OLOSTEP_API_KEY else await read_webpage_with_jina(url)
+        with timer(f"Extracting relevant information from web page at '{url}' took", logger):
+            extracted_info = await extract_relevant_info(subquery, content)
+        return subquery, extracted_info, url
+    except Exception as e:
+        logger.error(f"Failed to read web page at '{url}' with {e}")
+        return subquery, None, url
+
+
+async def read_webpage_at_url(web_url: str) -> str:
+    headers = {
+        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36",
+    }
+
+    async with aiohttp.ClientSession() as session:
+        async with session.get(web_url, headers=headers, timeout=30) as response:
+            response.raise_for_status()
+            html = await response.text()
+            parsed_html = BeautifulSoup(html, "html.parser")
+            body = parsed_html.body.get_text(separator="\n", strip=True)
+            return markdownify(body)
+
+
+async def read_webpage_with_olostep(web_url: str) -> str:
+    headers = {"Authorization": f"Bearer {OLOSTEP_API_KEY}"}
+    web_scraping_params: Dict[str, Union[str, int, bool]] = OLOSTEP_QUERY_PARAMS.copy()  # type: ignore
+    web_scraping_params["url"] = web_url
+
+    async with aiohttp.ClientSession() as session:
+        async with session.get(OLOSTEP_API_URL, params=web_scraping_params, headers=headers) as response:
+            response.raise_for_status()
+            response_json = await response.json()
+            return response_json["markdown_content"]
+
+
+async def read_webpage_with_jina(web_url: str) -> str:
+    jina_reader_api_url = f"{JINA_READER_API_URL}/{web_url}"
+    headers = {"Accept": "application/json", "X-Timeout": "30"}
+    if JINA_API_KEY:
+        headers["Authorization"] = f"Bearer {JINA_API_KEY}"
+
+    async with aiohttp.ClientSession() as session:
+        async with session.get(jina_reader_api_url, headers=headers) as response:
+            response.raise_for_status()
+            response_json = await response.json()
+            return response_json["data"]["content"]
+
+
+async def search_with_jina(query: str) -> Tuple[str, Dict[str, List[Dict]]]:
+    encoded_query = urllib.parse.quote(query)
+    jina_search_api_url = f"{JINA_SEARCH_API_URL}/{encoded_query}"
+    headers = {"Accept": "application/json"}
+    if JINA_API_KEY:
+        headers["Authorization"] = f"Bearer {JINA_API_KEY}"
+
+    async with aiohttp.ClientSession() as session:
+        async with session.get(jina_search_api_url, headers=headers) as response:
+            if response.status != 200:
+                logger.error(await response.text())
+                return query, {}
+            response_json = await response.json()
+            parsed_response = [
+                {
+                    "title": item["title"],
+                    "content": item.get("content"),
+                    # rename description -> snippet for consistency
+                    "snippet": item["description"],
+                    # rename url -> link for consistency
+                    "link": item["url"],
+                }
+                for item in response_json["data"]
+            ]
+            return query, {"organic": parsed_response}