ovos-solver-bm25-plugin 0.0.0__py3-none-any.whl

ovos_bm25_solver/__init__.py

@@ -0,0 +1,4 @@
+from ovos_bm25_solver.base import BM25MultipleChoiceSolver, BM25EvidenceSolverPlugin
+from ovos_bm25_solver.corpus import BM25CorpusSolver, BM25QACorpusSolver
+from ovos_bm25_solver.embed import JsonEmbeddingsDB, BM25TextEmbeddingsStore
+from ovos_bm25_solver.demo import BM25FreebaseQASolver, BM25SquadQASolver

ovos_bm25_solver/base.py

@@ -0,0 +1,161 @@
+from typing import List, Optional, Union, Tuple
+
+from ovos_plugin_manager.templates.solvers import MultipleChoiceSolver, EvidenceSolver
+from ovos_utils.log import LOG
+from quebra_frases import sentence_tokenize
+
+from ovos_bm25_solver.corpus import BM25CorpusSolver
+
+
+class BM25MultipleChoiceSolver(MultipleChoiceSolver):
+    """Select the best answer to a question from a list of options using BM25 ranking."""
+
+    def rerank(self, query: str, options: List[str],
+               lang: Optional[str] = None,
+               return_index: bool = False) -> List[Tuple[float, Union[str, int]]]:
+        """
+        Rank the list of options based on their relevance to the query.
+
+        Args:
+            query (str): The query to rank options against.
+            options (List[str]): A list of options to be ranked.
+            lang (Optional[str]): Optional language code for translation.
+            return_index (bool): Whether to return the index of the options instead of the option text.
+
+        Returns:
+            List[Tuple[float, Union[str, int]]]: A list of tuples containing the score and either the option text or its index.
+        """
+        bm25 = BM25CorpusSolver(internal_lang=self.default_lang,
+                                translator=self.translator,
+                                detector=self.detector,
+                                enable_tx=self.enable_tx)
+        bm25.load_corpus(options)
+        ranked: List[Tuple[float, str]] = list(bm25.retrieve_from_corpus(query, lang=lang, k=len(options)))
+        if return_index:
+            ranked = [(r[0], options.index(r[1])) for r in ranked]
+        return ranked
+
+
+class BM25EvidenceSolverPlugin(EvidenceSolver):
+    """Extract the best sentence from text that answers the question using BM25 algorithm."""
+
+    def get_best_passage(self, evidence: str, question: str,
+                         lang: Optional[str] = None) -> Optional[str]:
+        """
+        Extract the most relevant passage from the evidence that answers the question.
+
+        Args:
+            evidence (str): The text to search for the answer.
+            question (str): The question to find an answer for.
+            lang (Optional[str]): Optional language code for translation.
+
+        Returns:
+            Optional[str]: The best passage that answers the question, or None if no passage is found.
+        """
+        bm25 = BM25MultipleChoiceSolver(internal_lang=self.default_lang,
+                                        translator=self.translator,
+                                        detector=self.detector,
+                                        enable_tx=self.enable_tx)
+        sents = []
+        for s in evidence.split("\n"):
+            sents += sentence_tokenize(s)
+        sents = [s.strip() for s in sents if s]
+        return bm25.select_answer(question, sents, lang=lang)
+
+
+if __name__ == "__main__":
+    from ovos_bm25_solver.demo import BM25FreebaseQASolver, BM25SquadQASolver
+
+    LOG.set_level("DEBUG")
+    p = BM25MultipleChoiceSolver()
+    a = p.rerank("what is the speed of light", [
+        "very fast", "10m/s", "the speed of light is C"
+    ], lang="en")
+    print(a)
+    # 2024-07-22 15:03:10.295 - OVOS - __main__:load_corpus:61 - DEBUG - indexed 3 documents
+    # 2024-07-22 15:03:10.297 - OVOS - __main__:retrieve_from_corpus:70 - DEBUG - Rank 1 (score: 0.7198746800422668): the speed of light is C
+    # 2024-07-22 15:03:10.297 - OVOS - __main__:retrieve_from_corpus:70 - DEBUG - Rank 2 (score: 0.0): 10m/s
+    # 2024-07-22 15:03:10.297 - OVOS - __main__:retrieve_from_corpus:70 - DEBUG - Rank 3 (score: 0.0): very fast
+    # [(0.7198747, 'the speed of light is C'), (0.0, '10m/s'), (0.0, 'very fast')]
+
+    a = p.select_answer("what is the speed of light", [
+        "very fast", "10m/s", "the speed of light is C"
+    ], lang="en")
+    print(a)  # the speed of light is C
+
+    config = {
+        "lang": "en-us",
+        "min_conf": 0.4,
+        "n_answer": 1
+    }
+    solver = BM25EvidenceSolverPlugin(config)
+
+    text = """Mars is the fourth planet from the Sun. It is a dusty, cold, desert world with a very thin atmosphere. 
+Mars is also a dynamic planet with seasons, polar ice caps, canyons, extinct volcanoes, and evidence that it was even more active in the past.
+Mars is one of the most explored bodies in our solar system, and it's the only planet where we've sent rovers to roam the alien landscape. 
+NASA currently has two rovers (Curiosity and Perseverance), one lander (InSight), and one helicopter (Ingenuity) exploring the surface of Mars.
+"""
+    query = "how many rovers are currently exploring Mars"
+    answer = solver.get_best_passage(evidence=text, question=query, lang="en")
+    print("Query:", query)
+    print("Answer:", answer)
+    # 2024-07-22 15:05:14.209 - OVOS - __main__:load_corpus:61 - DEBUG - indexed 5 documents
+    # 2024-07-22 15:05:14.209 - OVOS - __main__:retrieve_from_corpus:70 - DEBUG - Rank 1 (score: 1.39238703250885): NASA currently has two rovers (Curiosity and Perseverance), one lander (InSight), and one helicopter (Ingenuity) exploring the surface of Mars.
+    # 2024-07-22 15:05:14.210 - OVOS - __main__:retrieve_from_corpus:70 - DEBUG - Rank 2 (score: 0.38667747378349304): Mars is one of the most explored bodies in our solar system, and it's the only planet where we've sent rovers to roam the alien landscape.
+    # 2024-07-22 15:05:14.210 - OVOS - __main__:retrieve_from_corpus:70 - DEBUG - Rank 3 (score: 0.15732118487358093): Mars is the fourth planet from the Sun.
+    # 2024-07-22 15:05:14.210 - OVOS - __main__:retrieve_from_corpus:70 - DEBUG - Rank 4 (score: 0.10177625715732574): Mars is also a dynamic planet with seasons, polar ice caps, canyons, extinct volcanoes, and evidence that it was even more active in the past.
+    # 2024-07-22 15:05:14.210 - OVOS - __main__:retrieve_from_corpus:70 - DEBUG - Rank 5 (score: 0.0): It is a dusty, cold, desert world with a very thin atmosphere.
+    # Query: how many rovers are currently exploring Mars
+    # Answer: NASA currently has two rovers (Curiosity and Perseverance), one lander (InSight), and one helicopter (Ingenuity) exploring the surface of Mars.
+
+    # Create your corpus here
+    corpus = [
+        "a cat is a feline and likes to purr",
+        "a dog is the human's best friend and loves to play",
+        "a bird is a beautiful animal that can fly",
+        "a fish is a creature that lives in water and swims",
+    ]
+
+    s = BM25CorpusSolver({})
+    s.load_corpus(corpus)
+
+    query = "does the fish purr like a cat?"
+    print(s.spoken_answer(query, lang="en"))
+
+    # 2024-07-19 20:03:29.979 - OVOS - ovos_plugin_manager.utils.config:get_plugin_config:40 - DEBUG - Loaded configuration: {'module': 'ovos-translate-plugin-server', 'lang': 'en-us'}
+    # 2024-07-19 20:03:30.024 - OVOS - __main__:load_corpus:28 - DEBUG - indexed 4 documents
+    # 2024-07-19 20:03:30.025 - OVOS - __main__:retrieve_from_corpus:37 - DEBUG - Rank 1 (score: 1.0584375858306885): a cat is a feline and likes to purr
+    # 2024-07-19 20:03:30.025 - OVOS - __main__:retrieve_from_corpus:37 - DEBUG - Rank 2 (score: 0.481589138507843): a fish is a creature that lives in water and swims
+    # a cat is a feline and likes to purr. a fish is a creature that lives in water and swims
+
+    # hotpotqa dataset
+    # data = requests.get("http://curtis.ml.cmu.edu/datasets/hotpot/hotpot_dev_fullwiki_v1.json").json()
+    # data = requests.get("http://curtis.ml.cmu.edu/datasets/hotpot/hotpot_train_v1.1.json").json()
+    # for qa in data:
+    #    corpus[qa["question"]] = qa["answer"]
+    # len_hotpot = len(corpus) - len_squad - len_freebase
+    # print(len_hotpot, "qa pairs imported from hotpotqa dataset")
+
+    # s = BM25QACorpusSolver({})
+    # s.load_corpus(corpus)
+
+    s = BM25FreebaseQASolver()
+    query = "What is the capital of France"
+    print("Query:", query)
+    print("Answer:", s.spoken_answer(query, lang="en"))
+    # 2024-07-19 22:31:09.468 - OVOS - __main__:load_corpus:60 - DEBUG - indexed 20357 documents
+    # Query: What is the capital of France
+    # 2024-07-19 22:31:09.468 - OVOS - __main__:retrieve_from_corpus:69 - DEBUG - Rank 1 (score: 5.996074199676514): what is the capital of france
+    # 2024-07-19 22:31:09.469 - OVOS - __main__:retrieve_from_corpus:93 - DEBUG - closest question in corpus: what is the capital of france
+    # Answer: paris
+
+    s = BM25SquadQASolver()
+    query = "is there life on mars"
+    print("Query:", query)
+    print("Answer:", s.spoken_answer(query, lang="en"))
+    # 2024-07-19 22:31:12.625 - OVOS - __main__:load_corpus:60 - DEBUG - indexed 86769 documents
+    # 2024-07-19 22:31:12.625 - OVOS - __main__:load_squad_corpus:119 - INFO - Loaded and indexed 86769 question-answer pairs from SQuAD dataset
+    # Query: is there life on mars
+    # 2024-07-19 22:31:12.628 - OVOS - __main__:retrieve_from_corpus:69 - DEBUG - Rank 1 (score: 6.334013938903809): How is it postulated that Mars life might have evolved?
+    # 2024-07-19 22:31:12.628 - OVOS - __main__:retrieve_from_corpus:93 - DEBUG - closest question in corpus: How is it postulated that Mars life might have evolved?
+    # Answer: similar to Antarctic

ovos_bm25_solver/corpus.py

@@ -0,0 +1,139 @@
+from typing import List, Optional, Tuple, Iterable
+
+import bm25s
+from ovos_plugin_manager.templates.language import LanguageTranslator, LanguageDetector
+from ovos_plugin_manager.templates.solvers import QACorpusSolver, CorpusSolver
+from ovos_utils.log import LOG
+
+
+class BM25CorpusSolver(CorpusSolver):
+    """
+    A corpus solver that uses the BM25 algorithm for information retrieval.
+
+    Attributes:
+        METHODS (List[str]): List of supported BM25 methods.
+        IDF_METHODS (List[str]): List of supported IDF methods.
+    """
+
+    METHODS = ["robertson", "lucene", "bm25l", "bm25+", "atire", "rank-bm25", "bm25-pt"]
+    IDF_METHODS = ['robertson', 'lucene', 'atire', 'bm25l', 'bm25+']
+
+    def __init__(self, config: Optional[dict] = None,
+                 translator: Optional[LanguageTranslator] = None,
+                 detector: Optional[LanguageDetector] = None,
+                 priority: int = 50,
+                 enable_tx: bool = False,
+                 enable_cache: bool = False,
+                 internal_lang: Optional[str] = None,
+                 *args, **kwargs):
+        """
+        Initialize the BM25CorpusSolver with optional configurations.
+
+        Args:
+            config (Optional[dict]): Configuration dictionary. Default is None.
+            translator (Optional[LanguageTranslator]): Optional language translator.
+            detector (Optional[LanguageDetector]): Optional language detector.
+            priority (int): Priority level for the solver. Default is 50.
+            enable_tx (bool): Whether to enable translation. Default is False.
+            enable_cache (bool): Whether to enable caching. Default is False.
+            internal_lang (Optional[str]): Language code for internal processing. Default is None.
+            *args: Additional arguments.
+            **kwargs: Additional keyword arguments.
+        """
+        config = config or {"min_conf": 0.0,
+                            "n_answer": 1,
+                            "method": None,
+                            "idf_method": None}
+        super().__init__(config, translator, detector, priority,
+                         enable_tx, enable_cache, internal_lang,
+                         *args, **kwargs)
+        # Create the BM25 model
+        self.retriever = None
+        self.corpus = None
+
+    @property
+    def method(self) -> Optional[str]:
+        """
+        Get the BM25 method from configuration.
+
+        Returns:
+            Optional[str]: The BM25 method or None if not configured.
+        """
+        m = self.config.get("method")
+        if m is None:
+            return None
+        if m not in self.METHODS:
+            LOG.warning(f"{m} is not a valid method, choose one of {self.METHODS}")
+            m = None
+        return m
+
+    @property
+    def idf_method(self) -> Optional[str]:
+        """
+        Get the IDF method from configuration.
+
+        Returns:
+            Optional[str]: The IDF method or None if not configured. Defaults to "lucene" if invalid.
+        """
+        m = self.config.get("idf_method")
+        if m is None:
+            return None
+        if m not in self.IDF_METHODS:
+            LOG.warning(f"{m} is not a valid method, choose one of {self.IDF_METHODS}")
+            m = "lucene"
+        return m
+
+    def load_corpus(self, corpus: List[str]):
+        """
+        Load and index the given corpus using the BM25 algorithm.
+
+        Args:
+            corpus (List[str]): A list of documents to be indexed.
+        """
+        if self.method == "rank-bm25":
+            self.retriever = bm25s.BM25(method="atire", idf_method="robertson")
+        elif self.method == "bm25-pt":
+            self.retriever = bm25s.BM25(method="atire", idf_method="lucene")
+        elif self.method is not None:
+            self.retriever = bm25s.BM25(method=self.method, idf_method=self.idf_method)
+        else:
+            self.retriever = bm25s.BM25()
+        self.corpus = corpus
+        # Tokenize the corpus and only keep the ids (faster and saves memory)
+        stopwords = self.default_lang.split("-")[0]
+        if stopwords != "en":
+            stopwords = []
+
+        corpus_tokens = bm25s.tokenize(corpus, stopwords=stopwords)
+        # index the corpus
+        self.retriever.index(corpus_tokens)
+        LOG.debug(f"indexed {len(corpus)} documents")
+
+    def query(self, query: str, lang: Optional[str] = None, k: int = 3) -> Iterable[Tuple[str, float]]:
+        """
+        Query the indexed corpus and yield the top-k results.
+
+        Args:
+            query (str): The query string to search for.
+            lang (Optional[str]): Optional language code for tokenization.
+            k (int): The number of top results to return. Defaults to 3.
+
+        Yields:
+            Tuple[str, float]: Tuples containing the document ID and its score.
+        """
+        lang = lang or self.default_lang
+        stopwords = lang.split("-")[0]
+        if stopwords != "en":
+            stopwords = []
+        query_tokens = bm25s.tokenize(query, stopwords=stopwords)
+        # Get top-k results as a tuple of (doc ids, scores). Both are arrays of shape (n_queries, k)
+        results, scores = self.retriever.retrieve(query_tokens, corpus=self.corpus, k=k)
+        for i in range(results.shape[1]):
+            doc, score = results[0, i], scores[0, i]
+            yield doc, score
+
+
+class BM25QACorpusSolver(QACorpusSolver, BM25CorpusSolver):
+    """
+    A QA corpus solver that combines BM25 retrieval with question answering capabilities.
+    """

ovos_bm25_solver/demo.py

@@ -0,0 +1,113 @@
+from typing import Optional, Dict, Any
+
+import requests
+from ovos_plugin_manager.templates.language import LanguageTranslator, LanguageDetector
+from ovos_utils.log import LOG
+
+from ovos_bm25_solver.corpus import BM25QACorpusSolver
+
+
+## Demo subclasses
+class BM25SquadQASolver(BM25QACorpusSolver):
+    """
+    A QA solver that uses the BM25 algorithm and loads data from the SQuAD dataset.
+
+    Attributes:
+        internal_lang (str): Language code used internally.
+    """
+
+    def __init__(self, config: Optional[dict] = None,
+                 translator: Optional[LanguageTranslator] = None,
+                 detector: Optional[LanguageDetector] = None,
+                 priority: int = 50,
+                 enable_tx: bool = False,
+                 enable_cache: bool = False,
+                 *args, **kwargs):
+        """
+        Initialize the BM25SquadQASolver with optional configurations.
+
+        Args:
+            config (Optional[dict]): Configuration dictionary. Defaults to {"n_answer": 1}.
+            translator (Optional[LanguageTranslator]): Optional language translator.
+            detector (Optional[LanguageDetector]): Optional language detector.
+            priority (int): Priority level for the solver. Defaults to 50.
+            enable_tx (bool): Whether to enable translation. Defaults to False.
+            enable_cache (bool): Whether to enable caching. Defaults to False.
+            *args: Additional arguments.
+            **kwargs: Additional keyword arguments.
+        """
+        internal_lang = "en-us"
+        config = config or {"n_answer": 1}
+        super().__init__(config, translator, detector, priority,
+                         enable_tx, enable_cache, internal_lang,
+                         *args, **kwargs)
+
+    def load_squad_corpus(self):
+        """
+        Load and index the SQuAD dataset.
+
+        This method fetches the SQuAD dataset from the specified URL, processes it, and
+        loads it into the BM25 corpus.
+        """
+        corpus: Dict[str, str] = {}
+        response = requests.get("https://github.com/chrischute/squad/raw/master/data/train-v2.0.json")
+        data = response.json()
+        for s in data["data"]:
+            for p in s["paragraphs"]:
+                for qa in p["qas"]:
+                    if "question" in qa and qa["answers"]:
+                        corpus[qa["question"]] = qa["answers"][0]["text"]
+        self.load_corpus(corpus)
+        LOG.info(f"Loaded and indexed {len(corpus)} question-answer pairs from SQuAD dataset")
+
+
+class BM25FreebaseQASolver(BM25QACorpusSolver):
+    """
+    A QA solver that uses the BM25 algorithm and loads data from the FreebaseQA dataset.
+
+    Attributes:
+        internal_lang (str): Language code used internally.
+    """
+
+    def __init__(self, config: Optional[dict] = None,
+                 translator: Optional[LanguageTranslator] = None,
+                 detector: Optional[LanguageDetector] = None,
+                 priority: int = 50,
+                 enable_tx: bool = False,
+                 enable_cache: bool = False,
+                 *args, **kwargs):
+        """
+        Initialize the BM25FreebaseQASolver with optional configurations.
+
+        Args:
+            config (Optional[dict]): Configuration dictionary. Defaults to {"n_answer": 1}.
+            translator (Optional[LanguageTranslator]): Optional language translator.
+            detector (Optional[LanguageDetector]): Optional language detector.
+            priority (int): Priority level for the solver. Defaults to 50.
+            enable_tx (bool): Whether to enable translation. Defaults to False.
+            enable_cache (bool): Whether to enable caching. Defaults to False.
+            *args: Additional arguments.
+            **kwargs: Additional keyword arguments.
+        """
+        internal_lang = "en-us"
+        config = config or {"n_answer": 1}
+        super().__init__(config, translator, detector, priority,
+                         enable_tx, enable_cache, internal_lang,
+                         *args, **kwargs)
+        self._load_freebase_dataset()
+
+    def _load_freebase_dataset(self):
+        """
+        Load and index the FreebaseQA dataset.
+
+        This method fetches the FreebaseQA dataset from the specified URL, processes it, and
+        loads it into the BM25 corpus.
+        """
+        corpus: Dict[str, str] = {}
+        response = requests.get("https://github.com/kelvin-jiang/FreebaseQA/raw/master/FreebaseQA-train.json")
+        data = response.json()
+        for qa in data["Questions"]:
+            q = qa["ProcessedQuestion"]
+            a = qa["Parses"][0]["Answers"][0]["AnswersName"][0]
+            corpus[q] = a
+        self.load_corpus(corpus)

ovos_bm25_solver/embed.py

@@ -0,0 +1,179 @@
+from typing import List, Tuple, Union, Optional, Dict
+
+from json_database import JsonStorageXDG
+from ovos_utils.log import LOG
+from ovos_utils.parse import MatchStrategy, fuzzy_match, match_all
+
+from ovos_bm25_solver import BM25CorpusSolver
+from ovos_plugin_manager.templates.embeddings import EmbeddingsDB, TextEmbeddingsStore
+
+
+class JsonEmbeddingsDB(EmbeddingsDB):
+    """An implementation of EmbeddingsDB using JSON-based storage for embeddings."""
+
+    def __init__(self, path: str):
+        """Initialize the JsonEmbeddingsDB.
+
+        Args:
+            path (str): The path to the JSON storage.
+        """
+        super().__init__()
+        self.corpus = JsonStorageXDG(name=path, subfolder="json_fake_embeddings")
+        LOG.debug(f"JsonEmbeddingsDB index path: {self.corpus.path}")
+
+    @property
+    def documents(self) -> List[str]:
+        """List of document keys in the database.
+
+        Returns:
+            List[str]: A list of document keys.
+        """
+        return list(self.corpus.keys())
+
+    def add_embeddings(self, key: str, _, metadata: Optional[Dict[str, any]] = None) -> str:
+        """Add or update an embedding in the database.
+
+        Args:
+            key (str): The unique key for the embedding.
+            metadata (Optional[Dict[str, any]]): Optional metadata associated with the embedding.
+
+        Returns:
+            str: The key of the added or updated embedding.
+        """
+        self.corpus[key] = metadata or {}
+        self.corpus.store()
+        return key
+
+    def delete_embedding(self, key: str) -> Optional[Dict[str, any]]:
+        """Delete an embedding from the database.
+
+        Args:
+            key (str): The unique key for the embedding to delete.
+
+        Returns:
+            Optional[Dict[str, any]]: The metadata of the deleted embedding, or None if the key was not found.
+        """
+        if key in self.corpus:
+            return self.corpus.pop(key)
+
+    def get_embeddings(self, key: str) -> Optional[Dict[str, any]]:
+        """Retrieve an embedding from the database.
+
+        Args:
+            key (str): The unique key for the embedding to retrieve.
+
+        Returns:
+            Optional[Dict[str, any]]: The metadata of the embedding, or None if the key was not found.
+        """
+        return self.corpus.get(key)
+
+    def query(self, embedding: str, top_k: int = 5, return_metadata: bool = False) -> List[Tuple[str, float]]:
+        """Query the database for the closest embeddings to the given query embedding.
+
+        Args:
+            embedding (str): The query embedding to match.
+            top_k (int, optional): The number of top results to return. Defaults to 5.
+            return_metadata (bool, optional): Whether to include metadata in the results. Defaults to False.
+
+        Returns:
+            List[Tuple[str, float]]: A list of tuples containing the matched document and the similarity score.
+        """
+        matches: List[Tuple[str, float]] = match_all(embedding, self.documents,
+                                                     strategy=MatchStrategy.DAMERAU_LEVENSHTEIN_SIMILARITY)[:top_k]
+        if return_metadata:
+            return [(k, v, self.get_embeddings(k)) for k, v in matches]
+        return matches
+
+
+class BM25TextEmbeddingsStore(TextEmbeddingsStore):
+    """A text embeddings store using BM25 for document retrieval."""
+
+    def __init__(self, db: Union[EmbeddingsDB, str]):
+        """Initialize the BM25TextEmbeddingsStore.
+
+        Args:
+            db (Union[EmbeddingsDB, str]): An instance of EmbeddingsDB or a path to initialize JsonEmbeddingsDB.
+
+        Raises:
+            ValueError: If `db` is not an instance of JsonEmbeddingsDB.
+        """
+        if isinstance(db, str):
+            db = JsonEmbeddingsDB(path=db)
+        super().__init__(db)
+        if not isinstance(self.db, JsonEmbeddingsDB):
+            raise ValueError("'db' should be a JsonEmbeddingsDB instance")
+
+    def get_text_embeddings(self, text: str) -> str:
+        """Convert text to its corresponding embeddings.
+
+        Args:
+            text (str): The input text to be converted.
+
+        Returns:
+            str: The text itself (no transformation applied in this implementation).
+        """
+        return text
+
+    def query(self, document: str, top_k: int = 5, return_metadata: bool = False) -> List[Tuple[str, float, Optional[Dict[str, any]]]]:
+        """Query the database for the top_k closest documents to the given document.
+
+        Args:
+            document (str): The document to query.
+            top_k (int, optional): The number of top results to return. Defaults to 5.
+            return_metadata (bool, optional): Whether to include metadata in the results. Defaults to False.
+
+        Returns:
+            List[Tuple[str, float, Optional[Dict[str, any]]]]: A list of tuples containing the document, score, and optionally metadata.
+        """
+
+        bm25 = BM25CorpusSolver()
+        bm25.load_corpus(self.db.documents)
+        if return_metadata:
+            return [(txt, conf, self.db.get_embeddings(txt)) for conf, txt in
+                    bm25.retrieve_from_corpus(document, k=top_k)]
+        return [(txt, conf) for conf, txt in
+                bm25.retrieve_from_corpus(document, k=top_k)]
+
+    def distance(self, text_a: str, text_b: str,
+                 metric: MatchStrategy = MatchStrategy.DAMERAU_LEVENSHTEIN_SIMILARITY) -> float:
+        """Calculate the distance between two texts using a specified metric.
+
+        Args:
+            text_a (str): The first text.
+            text_b (str): The second text.
+            metric (MatchStrategy): The match strategy to use. Defaults to DAMERAU_LEVENSHTEIN_SIMILARITY.
+
+        Returns:
+            float: The calculated distance between the texts.
+
+        Raises:
+            ValueError: If `metric` is not an instance of MatchStrategy.
+        """
+        if not isinstance(metric, MatchStrategy):
+            raise ValueError("'metric' must be a MatchStrategy for BM25 index")
+        return fuzzy_match(text_a, text_b, strategy=metric)
+
+
+if __name__ == "__main__":
+    LOG.set_level("DEBUG")
+
+    # Initialize the database
+    db = JsonEmbeddingsDB("bm25_index")
+    index = BM25TextEmbeddingsStore(db=db)
+
+    # Add documents
+    text = "hello world"
+    text2 = "goodbye cruel world"
+    index.add_document(text)
+    index.add_document(text2)
+
+    # Query with fuzzy match
+    results = db.query("the world", top_k=2)
+    print(results)
+
+    # Query with BM25
+    results = index.query("the world", top_k=2)
+    print(results)
+
+    # Compare strings via fuzzy match - DAMERAU_LEVENSHTEIN_SIMILARITY
+    print(index.distance(text, text2))

ovos_bm25_solver/version.py

@@ -0,0 +1,6 @@
+# START_VERSION_BLOCK
+VERSION_MAJOR = 0
+VERSION_MINOR = 0
+VERSION_BUILD = 0
+VERSION_ALPHA = 0
+# END_VERSION_BLOCK

ovos_solver_bm25_plugin-0.0.0.dist-info/METADATA

@@ -0,0 +1,351 @@
+Metadata-Version: 2.1
+Name: ovos-solver-bm25-plugin
+Version: 0.0.0
+Summary: A question solver plugin for OVOS
+Home-page: https://github.com/TigreGotico/ovos-solver-BM25-plugin
+Author: jarbasai
+Author-email: jarbasai@mailfence.com
+License: MIT
+Keywords: OVOS openvoiceos plugin utterance fallback query
+Platform: UNKNOWN
+Description-Content-Type: text/markdown
+Requires-Dist: ovos-plugin-manager
+Requires-Dist: bm25s
+Requires-Dist: quebra-frases
+Requires-Dist: rapidfuzz
+Requires-Dist: json-database
+
+# BM25CorpusSolver Plugin
+
+BM25CorpusSolver is an OVOS (OpenVoiceOS) plugin designed to retrieve answers from a corpus of documents using the [BM25](https://en.wikipedia.org/wiki/Okapi_BM25)
+algorithm. This solver is ideal for question-answering systems that require efficient and accurate retrieval of
+information from a predefined set of documents.
+
+- [Features](#features)
+- [Retrieval Chatbots](#retrieval-chatbots)
+   - [Example Solvers](#example-solvers)
+     - [SquadQASolver](#squadqasolver)
+     - [FreebaseQASolver](#freebaseqasolver)
+  - [Implementing a Retrieval Chatbot](#implementing-a-retrieval-chatbot)
+     - [BM25CorpusSolver](#bm25corpussolver)
+     - [BM25QACorpusSolver](#bm25qacorpussolver)
+  - [Limitations of Retrieval Chatbots](#limitations-of-retrieval-chatbots)
+- [ReRanking](#reranking)
+   - [BM25MultipleChoiceSolver](#bm25multiplechoicesolver)
+   - [BM25EvidenceSolverPlugin](#bm25evidencesolverplugin)
+- [Embeddings Store](#embeddings-store)
+- [Integrating with Persona Framework](#integrating-with-persona-framework)
+
+
+## Features
+
+- **BM25 Algorithm**: Utilizes the BM25 ranking function for information retrieval, providing relevance-based document scoring.
+- **Configurable**: Allows customization of language, minimum confidence score, and the number of answers to retrieve.
+- **Logging**: Integrates with OVOS logging system for debugging and monitoring.
+- **BM25QACorpusSolver**: Extends `BM25CorpusSolver` to handle question-answer pairs, optimizing the retrieval process for QA datasets.
+- **BM25MultipleChoiceSolver**: Reranks multiple-choice options based on relevance to the query.
+- **BM25EvidenceSolverPlugin**: Extracts the best sentence from a text that answers a question using the BM25 algorithm.
+
+## Retrieval Chatbots
+
+Retrieval chatbots use BM25CorpusSolver to provide answers to user queries by searching through a preloaded corpus of documents or QA pairs. 
+
+These chatbots excel in environments where the information is structured and the queries are straightforward.
+
+### Example solvers
+
+#### SquadQASolver
+
+The SquadQASolver is a subclass of BM25QACorpusSolver that automatically loads and indexes the [SQuAD dataset](https://rajpurkar.github.io/SQuAD-explorer/) upon initialization.
+
+This solver is suitable for usage with the ovos-persona framework.
+
+```python
+from ovos_bm25_solver import SquadQASolver
+
+s = SquadQASolver()
+query = "is there life on mars"
+print("Query:", query)
+print("Answer:", s.spoken_answer(query))
+# 2024-07-19 22:31:12.625 - OVOS - __main__:load_corpus:60 - DEBUG - indexed 86769 documents
+# 2024-07-19 22:31:12.625 - OVOS - __main__:load_squad_corpus:119 - INFO - Loaded and indexed 86769 question-answer pairs from SQuAD dataset
+# Query: is there life on mars
+# 2024-07-19 22:31:12.628 - OVOS - __main__:retrieve_from_corpus:69 - DEBUG - Rank 1 (score: 6.334013938903809): How is it postulated that Mars life might have evolved?
+# 2024-07-19 22:31:12.628 - OVOS - __main__:retrieve_from_corpus:93 - DEBUG - closest question in corpus: How is it postulated that Mars life might have evolved?
+# Answer: similar to Antarctic
+```
+
+#### FreebaseQASolver
+
+The FreebaseQASolver is a subclass of BM25QACorpusSolver that automatically loads and indexes the [FreebaseQA dataset](https://github.com/kelvin-jiang/FreebaseQA) upon initialization.
+
+This solver is suitable for usage with the ovos-persona framework.
+
+```python
+from ovos_bm25_solver import FreebaseQASolver
+
+s = FreebaseQASolver()
+query = "What is the capital of France"
+print("Query:", query)
+print("Answer:", s.spoken_answer(query))
+# 2024-07-19 22:31:09.468 - OVOS - __main__:load_corpus:60 - DEBUG - indexed 20357 documents
+# Query: What is the capital of France
+# 2024-07-19 22:31:09.468 - OVOS - __main__:retrieve_from_corpus:69 - DEBUG - Rank 1 (score: 5.996074199676514): what is the capital of france
+# 2024-07-19 22:31:09.469 - OVOS - __main__:retrieve_from_corpus:93 - DEBUG - closest question in corpus: what is the capital of france
+# Answer: paris
+```
+
+### Implementing a Retrieval Chatbot
+
+To use the BM25CorpusSolver, you need to create an instance of the solver, load your corpus, and then query it.
+
+#### BM25CorpusSolver
+
+This class is meant to be used to create your own solvers with a dedicated corpus.
+
+```python
+from ovos_bm25_solver import BM25CorpusSolver
+
+config = {
+    "lang": "en-us",
+    "min_conf": 0.4,
+    "n_answer": 2
+}
+solver = BM25CorpusSolver(config)
+
+corpus = [
+    "a cat is a feline and likes to purr",
+    "a dog is the human's best friend and loves to play",
+    "a bird is a beautiful animal that can fly",
+    "a fish is a creature that lives in water and swims",
+]
+solver.load_corpus(corpus)
+
+query = "does the fish purr like a cat?"
+answer = solver.get_spoken_answer(query)
+print(answer)
+
+# Expected Output:
+# 2024-07-19 20:03:29.979 - OVOS - ovos_plugin_manager.utils.config:get_plugin_config:40 - DEBUG - Loaded configuration: {'module': 'ovos-translate-plugin-server', 'lang': 'en-us'}
+# 2024-07-19 20:03:30.024 - OVOS - __main__:load_corpus:28 - DEBUG - indexed 4 documents
+# 2024-07-19 20:03:30.025 - OVOS - __main__:retrieve_from_corpus:37 - DEBUG - Rank 1 (score: 1.0584375858306885): a cat is a feline and likes to purr
+# 2024-07-19 20:03:30.025 - OVOS - __main__:retrieve_from_corpus:37 - DEBUG - Rank 2 (score: 0.481589138507843): a fish is a creature that lives in water and swims
+# a cat is a feline and likes to purr. a fish is a creature that lives in water and swims
+```
+
+#### BM25QACorpusSolver
+
+This class is meant to be used to create your own solvers with a dedicated corpus
+
+BM25QACorpusSolver is an extension of BM25CorpusSolver, designed to work with question-answer pairs. It is particularly
+useful when working with datasets like SQuAD, FreebaseQA, or similar QA datasets.
+
+```python
+import requests
+from ovos_bm25_solver import BM25QACorpusSolver
+
+# Load SQuAD dataset
+corpus = {}
+data = requests.get("https://github.com/chrischute/squad/raw/master/data/train-v2.0.json").json()
+for s in data["data"]:
+    for p in s["paragraphs"]:
+        for qa in p["qas"]:
+            if "question" in qa and qa["answers"]:
+                corpus[qa["question"]] = qa["answers"][0]["text"]
+
+# Load FreebaseQA dataset
+data = requests.get("https://github.com/kelvin-jiang/FreebaseQA/raw/master/FreebaseQA-train.json").json()
+for qa in data["Questions"]:
+    q = qa["ProcessedQuestion"]
+    a = qa["Parses"][0]["Answers"][0]["AnswersName"][0]
+    corpus[q] = a
+
+# Initialize BM25QACorpusSolver with config
+config = {
+    "lang": "en-us",
+    "min_conf": 0.4,
+    "n_answer": 1
+}
+solver = BM25QACorpusSolver(config)
+solver.load_corpus(corpus)
+
+query = "is there life on mars?"
+answer = solver.get_spoken_answer(query)
+print("Query:", query)
+print("Answer:", answer)
+
+# Expected Output:
+# 86769 qa pairs imports from squad dataset
+# 20357 qa pairs imports from freebaseQA dataset
+# 2024-07-19 21:49:31.360 - OVOS - ovos_plugin_manager.language:create:233 - INFO - Loaded the Language Translation plugin ovos-translate-plugin-server
+# 2024-07-19 21:49:31.360 - OVOS - ovos_plugin_manager.utils.config:get_plugin_config:40 - DEBUG - Loaded configuration: {'module': 'ovos-translate-plugin-server', 'lang': 'en-us'}
+# 2024-07-19 21:49:32.759 - OVOS - __main__:load_corpus:61 - DEBUG - indexed 107126 documents
+# Query: is there life on mars
+# 2024-07-19 21:49:32.760 - OVOS - __main__:retrieve_from_corpus:70 - DEBUG - Rank 1 (score: 6.037893295288086): How is it postulated that Mars life might have evolved?
+# 2024-07-19 21:49:32.760 - OVOS - __main__:retrieve_from_corpus:94 - DEBUG - closest question in corpus: How is it postulated that Mars life might have evolved?
+# Answer: similar to Antarctic
+```
+
+In this example, BM25QACorpusSolver is used to load a large corpus of question-answer pairs from the SQuAD and
+FreebaseQA datasets. The solver retrieves the best matching answer for the given query.
+
+### Limitations of Retrieval Chatbots
+
+Retrieval chatbots, while powerful, have certain limitations. These include:
+
+1. **Dependence on Corpus Quality and Size**: The accuracy of a retrieval chatbot heavily relies on the quality and comprehensiveness of the underlying corpus. A limited or biased corpus can lead to inaccurate or irrelevant responses.
+2. **Static Knowledge Base**: Unlike generative models, retrieval chatbots can't generate new information or answers. They can only retrieve and rephrase content from the pre-existing corpus.
+3. **Contextual Understanding**: While advanced algorithms like BM25 can rank documents based on relevance, they may still struggle with understanding nuanced or complex queries, especially those requiring deep contextual understanding.
+4. **Scalability**: As the size of the corpus increases, the computational resources required for indexing and retrieving relevant documents also increase, potentially impacting performance.
+5. **Dynamic Updates**: Keeping the corpus updated with the latest information can be challenging, especially in fast-evolving domains.
+
+Despite these limitations, retrieval chatbots are effective for domains where the corpus is well-defined and relatively static, such as FAQs, documentation, and knowledge bases.
+
+### ReRanking
+
+ReRanking is a technique used to refine a list of potential answers by evaluating their relevance to a given query. 
+This process is crucial in scenarios where multiple options or responses need to be assessed to determine the most appropriate one.
+
+In retrieval chatbots, ReRanking helps in selecting the best answer from a set of retrieved documents or options, enhancing the accuracy of the response provided to the user.
+
+`MultipleChoiceSolver` are integrated into the OVOS Common Query framework, where they are used to select the most relevant answer from a set of multiple skill responses.
+
+#### BM25MultipleChoiceSolver
+
+BM25MultipleChoiceSolver is designed to select the best answer to a question from a list of options.
+
+In the context of retrieval chatbots, BM25MultipleChoiceSolver is useful for scenarios where a user query results in a list of predefined answers or options. 
+The solver ranks these options based on their relevance to the query and selects the most suitable one.
+
+
+```python
+from ovos_bm25_solver import BM25MultipleChoiceSolver
+
+solver = BM25MultipleChoiceSolver()
+a = solver.rerank("what is the speed of light", [
+    "very fast", "10m/s", "the speed of light is C"
+])
+print(a)
+# 2024-07-22 15:03:10.295 - OVOS - __main__:load_corpus:61 - DEBUG - indexed 3 documents
+# 2024-07-22 15:03:10.297 - OVOS - __main__:retrieve_from_corpus:70 - DEBUG - Rank 1 (score: 0.7198746800422668): the speed of light is C
+# 2024-07-22 15:03:10.297 - OVOS - __main__:retrieve_from_corpus:70 - DEBUG - Rank 2 (score: 0.0): 10m/s
+# 2024-07-22 15:03:10.297 - OVOS - __main__:retrieve_from_corpus:70 - DEBUG - Rank 3 (score: 0.0): very fast
+# [(0.7198747, 'the speed of light is C'), (0.0, '10m/s'), (0.0, 'very fast')]
+
+# NOTE: select_answer is part of the MultipleChoiceSolver base class and uses rerank internally
+a = solver.select_answer("what is the speed of light", [
+    "very fast", "10m/s", "the speed of light is C"
+])
+print(a) # the speed of light is C
+```
+
+#### BM25EvidenceSolverPlugin
+
+BM25EvidenceSolverPlugin is designed to extract the most relevant sentence from a text passage that answers a given question. This plugin uses the BM25 algorithm to evaluate and rank sentences based on their relevance to the query.
+
+In text extraction and machine comprehension tasks, BM25EvidenceSolverPlugin enables the identification of specific sentences within a larger body of text that directly address a user's query. 
+
+For example, in a scenario where a user queries about the number of rovers exploring Mars, BM25EvidenceSolverPlugin scans the provided text passage, ranks sentences based on their relevance, and extracts the most informative sentence.
+
+```python
+from ovos_bm25_solver import BM25EvidenceSolverPlugin
+
+config = {
+    "lang": "en-us",
+    "min_conf": 0.4,
+    "n_answer": 1
+}
+solver = BM25EvidenceSolverPlugin(config)
+
+text = """Mars is the fourth planet from the Sun. It is a dusty, cold, desert world with a very thin atmosphere. 
+Mars is also a dynamic planet with seasons, polar ice caps, canyons, extinct volcanoes, and evidence that it was even more active in the past.
+Mars is one of the most explored bodies in our solar system, and it's the only planet where we've sent rovers to roam the alien landscape. 
+NASA currently has two rovers (Curiosity and Perseverance), one lander (InSight), and one helicopter (Ingenuity) exploring the surface of Mars.
+"""
+query = "how many rovers are currently exploring Mars"
+answer = solver.get_best_passage(evidence=text, question=query)
+print("Query:", query)
+print("Answer:", answer)
+# 2024-07-22 15:05:14.209 - OVOS - __main__:load_corpus:61 - DEBUG - indexed 5 documents
+# 2024-07-22 15:05:14.209 - OVOS - __main__:retrieve_from_corpus:70 - DEBUG - Rank 1 (score: 1.39238703250885): NASA currently has two rovers (Curiosity and Perseverance), one lander (InSight), and one helicopter (Ingenuity) exploring the surface of Mars.
+# 2024-07-22 15:05:14.210 - OVOS - __main__:retrieve_from_corpus:70 - DEBUG - Rank 2 (score: 0.38667747378349304): Mars is one of the most explored bodies in our solar system, and it's the only planet where we've sent rovers to roam the alien landscape.
+# 2024-07-22 15:05:14.210 - OVOS - __main__:retrieve_from_corpus:70 - DEBUG - Rank 3 (score: 0.15732118487358093): Mars is the fourth planet from the Sun.
+# 2024-07-22 15:05:14.210 - OVOS - __main__:retrieve_from_corpus:70 - DEBUG - Rank 4 (score: 0.10177625715732574): Mars is also a dynamic planet with seasons, polar ice caps, canyons, extinct volcanoes, and evidence that it was even more active in the past.
+# 2024-07-22 15:05:14.210 - OVOS - __main__:retrieve_from_corpus:70 - DEBUG - Rank 5 (score: 0.0): It is a dusty, cold, desert world with a very thin atmosphere.
+# Query: how many rovers are currently exploring Mars
+# Answer: NASA currently has two rovers (Curiosity and Perseverance), one lander (InSight), and one helicopter (Ingenuity) exploring the surface of Mars.
+
+```
+
+In this example, `BM25EvidenceSolverPlugin` effectively identifies and retrieves the most relevant sentence from the provided text that answers the query about the number of rovers exploring Mars. 
+This capability is essential for applications requiring information extraction from extensive textual content, such as automated research assistants or content summarizers.
+
+## Embeddings Store
+
+A fake embeddings store is provided using only text search
+
+> NOTE: this does not scale to large datasets
+
+```python
+from ovos_bm25_solver.embed import JsonEmbeddingsDB, BM25TextEmbeddingsStore
+db = JsonEmbeddingsDB("bm25_index")
+# Initialize the BM25 text embeddings store
+index = BM25TextEmbeddingsStore(db=db)
+
+# Add documents to the database
+text = "hello world"
+text2 = "goodbye cruel world"
+index.add_document(text)
+index.add_document(text2)
+
+# Querying with fuzzy match
+results = db.query("the world", top_k=2)
+print("Fuzzy Match Results:", results)
+
+# Querying with BM25
+results = index.query("the world", top_k=2)
+print("BM25 Query Results:", results)
+
+# Comparing strings using fuzzy match
+distance = index.distance(text, text2)
+print("Distance between strings:", distance)
+```
+
+## Integrating with Persona Framework
+
+To use the `SquadQASolver` and `FreebaseQASolver` in the persona framework, you can define a persona configuration file and specify the solvers to be used.
+
+Here's an example of how to define a persona that uses the `SquadQASolver` and `FreebaseQASolver`:
+
+1. Create a persona configuration file, e.g., `qa_persona.json`:
+
+```json
+{
+  "name": "QAPersona",
+  "solvers": [
+    "ovos-solver-squadqa-plugin",
+    "ovos-solver-freebaseqa-plugin",
+    "ovos-solver-failure-plugin"
+  ]
+}
+```
+
+2. Run [ovos-persona-server](https://github.com/OpenVoiceOS/ovos-persona-server) with the defined persona:
+
+```bash
+$ ovos-persona-server --persona qa_persona.json
+```
+
+In this example, the persona named "QAPersona" will first use the `SquadQASolver` to answer questions. If it cannot find an answer, it will fall back to the `FreebaseQASolver`. Finally, it will use the `ovos-solver-failure-plugin` to ensure it always responds with something, even if the previous solvers fail.
+
+
+Check setup.py for reference in how to package your own corpus backed solvers
+
+```python
+PLUGIN_ENTRY_POINTS = [
+    'ovos-solver-bm25-squad-plugin=ovos_bm25_solver:SquadQASolver',
+    'ovos-solver-bm25-freebase-plugin=ovos_bm25_solver:FreebaseQASolver'
+]
+```
+
+

ovos_solver_bm25_plugin-0.0.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+ovos_bm25_solver/__init__.py,sha256=U9Qw5Q3f-jG3fKIatniLiPF78oA9nEH-eqO8Ub0je24,309
+ovos_bm25_solver/base.py,sha256=dt5FPp1V_dVVlwmz_3xYRSX1l5BUU8jI9mgH6sARqGs,9175
+ovos_bm25_solver/corpus.py,sha256=MDIj10AkLolYVZfw2pYlHsHZjWAOiytujO1Zpp1N8x0,5507
+ovos_bm25_solver/demo.py,sha256=1adK3M345CYYGXKHUUPZiz0VJ_XSJy6_qWKqXBeLy2I,4696
+ovos_bm25_solver/embed.py,sha256=zsWyr27q86os0pq9JIo-9AySrEYthlv6Vc7CK9B9i7k,6656
+ovos_bm25_solver/version.py,sha256=peYHqopxd1W3l3PTIN2GzerAUX8Su-_tfbPZbZ4og3o,114
+ovos_solver_bm25_plugin-0.0.0.dist-info/METADATA,sha256=IHPHrZ2NEQSNM7cm2tBCFi5JMqrVafelkZxgr15ngj0,17107
+ovos_solver_bm25_plugin-0.0.0.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+ovos_solver_bm25_plugin-0.0.0.dist-info/entry_points.txt,sha256=FKbCt9E6gO06WPMtZuUq9GOJunYk3NGEvkaCUS_itcM,558
+ovos_solver_bm25_plugin-0.0.0.dist-info/top_level.txt,sha256=AVGCOfJCv2a9Vyo7dbgszHSjM_hLnwXaUcX9Ox1srrI,17
+ovos_solver_bm25_plugin-0.0.0.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+ovos_solver_bm25_plugin-0.0.0.dist-info/RECORD,,

ovos_solver_bm25_plugin-0.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: bdist_wheel (0.44.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

ovos_solver_bm25_plugin-0.0.0.dist-info/entry_points.txt

@@ -0,0 +1,16 @@
+[neon.plugin.solver]
+ovos-solver-bm25-freebase-plugin = ovos_bm25_solver.demo:BM25FreebaseQASolver
+ovos-solver-bm25-squad-plugin = ovos_bm25_solver.demo:BM25SquadQASolver
+
+[opm.embeddings]
+ovos-json-embeddings-plugin=ovos_bm25_solver.embed:JsonEmbeddingsDB
+
+[opm.embeddings.text]
+ovos-bm25-embeddings-plugin=ovos_bm25_solver.embed:BM25TextEmbeddingsStore
+
+[opm.solver.multiple_choice]
+ovos-choice-solver-bm25=ovos_bm25_solver.base:BM25MultipleChoiceSolver
+
+[opm.solver.reading_comprehension]
+ovos-evidence-solver-bm25=ovos_bm25_solver.base:BM25SolverPlugin
+

ovos_solver_bm25_plugin-0.0.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+ovos_bm25_solver

ovos_solver_bm25_plugin-0.0.0.dist-info/zip-safe

@@ -0,0 +1 @@
+