lexmatch 0.1.0__py3-none-any.whl

lexmatch/__init__.py

@@ -0,0 +1,100 @@
+"""BM25 question matching with phrase scoring, and no dependencies.
+
+Built for the case where a vector database is more machinery than the problem
+deserves: a few dozen to a few thousand short question/answer pairs, where the
+answers are written by hand and the matching has to be predictable.
+
+    from lexmatch import Entry, Index
+
+    index = Index([
+        Entry("What do you do?", "I build backend services.",
+              aliases=["what is your job", "what is your role"]),
+        Entry("Are you open to remote work?", "Yes, and to relocation."),
+    ])
+
+    entry, score = index.answer("what is your role")
+    # -> the first entry, 1.0
+
+Two things it does that plain BM25 does not:
+
+**Phrase scoring.** BM25 cannot rank a query made entirely of stopwords —
+"what do you do?" — because those words appear in every entry and carry no
+IDF. Phrasing is scored separately so such questions still match.
+
+**A refusal threshold.** Below it, `answer()` returns None. For a question
+answering system this matters more than ranking: a confident wrong answer is
+worse than no answer, and an off-topic question scores 0.2-0.4 on shared
+filler words alone.
+
+Scores are scaled to 0..1, so a threshold means the same thing whatever the
+corpus size.
+"""
+
+from lexmatch.composer import Composer
+from lexmatch.constants import (
+    DEFAULT_THRESHOLD,
+    LENGTH_PENALTY,
+    MIN_CONTENT_OVERLAP,
+    TERM_FREQUENCY_SATURATION,
+)
+from lexmatch.contractions import Contractions
+from lexmatch.exceptions import (
+    EntryError,
+    LexmatchError,
+    ParseError,
+    SmallTalkError,
+    SynonymError,
+    TuningError,
+)
+from lexmatch.index import Index
+from lexmatch.models import (
+    Corrections,
+    Entry,
+    Outcome,
+    Reply,
+    Synonyms,
+    Tuning,
+)
+from lexmatch.morphology import Morphology
+from lexmatch.negation import Negation
+from lexmatch.parsers import CsvParser, JsonParser, MarkdownParser
+from lexmatch.questions import QuestionSplitter
+from lexmatch.smalltalk import SmallTalk
+from lexmatch.stemming import Stemmer
+from lexmatch.stopwords import STOPWORDS
+from lexmatch.text import normalize, query_terms
+
+__all__ = [
+    "DEFAULT_THRESHOLD",
+    "LENGTH_PENALTY",
+    "MIN_CONTENT_OVERLAP",
+    "STOPWORDS",
+    "TERM_FREQUENCY_SATURATION",
+    "Composer",
+    "Contractions",
+    "Corrections",
+    "CsvParser",
+    "Entry",
+    "EntryError",
+    "Index",
+    "JsonParser",
+    "LexmatchError",
+    "MarkdownParser",
+    "Morphology",
+    "Negation",
+    "Outcome",
+    "ParseError",
+    "QuestionSplitter",
+    "Reply",
+    "SmallTalk",
+    "SmallTalkError",
+    "Stemmer",
+    "SynonymError",
+    "Synonyms",
+    "Tuning",
+    "TuningError",
+    "__version__",
+    "normalize",
+    "query_terms",
+]
+__version__ = "0.1.0"

lexmatch/composer.py

@@ -0,0 +1,190 @@
+"""Turning what was found into something a person would say."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+from lexmatch.models.entry import Entry
+from lexmatch.models.outcome import Outcome
+from lexmatch.models.reply import Reply
+
+# Two parts join with "and" and no comma; three or more take commas and a
+# final "and". Named so the comparison says which case it is.
+_PAIR = 2
+
+
+class Composer:
+    """Writes one reply out of several answers, or out of none.
+
+    `answers()` returns a list and `unanswered()` returns the questions it
+    missed, which leaves the caller writing the same joining logic every
+    time — and writing it badly, because the cases are not obvious until you
+    have hit them. Three answers run together with no connective read as a
+    dump; a missed question left unmentioned reads as having been understood;
+    an ambiguous question answered at all reads as a guess.
+
+    The wording is deliberately plain and replaceable. Every phrase is a
+    class attribute, so an application with a voice of its own overrides what
+    it needs and keeps the sequencing:
+
+        class Warmer(Composer):
+            NOTHING = "Hmm, I'm not sure about that one."
+            MISSED_ONE = "I don't know about {questions}, sorry!"
+
+    Nothing here invents content. Every sentence is either an entry's own
+    answer or one of these fixed phrases, so a composed reply can say nothing
+    the owner did not write.
+    """
+
+    #: Said when nothing matched at all.
+    NOTHING = "I don't have an answer for that."
+    #: Appended to `NOTHING` when there are questions worth offering.
+    TRY_INSTEAD = "You could try: {questions}"
+    #: Said when the question was understood but was not specific enough.
+    WHICH = "Which did you mean — {options}?"
+    #: Said when the question was asked in the negative.
+    NEGATIVE = "I want to be careful with that one — could you rephrase it?"
+    #: Appended when some of a compound message went unanswered.
+    MISSED_ONE = "I don't have an answer about {questions}, though."
+    MISSED_SEVERAL = "I don't have answers about {questions}, though."
+    #: Joins the final answer of several.
+    AND = "And "
+
+    @classmethod
+    def compose(
+        cls,
+        answers: Sequence[tuple[Entry, float]],
+        missed: Sequence[str] = (),
+    ) -> str:
+        """Write one reply covering every answer, and what was missed.
+
+        The order of `answers` is the order asked, which is the order a
+        person expects them back in.
+        """
+        if not answers:
+            return cls._nothing(missed)
+        sentences = [cls._sentence(entry.answer) for entry, _ in answers]
+        if len(sentences) > 1:
+            # A connective before the last one, so three answers read as a
+            # list rather than as three separate statements that happen to
+            # be adjacent.
+            sentences[-1] = cls.AND + cls._lower_first(sentences[-1])
+        if missed:
+            sentences.append(cls._missed(missed))
+        return " ".join(sentences)
+
+    @classmethod
+    def compose_reply(cls, reply: Reply) -> str:
+        """Write one reply from a single `Reply`, whatever its outcome."""
+        if reply.outcome is Outcome.ANSWERED and reply.entry:
+            return cls._sentence(reply.entry.answer)
+        if reply.outcome is Outcome.AMBIGUOUS:
+            return cls.WHICH.format(options=cls._options(reply.alternatives))
+        if reply.outcome is Outcome.NEGATED:
+            return cls.NEGATIVE
+        return cls._nothing(reply.suggestions)
+
+    @classmethod
+    def _nothing(cls, suggestions: Sequence[str]) -> str:
+        """Say there is no answer, and offer what there is."""
+        if not suggestions:
+            return cls.NOTHING
+        offered = ", ".join(question.strip().rstrip("?") for question in suggestions)
+        return f"{cls.NOTHING} {cls.TRY_INSTEAD.format(questions=offered)}"
+
+    #: Interrogative openings stripped when a missed question is quoted
+    #: back. "I don't have an answer about *what is* your favourite film"
+    #: reads as a transcript; "about your favourite film" reads as a reply.
+    OPENERS = (
+        "what is",
+        "what are",
+        "what do",
+        "whats",
+        "when is",
+        "when can",
+        "where is",
+        "who is",
+        "how is",
+        "how do",
+        "how much",
+        "why is",
+        "do you know",
+        "do you have",
+        "do you",
+        "does your",
+        "did you",
+        "are you",
+        "is your",
+        "can you",
+        "could you",
+        "will you",
+        "would you",
+        "tell me about",
+    )
+
+    @classmethod
+    def _missed(cls, missed: Sequence[str]) -> str:
+        """Name the parts of a message that found nothing.
+
+        Said rather than left out. Answering two of three questions and
+        saying nothing about the third reads as having understood it.
+        """
+        questions = cls._join([cls._as_topic(question) for question in missed])
+        template = cls.MISSED_ONE if len(missed) == 1 else cls.MISSED_SEVERAL
+        return template.format(questions=questions)
+
+    @classmethod
+    def _options(cls, entries: Sequence[Entry]) -> str:
+        """List the contenders for a clarifying question."""
+        return " or ".join(entry.question.rstrip("?") for entry in entries)
+
+    @classmethod
+    def _as_topic(cls, question: str) -> str:
+        """Turn a question back into the thing it was about.
+
+        Quoted whole, a missed question reads as a transcript of the input
+        rather than as part of a reply. Stripping the interrogative opening
+        is enough to make it read as a topic, and anything it does not
+        recognize is quoted as it came — a slightly stiff sentence being
+        better than a mangled one.
+        """
+        trimmed = question.strip().rstrip("?").strip()
+        lowered = trimmed.lower()
+        for opener in cls.OPENERS:
+            if lowered.startswith(f"{opener} "):
+                return trimmed[len(opener) + 1 :]
+        return trimmed
+
+    @staticmethod
+    def _join(parts: Sequence[str]) -> str:
+        """Join with commas and a final "and", as somebody would say it."""
+        if len(parts) == 1:
+            return parts[0]
+        if len(parts) == _PAIR:
+            return f"{parts[0]} and {parts[1]}"
+        return f"{', '.join(parts[:-1])} and {parts[-1]}"
+
+    @staticmethod
+    def _sentence(text: str) -> str:
+        """Give the text a full stop if it has no closing punctuation.
+
+        An owner's answer is their wording and is not otherwise touched —
+        but run together without punctuation, several answers read as one
+        long sentence that means something else.
+        """
+        stripped = text.strip()
+        if not stripped or stripped[-1] in ".!?…":
+            return stripped
+        return f"{stripped}."
+
+    @staticmethod
+    def _lower_first(text: str) -> str:
+        """Lower-case the first letter after a connective.
+
+        "And Yes, I have" reads wrong. Only the first character, and only
+        when the second is lower-case — so an initialism or a name keeps its
+        capital: "And NDAs are fine" stays as it is.
+        """
+        if len(text) > 1 and text[1].islower():
+            return text[0].lower() + text[1:]
+        return text

lexmatch/constants.py

@@ -0,0 +1,124 @@
+"""The tunable numbers, in one place.
+
+These are what someone adjusts when matching is too eager or too reluctant.
+Scattered through the modules that use them they are effectively
+undiscoverable; collected here they can be read and changed as a set.
+
+What is deliberately not here: the private regex `text.py` uses to split
+words. That is implementation detail for one function, and moving it away
+from that function would make both harder to read. A constant earns a place
+in this file by being something a caller might want to change.
+"""
+
+from __future__ import annotations
+
+# Below this, `answer()` returns None rather than a weak match.
+#
+# The number is not arbitrary: an off-topic question scores roughly 0.2 to 0.4
+# on shared filler words alone, so the bar has to sit above that band. For
+# question answering, refusing matters more than ranking — a confident wrong
+# answer is worse for the reader than no answer at all.
+DEFAULT_THRESHOLD = 0.45
+
+# BM25 parameters, named for what they do rather than for the paper.
+#
+# Saturation is the paper's k1. Above it, repeating a term stops helping —
+# a question mentioning "python" five times is not five times more about
+# Python than one mentioning it once.
+TERM_FREQUENCY_SATURATION = 1.4
+
+# The paper's b. How much a long entry is discounted, because a long entry
+# matches more queries by accident than a short one does.
+LENGTH_PENALTY = 0.72
+
+# Phrase scoring needs at least this much overlap of substantive words before
+# a shared phrasing counts for anything. Without the guard, two questions
+# beginning "what is" match on that alone.
+MIN_CONTENT_OVERLAP = 1
+
+# How alike a mistyped word and a real one have to be before the first is
+# read as the second. Measured as difflib's ratio over the whole token, so
+# 0.8 is roughly one edit in five characters.
+#
+# High on purpose: a wrong correction answers a question nobody asked, which
+# is worse than refusing a question somebody did.
+CORRECTION_SIMILARITY = 0.8
+
+# Tokens shorter than this are never corrected. At three characters or fewer
+# almost everything is within one edit of everything else, so "api" would
+# "correct" to "and" and a query about an API would be answered as a
+# conjunction.
+CORRECTION_MINIMUM_LENGTH = 4
+
+# How close the runner-up has to be before a question counts as ambiguous
+# rather than answered. Measured as a difference in score.
+#
+# Both entries cleared the threshold, so the question was understood — it
+# just was not specific enough to choose between them. Guessing is a
+# confident wrong answer half the time, and the visitor cannot tell which
+# half they got.
+AMBIGUITY_MARGIN = 0.05
+
+# The score a contender has to reach before a tie counts as ambiguity rather
+# than as two weak guesses.
+#
+# A tie is only worth asking about when both candidates are confidently
+# matched. Below the floor the tie is reported as unanswered with the
+# contenders as suggestions, which is honest: several candidates at a middling
+# score mean the question was not understood, and letting the asker choose
+# beats a one-in-four guess.
+#
+# This started at 0.65, chosen when "what do you work on" tied four ways at
+# 0.537 and asked "remote work, or visa sponsorship?" — which read as
+# nonsense. That tie was an artifact of weighting every shared token equally:
+# the four entries shared "work" plus filler and nothing else. Once phrase
+# overlap was weighted by inverse document frequency the same query scored
+# 0.921 against one entry and 0.000 against the rest, so the floor was no
+# longer holding back anything it was built for — and at 0.65 it had started
+# refusing *genuine* ambiguity instead, which measures 0.580 for two rate
+# entries and 0.633 for two notice-period entries.
+#
+# 0.50 sits below both and above the noise. The knob stays because an index
+# whose entries are near-duplicates may want it higher.
+AMBIGUITY_FLOOR = 0.50
+
+# How many alternatives to offer for a genuine ambiguity. Two is a question;
+# five is a menu, and a menu is what `suggestions` is for.
+AMBIGUITY_ALTERNATIVES = 3
+
+
+# The tag `SmallTalk.entries` puts on what it builds, and the marker `Index`
+# reads to know an entry is a social formula rather than a subject.
+#
+# Social phrasings are short, and the length penalty favours short entries, so
+# a message that merely *contains* a pleasantry used to be answered by one.
+# Measured over fifteen simulated visitors: "is the pay good" scored 1.000
+# against the greeting entry, on "good" matching "good morning"; "do you care
+# about the frontend at all" scored 0.957 against the farewell, on "care"
+# matching "take care". Both are real questions with a social word in them.
+#
+# Renaming this is enough to opt a custom set of entries into the same
+# treatment, or out of it.
+SMALL_TALK_TAG = "smalltalk"
+
+
+# How much a word the index has never seen costs the query's confidence.
+#
+# An unknown word has no inverse document frequency, so it contributes
+# nothing to the score ceiling and is effectively invisible — which leaves a
+# query satisfiable by matching only the words it happened to share. Measured
+# against a twenty-five entry index, "what dashboards do you build" scored
+# 1.000 against "How do you decide build versus buy?" with "dashboards"
+# discarded before anything was scored.
+#
+# Charging for it is nonetheless **off by default**, because measuring it
+# across 133 questions from thirty-seven simulated askers showed it costs
+# more than it saves: at 1.0 the suite fell from 81.2% to 73.7%, because a
+# real question with one unrecognized word has its confidence halved and
+# drops below the threshold. It converts correct answers into refusals faster
+# than it converts wrong answers into refusals.
+#
+# Raise it toward 1.0 for an index where a wrong answer is worse than no
+# answer — a support bot whose topics are deliberately narrow, where most of
+# what it is asked is out of scope and the refusal is the product.
+UNKNOWN_WORD_COST = 0.0

lexmatch/contractions.py

@@ -0,0 +1,100 @@
+"""Reading a contraction as the words it stands for.
+
+Normalizing drops the apostrophe, so "don't" arrives as `dont` — a token in
+no stopword list and in no entry, which the weighting therefore treats as one
+of the most discriminating words in the index. Measured against a
+twenty-five entry index, `youv` (from "you've") scored 3.06, the same as the
+rarest real term there.
+
+Informal writing is full of them, and informal writing is most of what a
+question box receives. So each one is replaced by the words it stands for,
+and those words are ordinary stopwords:
+
+    dont   -> do not
+    youve  -> you have
+    whats  -> what is
+
+Expanding rather than listing the contracted forms as stopwords, because the
+words are sometimes wanted: "what is your rate" needs its "what is" for
+phrase scoring, and "I don't need sponsorship" needs its "not" for negation
+to see.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from types import MappingProxyType
+from typing import ClassVar
+
+
+class Contractions:
+    """Expands contracted forms, with the apostrophe already gone.
+
+    Replace `FORMS` to add to them, or to drop one whose expansion is wrong
+    for a particular corpus.
+    """
+
+    #: Keyed on the apostrophe-stripped form, because that is what reaches
+    #: this after normalization — which also means "dont" and "don't" are
+    #: handled by one entry.
+    #:
+    #: Ambiguous forms are deliberately absent, because expanding them is
+    #: wrong as often as it is right: "ill" is a word, and so are "well",
+    #: "id", "wed", "shed" and "lets". "its", "were", "hes" and "shes" are
+    #: left alone for the same reason, and the first two are stopwords
+    #: already.
+    FORMS: ClassVar[Mapping[str, tuple[str, ...]]] = MappingProxyType(
+        {
+            "im": ("i", "am"),
+            "ive": ("i", "have"),
+            "youve": ("you", "have"),
+            "weve": ("we", "have"),
+            "theyve": ("they", "have"),
+            "youre": ("you", "are"),
+            "theyre": ("they", "are"),
+            "whats": ("what", "is"),
+            "thats": ("that", "is"),
+            "hows": ("how", "is"),
+            "wheres": ("where", "is"),
+            "theres": ("there", "is"),
+            "whos": ("who", "is"),
+            "whens": ("when", "is"),
+            "whys": ("why", "is"),
+            "youd": ("you", "would"),
+            "youll": ("you", "will"),
+            "theyll": ("they", "will"),
+            "dont": ("do", "not"),
+            "doesnt": ("does", "not"),
+            "didnt": ("did", "not"),
+            "cant": ("can", "not"),
+            "cannot": ("can", "not"),
+            "couldnt": ("could", "not"),
+            "wouldnt": ("would", "not"),
+            "shouldnt": ("should", "not"),
+            "wont": ("will", "not"),
+            "isnt": ("is", "not"),
+            "arent": ("are", "not"),
+            "wasnt": ("was", "not"),
+            "werent": ("were", "not"),
+            "havent": ("have", "not"),
+            "hasnt": ("has", "not"),
+            "hadnt": ("had", "not"),
+            "aint": ("is", "not"),
+            "gonna": ("going", "to"),
+            "wanna": ("want", "to"),
+            "gotta": ("got", "to"),
+        }
+    )
+
+    @classmethod
+    def expand(cls, words: list[str]) -> list[str]:
+        """Replace each contraction with its words, keeping the order."""
+        expanded: list[str] = []
+        for word in words:
+            expanded.extend(cls.FORMS.get(word, (word,)))
+        return expanded
+
+    @classmethod
+    def holds(cls, word: str) -> bool:
+        """Whether this word is a contraction this knows how to expand."""
+        return word in cls.FORMS

lexmatch/exceptions/__init__.py

@@ -0,0 +1,19 @@
+"""Every error this library raises."""
+
+from __future__ import annotations
+
+from lexmatch.exceptions.base import LexmatchError
+from lexmatch.exceptions.entry import EntryError
+from lexmatch.exceptions.parsing import ParseError
+from lexmatch.exceptions.smalltalk import SmallTalkError
+from lexmatch.exceptions.synonyms import SynonymError
+from lexmatch.exceptions.tuning import TuningError
+
+__all__ = [
+    "EntryError",
+    "LexmatchError",
+    "ParseError",
+    "SmallTalkError",
+    "SynonymError",
+    "TuningError",
+]

lexmatch/exceptions/base.py

@@ -0,0 +1,11 @@
+"""The base every lexmatch error derives from."""
+
+from __future__ import annotations
+
+
+class LexmatchError(Exception):
+    """Something this library was asked to do that it cannot.
+
+    One base so an application can catch everything from here with a single
+    `except`, whichever part of the library raised it.
+    """

lexmatch/exceptions/entry.py

@@ -0,0 +1,20 @@
+"""An entry that cannot answer anything."""
+
+from __future__ import annotations
+
+from lexmatch.exceptions.base import LexmatchError
+
+
+class EntryError(LexmatchError):
+    """An entry is missing the question or the answer it needs.
+
+    Raised when the entry is built rather than when a query fails to match
+    it, because the two are indistinguishable afterwards: an entry with a
+    blank question scores zero against everything, exactly as an entry on a
+    subject nobody asks about does. One is a bug in the question set and the
+    other is normal, and nothing downstream can tell them apart.
+
+    Question sets are generated as often as they are written — from a
+    spreadsheet export, a content system, a CV parser — and a blank row is
+    the ordinary failure of all three.
+    """

lexmatch/exceptions/parsing.py

@@ -0,0 +1,14 @@
+"""A question set that could not be read."""
+
+from __future__ import annotations
+
+from lexmatch.exceptions.base import LexmatchError
+
+
+class ParseError(LexmatchError):
+    """A question set file is not shaped the way its format requires.
+
+    Carries where the problem is — a line number, a row number, a key —
+    because a question set is usually long and generated, and "something is
+    wrong with your file" costs whoever owns it an afternoon.
+    """

lexmatch/exceptions/smalltalk.py

@@ -0,0 +1,17 @@
+"""Raised when small talk is configured for an intent that does not exist."""
+
+from __future__ import annotations
+
+from lexmatch.exceptions.base import LexmatchError
+
+
+class SmallTalkError(LexmatchError, KeyError):
+    """A small-talk intent this library does not know.
+
+    Raised rather than ignored. A typo in an intent name leaves that handler
+    silently absent, which looks exactly like one that is working right up
+    until a visitor says hello and gets a refusal.
+
+    Also a `KeyError`, since naming an intent is a lookup and that is what a
+    caller validating one will already be catching.
+    """

lexmatch/exceptions/synonyms.py

@@ -0,0 +1,16 @@
+"""Raised when a synonym map cannot work."""
+
+from __future__ import annotations
+
+from lexmatch.exceptions.base import LexmatchError
+
+
+class SynonymError(LexmatchError, ValueError):
+    """A synonym is declared in a way that could never match.
+
+    A multi-word key is the common case: expansion happens one token at a
+    time, so `{"work from home": "remote"}` would never fire. Raised rather
+    than ignored, because a synonym map that silently does nothing looks
+    exactly like one that is working — the queries it was meant to catch keep
+    scoring zero and the cause is invisible.
+    """

lexmatch/exceptions/tuning.py

@@ -0,0 +1,18 @@
+"""Raised when scoring is configured in a way that cannot work."""
+
+from __future__ import annotations
+
+from lexmatch.exceptions.base import LexmatchError
+
+
+class TuningError(LexmatchError, ValueError):
+    """A tuning value is outside the range it has to be in.
+
+    Raised at construction rather than at scoring time. A threshold above 1
+    answers nothing and a saturation of 0 divides by zero, and both would
+    otherwise surface as an index that silently never matches — which reads
+    as "my questions are worded badly" rather than as a configuration error.
+
+    Also a `ValueError`, because that is what a caller validating input will
+    already be catching.
+    """