whoosh3 3.0.0__py2.py3-none-any.whl

whoosh/__init__.py

@@ -0,0 +1,50 @@
+# Copyright 2008 Matt Chaput. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+#    1. Redistributions of source code must retain the above copyright notice,
+#       this list of conditions and the following disclaimer.
+#
+#    2. Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY MATT CHAPUT ``AS IS'' AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+# EVENT SHALL MATT CHAPUT OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
+# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# The views and conclusions contained in the software and documentation are
+# those of the authors and should not be interpreted as representing official
+# policies, either expressed or implied, of Matt Chaput.
+
+__version__ = (3, 0, 0)
+__version_str__ = "3.0.0"
+
+
+def versionstring(build=True, extra=True):
+    """Returns the version number of Whoosh as a string.
+
+    :param build: Whether to include the build number in the string.
+    :param extra: Whether to include alpha/beta/rc etc. tags. Only
+        checked if build is True.
+    :rtype: str
+    """
+
+    if build:
+        first = 3
+    else:
+        first = 2
+
+    s = ".".join(str(n) for n in __version__[:first])
+    if build and extra:
+        s += "".join(str(n) for n in __version__[3:])
+
+    return s

whoosh/analysis/__init__.py

@@ -0,0 +1,119 @@
+# Copyright 2007 Matt Chaput. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+#    1. Redistributions of source code must retain the above copyright notice,
+#       this list of conditions and the following disclaimer.
+#
+#    2. Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY MATT CHAPUT ``AS IS'' AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+# EVENT SHALL MATT CHAPUT OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
+# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# The views and conclusions contained in the software and documentation are
+# those of the authors and should not be interpreted as representing official
+# policies, either expressed or implied, of Matt Chaput.
+
+"""Classes and functions for turning a piece of text into an indexable stream
+of "tokens" (usually equivalent to words). There are three general classes
+involved in analysis:
+
+* Tokenizers are always at the start of the text processing pipeline. They take
+  a string and yield Token objects (actually, the same token object over and
+  over, for performance reasons) corresponding to the tokens (words) in the
+  text.
+
+  Every tokenizer is a callable that takes a string and returns an iterator of
+  tokens.
+
+* Filters take the tokens from the tokenizer and perform various
+  transformations on them. For example, the LowercaseFilter converts all tokens
+  to lowercase, which is usually necessary when indexing regular English text.
+
+  Every filter is a callable that takes a token generator and returns a token
+  generator.
+
+* Analyzers are convenience functions/classes that "package up" a tokenizer and
+  zero or more filters into a single unit. For example, the StandardAnalyzer
+  combines a RegexTokenizer, LowercaseFilter, and StopFilter.
+
+  Every analyzer is a callable that takes a string and returns a token
+  iterator. (So Tokenizers can be used as Analyzers if you don't need any
+  filtering).
+
+You can compose tokenizers and filters together using the ``|`` character::
+
+    my_analyzer = RegexTokenizer() | LowercaseFilter() | StopFilter()
+
+The first item must be a tokenizer and the rest must be filters (you can't put
+a filter first or a tokenizer after the first item).
+"""
+
+from whoosh.analysis.acore import (
+    Composable,
+    CompositionError,
+    Token,
+    entoken,
+    unstopped,
+)
+from whoosh.analysis.analyzers import (
+    Analyzer,
+    FancyAnalyzer,
+    IDAnalyzer,
+    KeywordAnalyzer,
+    LanguageAnalyzer,
+    RegexAnalyzer,
+    SimpleAnalyzer,
+    StandardAnalyzer,
+    StemmingAnalyzer,
+)
+from whoosh.analysis.filters import (
+    STOP_WORDS,
+    CharsetFilter,
+    Composable,
+    DelimitedAttributeFilter,
+    Filter,
+    LoggingFilter,
+    LowercaseFilter,
+    MultiFilter,
+    PassFilter,
+    ReverseTextFilter,
+    StopFilter,
+    StripFilter,
+    SubstitutionFilter,
+    TeeFilter,
+    url_pattern,
+)
+from whoosh.analysis.intraword import (
+    BiWordFilter,
+    CompoundWordFilter,
+    IntraWordFilter,
+    ShingleFilter,
+)
+from whoosh.analysis.morph import DoubleMetaphoneFilter, PyStemmerFilter, StemFilter
+from whoosh.analysis.ngrams import (
+    NgramAnalyzer,
+    NgramFilter,
+    NgramTokenizer,
+    NgramWordAnalyzer,
+)
+from whoosh.analysis.tokenizers import (
+    CharsetTokenizer,
+    CommaSeparatedTokenizer,
+    IDTokenizer,
+    PathTokenizer,
+    RegexTokenizer,
+    SpaceSeparatedTokenizer,
+    Tokenizer,
+)

whoosh/analysis/acore.py

@@ -0,0 +1,157 @@
+# Copyright 2007 Matt Chaput. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+#    1. Redistributions of source code must retain the above copyright notice,
+#       this list of conditions and the following disclaimer.
+#
+#    2. Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY MATT CHAPUT ``AS IS'' AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+# EVENT SHALL MATT CHAPUT OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
+# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# The views and conclusions contained in the software and documentation are
+# those of the authors and should not be interpreted as representing official
+# policies, either expressed or implied, of Matt Chaput.
+
+# Exceptions
+
+
+class CompositionError(Exception):
+    pass
+
+
+# Utility functions
+
+
+def unstopped(tokenstream):
+    """Removes tokens from a token stream where token.stopped = True."""
+    return (t for t in tokenstream if not t.stopped)
+
+
+def entoken(
+    textstream, positions=False, chars=False, start_pos=0, start_char=0, **kwargs
+):
+    """Takes a sequence of unicode strings and yields a series of Token objects
+    (actually the same Token object over and over, for performance reasons),
+    with the attributes filled in with reasonable values (for example, if
+    ``positions`` or ``chars`` is True, the function assumes each token was
+    separated by one space).
+    """
+
+    pos = start_pos
+    char = start_char
+    t = Token(positions=positions, chars=chars, **kwargs)
+
+    for text in textstream:
+        t.text = text
+
+        if positions:
+            t.pos = pos
+            pos += 1
+
+        if chars:
+            t.startchar = char
+            char = char + len(text)
+            t.endchar = char
+
+        yield t
+
+
+# Token object
+
+
+class Token:
+    """
+    Represents a "token" (usually a word) extracted from the source text being
+    indexed.
+
+    See "Advanced analysis" in the user guide for more information.
+
+    Because object instantiation in Python is slow, tokenizers should create
+    ONE SINGLE Token object and YIELD IT OVER AND OVER, changing the attributes
+    each time.
+
+    This trick means that consumers of tokens (i.e. filters) must never try to
+    hold onto the token object between loop iterations, or convert the token
+    generator into a list. Instead, save the attributes between iterations,
+    not the object::
+
+        def RemoveDuplicatesFilter(self, stream):
+            # Removes duplicate words.
+            lasttext = None
+            for token in stream:
+                # Only yield the token if its text doesn't
+                # match the previous token.
+                if lasttext != token.text:
+                    yield token
+                lasttext = token.text
+
+    ...or, call token.copy() to get a copy of the token object.
+    """
+
+    def __init__(
+        self, positions=False, chars=False, removestops=True, mode="", **kwargs
+    ):
+        """
+        :param positions: Whether tokens should have the token position in the
+            'pos' attribute.
+        :param chars: Whether tokens should have character offsets in the
+            'startchar' and 'endchar' attributes.
+        :param removestops: whether to remove stop words from the stream (if
+            the tokens pass through a stop filter).
+        :param mode: contains a string describing the purpose for which the
+            analyzer is being called, i.e. 'index' or 'query'.
+        """
+
+        self.positions = positions
+        self.chars = chars
+        self.stopped = False
+        self.boost = 1.0
+        self.removestops = removestops
+        self.mode = mode
+        self.__dict__.update(kwargs)
+
+    def __repr__(self):
+        parms = ", ".join(f"{name}={value!r}" for name, value in self.__dict__.items())
+        return f"{self.__class__.__name__}({parms})"
+
+    def copy(self):
+        # This is faster than using the copy module
+        return Token(**self.__dict__)
+
+
+# Composition support
+
+
+class Composable:
+    is_morph = False
+
+    def __or__(self, other):
+        from whoosh.analysis.analyzers import CompositeAnalyzer
+
+        if not isinstance(other, Composable):
+            raise TypeError(f"{self!r} is not composable with {other!r}")
+        return CompositeAnalyzer(self, other)
+
+    def __repr__(self):
+        attrs = ""
+        if self.__dict__:
+            attrs = ", ".join(
+                f"{key}={value!r}" for key, value in self.__dict__.items()
+            )
+        return self.__class__.__name__ + f"({attrs})"
+
+    def has_morph(self):
+        return self.is_morph

whoosh/analysis/analyzers.py

@@ -0,0 +1,315 @@
+# Copyright 2007 Matt Chaput. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+#    1. Redistributions of source code must retain the above copyright notice,
+#       this list of conditions and the following disclaimer.
+#
+#    2. Redistributions in binary form must reproduce the above copyright
+#       notice, this list of conditions and the following disclaimer in the
+#       documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY MATT CHAPUT ``AS IS'' AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+# EVENT SHALL MATT CHAPUT OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
+# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# The views and conclusions contained in the software and documentation are
+# those of the authors and should not be interpreted as representing official
+# policies, either expressed or implied, of Matt Chaput.
+
+from whoosh.analysis.acore import Composable, CompositionError
+from whoosh.analysis.filters import STOP_WORDS, LowercaseFilter, StopFilter
+from whoosh.analysis.intraword import IntraWordFilter
+from whoosh.analysis.morph import StemFilter
+from whoosh.analysis.tokenizers import (
+    CommaSeparatedTokenizer,
+    IDTokenizer,
+    RegexTokenizer,
+    SpaceSeparatedTokenizer,
+    Tokenizer,
+    default_pattern,
+)
+from whoosh.lang.porter import stem
+
+# Analyzers
+
+
+class Analyzer(Composable):
+    """Abstract base class for analyzers."""
+
+    def __repr__(self):
+        return f"{self.__class__.__name__}()"
+
+    def __eq__(self, other):
+        return (
+            other
+            and self.__class__ is other.__class__
+            and self.__dict__ == other.__dict__
+        )
+
+    def __call__(self, value, **kwargs):
+        raise NotImplementedError
+
+    def clean(self):
+        # This method is intentionally left empty.
+        pass
+
+
+class CompositeAnalyzer(Analyzer):
+    def __init__(self, *composables):
+        self.items = []
+
+        for comp in composables:
+            if isinstance(comp, CompositeAnalyzer):
+                self.items.extend(comp.items)
+            else:
+                self.items.append(comp)
+
+        # Tokenizers must start a chain, and then only filters after that
+        # (because analyzers take a string and return a generator of tokens,
+        # and filters take and return generators of tokens)
+        for item in self.items[1:]:
+            if isinstance(item, Tokenizer):
+                raise CompositionError(
+                    f"Only one tokenizer allowed at the start of the analyzer: {self.items}"
+                )
+
+    def __repr__(self):
+        return "{}({})".format(
+            self.__class__.__name__,
+            ", ".join(repr(item) for item in self.items),
+        )
+
+    def __call__(self, value, no_morph=False, **kwargs):
+        items = self.items
+        # Start with tokenizer
+        gen = items[0](value, **kwargs)
+        # Run filters
+        for item in items[1:]:
+            if not (no_morph and hasattr(item, "is_morph") and item.is_morph):
+                gen = item(gen)
+        return gen
+
+    def __getitem__(self, item):
+        return self.items.__getitem__(item)
+
+    def __len__(self):
+        return len(self.items)
+
+    def __eq__(self, other):
+        return other and self.__class__ is other.__class__ and self.items == other.items
+
+    def clean(self):
+        for item in self.items:
+            if hasattr(item, "clean"):
+                item.clean()
+
+    def has_morph(self):
+        return any(item.is_morph for item in self.items)
+
+
+# Functions that return composed analyzers
+
+
+def IDAnalyzer(lowercase=False):
+    """Deprecated, just use an IDTokenizer directly, with a LowercaseFilter if
+    desired.
+    """
+
+    tokenizer = IDTokenizer()
+    if lowercase:
+        tokenizer = tokenizer | LowercaseFilter()
+    return tokenizer
+
+
+def KeywordAnalyzer(lowercase=False, commas=False):
+    """Parses whitespace- or comma-separated tokens.
+
+    >>> ana = KeywordAnalyzer()
+    >>> [token.text for token in ana("Hello there, this is a TEST")]
+    ["Hello", "there,", "this", "is", "a", "TEST"]
+
+    :param lowercase: whether to lowercase the tokens.
+    :param commas: if True, items are separated by commas rather than
+        whitespace.
+    """
+
+    if commas:
+        tokenizer = CommaSeparatedTokenizer()
+    else:
+        tokenizer = SpaceSeparatedTokenizer()
+    if lowercase:
+        tokenizer = tokenizer | LowercaseFilter()
+    return tokenizer
+
+
+def RegexAnalyzer(expression=r"\w+(\.?\w+)*", gaps=False):
+    """Deprecated, just use a RegexTokenizer directly."""
+
+    return RegexTokenizer(expression=expression, gaps=gaps)
+
+
+def SimpleAnalyzer(expression=default_pattern, gaps=False):
+    """Composes a RegexTokenizer with a LowercaseFilter.
+
+    >>> ana = SimpleAnalyzer()
+    >>> [token.text for token in ana("Hello there, this is a TEST")]
+    ["hello", "there", "this", "is", "a", "test"]
+
+    :param expression: The regular expression pattern to use to extract tokens.
+    :param gaps: If True, the tokenizer *splits* on the expression, rather
+        than matching on the expression.
+    """
+
+    return RegexTokenizer(expression=expression, gaps=gaps) | LowercaseFilter()
+
+
+def StandardAnalyzer(
+    expression=default_pattern, stoplist=STOP_WORDS, minsize=2, maxsize=None, gaps=False
+):
+    """Composes a RegexTokenizer with a LowercaseFilter and optional
+    StopFilter.
+
+    >>> ana = StandardAnalyzer()
+    >>> [token.text for token in ana("Testing is testing and testing")]
+    ["testing", "testing", "testing"]
+
+    :param expression: The regular expression pattern to use to extract tokens.
+    :param stoplist: A list of stop words. Set this to None to disable
+        the stop word filter.
+    :param minsize: Words smaller than this are removed from the stream.
+    :param maxsize: Words longer that this are removed from the stream.
+    :param gaps: If True, the tokenizer *splits* on the expression, rather
+        than matching on the expression.
+    """
+
+    ret = RegexTokenizer(expression=expression, gaps=gaps)
+    chain = ret | LowercaseFilter()
+    if stoplist is not None:
+        chain = chain | StopFilter(stoplist=stoplist, minsize=minsize, maxsize=maxsize)
+    return chain
+
+
+def StemmingAnalyzer(
+    expression=default_pattern,
+    stoplist=STOP_WORDS,
+    minsize=2,
+    maxsize=None,
+    gaps=False,
+    stemfn=stem,
+    ignore=None,
+    cachesize=50000,
+):
+    """Composes a RegexTokenizer with a lower case filter, an optional stop
+    filter, and a stemming filter.
+
+    >>> ana = StemmingAnalyzer()
+    >>> [token.text for token in ana("Testing is testing and testing")]
+    ["test", "test", "test"]
+
+    :param expression: The regular expression pattern to use to extract tokens.
+    :param stoplist: A list of stop words. Set this to None to disable
+        the stop word filter.
+    :param minsize: Words smaller than this are removed from the stream.
+    :param maxsize: Words longer that this are removed from the stream.
+    :param gaps: If True, the tokenizer *splits* on the expression, rather
+        than matching on the expression.
+    :param ignore: a set of words to not stem.
+    :param cachesize: the maximum number of stemmed words to cache. The larger
+        this number, the faster stemming will be but the more memory it will
+        use. Use None for no cache, or -1 for an unbounded cache.
+    """
+
+    ret = RegexTokenizer(expression=expression, gaps=gaps)
+    chain = ret | LowercaseFilter()
+    if stoplist is not None:
+        chain = chain | StopFilter(stoplist=stoplist, minsize=minsize, maxsize=maxsize)
+    return chain | StemFilter(stemfn=stemfn, ignore=ignore, cachesize=cachesize)
+
+
+def FancyAnalyzer(
+    expression=r"\s+",
+    stoplist=STOP_WORDS,
+    minsize=2,
+    gaps=True,
+    splitwords=True,
+    splitnums=True,
+    mergewords=False,
+    mergenums=False,
+):
+    """Composes a RegexTokenizer with an IntraWordFilter, LowercaseFilter, and
+    StopFilter.
+
+    >>> ana = FancyAnalyzer()
+    >>> [token.text for token in ana("Should I call getInt or get_real?")]
+    ["should", "call", "getInt", "get", "int", "get_real", "get", "real"]
+
+    :param expression: The regular expression pattern to use to extract tokens.
+    :param stoplist: A list of stop words. Set this to None to disable
+        the stop word filter.
+    :param minsize: Words smaller than this are removed from the stream.
+    :param maxsize: Words longer that this are removed from the stream.
+    :param gaps: If True, the tokenizer *splits* on the expression, rather
+        than matching on the expression.
+    """
+
+    return (
+        RegexTokenizer(expression=expression, gaps=gaps)
+        | IntraWordFilter(
+            splitwords=splitwords,
+            splitnums=splitnums,
+            mergewords=mergewords,
+            mergenums=mergenums,
+        )
+        | LowercaseFilter()
+        | StopFilter(stoplist=stoplist, minsize=minsize)
+    )
+
+
+def LanguageAnalyzer(lang, expression=default_pattern, gaps=False, cachesize=50000):
+    """Configures a simple analyzer for the given language, with a
+    LowercaseFilter, StopFilter, and StemFilter.
+
+    >>> ana = LanguageAnalyzer("es")
+    >>> [token.text for token in ana("Por el mar corren las liebres")]
+    ['mar', 'corr', 'liebr']
+
+    The list of available languages is in `whoosh.lang.languages`.
+    You can use :func:`whoosh.lang.has_stemmer` and
+    :func:`whoosh.lang.has_stopwords` to check if a given language has a
+    stemming function and/or stop word list available.
+
+    :param expression: The regular expression pattern to use to extract tokens.
+    :param gaps: If True, the tokenizer *splits* on the expression, rather
+        than matching on the expression.
+    :param cachesize: the maximum number of stemmed words to cache. The larger
+        this number, the faster stemming will be but the more memory it will
+        use.
+    """
+
+    from whoosh.lang import NoStemmer, NoStopWords
+
+    # Make the start of the chain
+    chain = RegexTokenizer(expression=expression, gaps=gaps) | LowercaseFilter()
+
+    # Add a stop word filter
+    try:
+        chain = chain | StopFilter(lang=lang)
+    except NoStopWords:
+        pass
+
+    # Add a stemming filter
+    try:
+        chain = chain | StemFilter(lang=lang, cachesize=cachesize)
+    except NoStemmer:
+        pass
+
+    return chain