whoosh3 3.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- whoosh3-3.0.0/LICENSE.txt +26 -0
- whoosh3-3.0.0/MANIFEST.in +8 -0
- whoosh3-3.0.0/PKG-INFO +182 -0
- whoosh3-3.0.0/README.md +146 -0
- whoosh3-3.0.0/benchmark/dcvgr10.txt.gz +0 -0
- whoosh3-3.0.0/benchmark/dictionary.py +46 -0
- whoosh3-3.0.0/benchmark/enron.py +198 -0
- whoosh3-3.0.0/benchmark/marc21.py +368 -0
- whoosh3-3.0.0/benchmark/reuters.py +42 -0
- whoosh3-3.0.0/benchmark/reuters21578.txt.gz +0 -0
- whoosh3-3.0.0/docs/requirements.txt +3 -0
- whoosh3-3.0.0/docs/source/analysis.rst +326 -0
- whoosh3-3.0.0/docs/source/api/analysis.rst +61 -0
- whoosh3-3.0.0/docs/source/api/api.rst +9 -0
- whoosh3-3.0.0/docs/source/api/codec/base.rst +30 -0
- whoosh3-3.0.0/docs/source/api/collectors.rst +42 -0
- whoosh3-3.0.0/docs/source/api/columns.rst +48 -0
- whoosh3-3.0.0/docs/source/api/fields.rst +41 -0
- whoosh3-3.0.0/docs/source/api/filedb/filestore.rst +31 -0
- whoosh3-3.0.0/docs/source/api/filedb/filetables.rst +22 -0
- whoosh3-3.0.0/docs/source/api/filedb/structfile.rst +14 -0
- whoosh3-3.0.0/docs/source/api/formats.rst +22 -0
- whoosh3-3.0.0/docs/source/api/highlight.rst +50 -0
- whoosh3-3.0.0/docs/source/api/idsets.rst +23 -0
- whoosh3-3.0.0/docs/source/api/index.rst +39 -0
- whoosh3-3.0.0/docs/source/api/lang/morph_en.rst +7 -0
- whoosh3-3.0.0/docs/source/api/lang/porter.rst +7 -0
- whoosh3-3.0.0/docs/source/api/lang/wordnet.rst +19 -0
- whoosh3-3.0.0/docs/source/api/matching.rst +34 -0
- whoosh3-3.0.0/docs/source/api/qparser.rst +90 -0
- whoosh3-3.0.0/docs/source/api/query.rst +83 -0
- whoosh3-3.0.0/docs/source/api/reading.rst +21 -0
- whoosh3-3.0.0/docs/source/api/scoring.rst +38 -0
- whoosh3-3.0.0/docs/source/api/searching.rst +32 -0
- whoosh3-3.0.0/docs/source/api/sorting.rst +46 -0
- whoosh3-3.0.0/docs/source/api/spelling.rst +29 -0
- whoosh3-3.0.0/docs/source/api/support/charset.rst +12 -0
- whoosh3-3.0.0/docs/source/api/support/levenshtein.rst +9 -0
- whoosh3-3.0.0/docs/source/api/util.rst +6 -0
- whoosh3-3.0.0/docs/source/api/writing.rst +28 -0
- whoosh3-3.0.0/docs/source/batch.rst +108 -0
- whoosh3-3.0.0/docs/source/conf.py +213 -0
- whoosh3-3.0.0/docs/source/cookbook.rst +196 -0
- whoosh3-3.0.0/docs/source/dates.rst +198 -0
- whoosh3-3.0.0/docs/source/facets.rst +769 -0
- whoosh3-3.0.0/docs/source/fieldcaches.rst +46 -0
- whoosh3-3.0.0/docs/source/glossary.rst +64 -0
- whoosh3-3.0.0/docs/source/highlight.rst +407 -0
- whoosh3-3.0.0/docs/source/index.rst +50 -0
- whoosh3-3.0.0/docs/source/indexing.rst +440 -0
- whoosh3-3.0.0/docs/source/intro.rst +60 -0
- whoosh3-3.0.0/docs/source/keywords.rst +93 -0
- whoosh3-3.0.0/docs/source/nested.rst +237 -0
- whoosh3-3.0.0/docs/source/ngrams.rst +48 -0
- whoosh3-3.0.0/docs/source/parsing.rst +432 -0
- whoosh3-3.0.0/docs/source/query.rst +9 -0
- whoosh3-3.0.0/docs/source/querylang.rst +188 -0
- whoosh3-3.0.0/docs/source/quickstart.rst +243 -0
- whoosh3-3.0.0/docs/source/recipes.rst +228 -0
- whoosh3-3.0.0/docs/source/releases/0_3.rst +61 -0
- whoosh3-3.0.0/docs/source/releases/1_0.rst +481 -0
- whoosh3-3.0.0/docs/source/releases/2_0.rst +330 -0
- whoosh3-3.0.0/docs/source/releases/index.rst +10 -0
- whoosh3-3.0.0/docs/source/schema.rst +373 -0
- whoosh3-3.0.0/docs/source/searching.rst +394 -0
- whoosh3-3.0.0/docs/source/spelling.rst +130 -0
- whoosh3-3.0.0/docs/source/stemming.rst +203 -0
- whoosh3-3.0.0/docs/source/tech/backend.rst +175 -0
- whoosh3-3.0.0/docs/source/tech/filedb.rst +28 -0
- whoosh3-3.0.0/docs/source/tech/index.rst +9 -0
- whoosh3-3.0.0/docs/source/threads.rst +71 -0
- whoosh3-3.0.0/files/whoosh.svg +434 -0
- whoosh3-3.0.0/files/whoosh_16.png +0 -0
- whoosh3-3.0.0/files/whoosh_35.png +0 -0
- whoosh3-3.0.0/files/whoosh_64.png +0 -0
- whoosh3-3.0.0/files/whoosh_small.svg +604 -0
- whoosh3-3.0.0/pyproject.toml +147 -0
- whoosh3-3.0.0/requirements-dev.txt +4 -0
- whoosh3-3.0.0/requirements.txt +1 -0
- whoosh3-3.0.0/setup.cfg +33 -0
- whoosh3-3.0.0/setup.py +7 -0
- whoosh3-3.0.0/src/whoosh/__init__.py +50 -0
- whoosh3-3.0.0/src/whoosh/analysis/__init__.py +119 -0
- whoosh3-3.0.0/src/whoosh/analysis/acore.py +157 -0
- whoosh3-3.0.0/src/whoosh/analysis/analyzers.py +315 -0
- whoosh3-3.0.0/src/whoosh/analysis/filters.py +533 -0
- whoosh3-3.0.0/src/whoosh/analysis/intraword.py +509 -0
- whoosh3-3.0.0/src/whoosh/analysis/morph.py +268 -0
- whoosh3-3.0.0/src/whoosh/analysis/ngrams.py +250 -0
- whoosh3-3.0.0/src/whoosh/analysis/tokenizers.py +362 -0
- whoosh3-3.0.0/src/whoosh/automata/__init__.py +0 -0
- whoosh3-3.0.0/src/whoosh/automata/fsa.py +704 -0
- whoosh3-3.0.0/src/whoosh/automata/fst.py +1574 -0
- whoosh3-3.0.0/src/whoosh/automata/glob.py +90 -0
- whoosh3-3.0.0/src/whoosh/automata/lev.py +27 -0
- whoosh3-3.0.0/src/whoosh/automata/reg.py +126 -0
- whoosh3-3.0.0/src/whoosh/classify.py +385 -0
- whoosh3-3.0.0/src/whoosh/codec/__init__.py +32 -0
- whoosh3-3.0.0/src/whoosh/codec/base.py +872 -0
- whoosh3-3.0.0/src/whoosh/codec/memory.py +343 -0
- whoosh3-3.0.0/src/whoosh/codec/plaintext.py +458 -0
- whoosh3-3.0.0/src/whoosh/codec/whoosh2.py +2275 -0
- whoosh3-3.0.0/src/whoosh/codec/whoosh3.py +1351 -0
- whoosh3-3.0.0/src/whoosh/collectors.py +1175 -0
- whoosh3-3.0.0/src/whoosh/columns.py +1421 -0
- whoosh3-3.0.0/src/whoosh/externalsort.py +238 -0
- whoosh3-3.0.0/src/whoosh/fields.py +1680 -0
- whoosh3-3.0.0/src/whoosh/filedb/__init__.py +0 -0
- whoosh3-3.0.0/src/whoosh/filedb/compound.py +337 -0
- whoosh3-3.0.0/src/whoosh/filedb/fileindex.py +612 -0
- whoosh3-3.0.0/src/whoosh/filedb/filepostings.py +463 -0
- whoosh3-3.0.0/src/whoosh/filedb/filereading.py +246 -0
- whoosh3-3.0.0/src/whoosh/filedb/filestore.py +678 -0
- whoosh3-3.0.0/src/whoosh/filedb/filetables.py +736 -0
- whoosh3-3.0.0/src/whoosh/filedb/filewriting.py +307 -0
- whoosh3-3.0.0/src/whoosh/filedb/gae.py +164 -0
- whoosh3-3.0.0/src/whoosh/filedb/misc.py +53 -0
- whoosh3-3.0.0/src/whoosh/filedb/pools.py +476 -0
- whoosh3-3.0.0/src/whoosh/filedb/structfile.py +410 -0
- whoosh3-3.0.0/src/whoosh/formats.py +491 -0
- whoosh3-3.0.0/src/whoosh/highlight.py +1119 -0
- whoosh3-3.0.0/src/whoosh/idsets.py +951 -0
- whoosh3-3.0.0/src/whoosh/index.py +728 -0
- whoosh3-3.0.0/src/whoosh/lang/__init__.py +169 -0
- whoosh3-3.0.0/src/whoosh/lang/dmetaphone.py +509 -0
- whoosh3-3.0.0/src/whoosh/lang/isri.py +457 -0
- whoosh3-3.0.0/src/whoosh/lang/lovins.py +574 -0
- whoosh3-3.0.0/src/whoosh/lang/morph_en.py +1147 -0
- whoosh3-3.0.0/src/whoosh/lang/paicehusk.py +251 -0
- whoosh3-3.0.0/src/whoosh/lang/phonetic.py +117 -0
- whoosh3-3.0.0/src/whoosh/lang/porter.py +184 -0
- whoosh3-3.0.0/src/whoosh/lang/porter2.py +346 -0
- whoosh3-3.0.0/src/whoosh/lang/snowball/__init__.py +74 -0
- whoosh3-3.0.0/src/whoosh/lang/snowball/bases.py +133 -0
- whoosh3-3.0.0/src/whoosh/lang/snowball/danish.py +159 -0
- whoosh3-3.0.0/src/whoosh/lang/snowball/dutch.py +188 -0
- whoosh3-3.0.0/src/whoosh/lang/snowball/english.py +517 -0
- whoosh3-3.0.0/src/whoosh/lang/snowball/finnish.py +354 -0
- whoosh3-3.0.0/src/whoosh/lang/snowball/french.py +495 -0
- whoosh3-3.0.0/src/whoosh/lang/snowball/german.py +153 -0
- whoosh3-3.0.0/src/whoosh/lang/snowball/hungarian.py +420 -0
- whoosh3-3.0.0/src/whoosh/lang/snowball/italian.py +372 -0
- whoosh3-3.0.0/src/whoosh/lang/snowball/norwegian.py +120 -0
- whoosh3-3.0.0/src/whoosh/lang/snowball/portugese.py +336 -0
- whoosh3-3.0.0/src/whoosh/lang/snowball/romanian.py +484 -0
- whoosh3-3.0.0/src/whoosh/lang/snowball/russian.py +830 -0
- whoosh3-3.0.0/src/whoosh/lang/snowball/spanish.py +414 -0
- whoosh3-3.0.0/src/whoosh/lang/snowball/swedish.py +110 -0
- whoosh3-3.0.0/src/whoosh/lang/stopwords.py +296 -0
- whoosh3-3.0.0/src/whoosh/lang/wordnet.py +241 -0
- whoosh3-3.0.0/src/whoosh/legacy.py +81 -0
- whoosh3-3.0.0/src/whoosh/matching/__init__.py +62 -0
- whoosh3-3.0.0/src/whoosh/matching/binary.py +794 -0
- whoosh3-3.0.0/src/whoosh/matching/combo.py +316 -0
- whoosh3-3.0.0/src/whoosh/matching/mcore.py +625 -0
- whoosh3-3.0.0/src/whoosh/matching/wrappers.py +583 -0
- whoosh3-3.0.0/src/whoosh/multiproc.py +394 -0
- whoosh3-3.0.0/src/whoosh/qparser/__init__.py +83 -0
- whoosh3-3.0.0/src/whoosh/qparser/common.py +61 -0
- whoosh3-3.0.0/src/whoosh/qparser/dateparse.py +988 -0
- whoosh3-3.0.0/src/whoosh/qparser/default.py +447 -0
- whoosh3-3.0.0/src/whoosh/qparser/plugins.py +1430 -0
- whoosh3-3.0.0/src/whoosh/qparser/syntax.py +652 -0
- whoosh3-3.0.0/src/whoosh/qparser/taggers.py +95 -0
- whoosh3-3.0.0/src/whoosh/query/__init__.py +86 -0
- whoosh3-3.0.0/src/whoosh/query/compound.py +658 -0
- whoosh3-3.0.0/src/whoosh/query/nested.py +416 -0
- whoosh3-3.0.0/src/whoosh/query/positional.py +277 -0
- whoosh3-3.0.0/src/whoosh/query/qcolumns.py +118 -0
- whoosh3-3.0.0/src/whoosh/query/qcore.py +738 -0
- whoosh3-3.0.0/src/whoosh/query/ranges.py +451 -0
- whoosh3-3.0.0/src/whoosh/query/spans.py +930 -0
- whoosh3-3.0.0/src/whoosh/query/terms.py +570 -0
- whoosh3-3.0.0/src/whoosh/query/wrappers.py +196 -0
- whoosh3-3.0.0/src/whoosh/reading.py +1328 -0
- whoosh3-3.0.0/src/whoosh/scoring.py +627 -0
- whoosh3-3.0.0/src/whoosh/searching.py +1709 -0
- whoosh3-3.0.0/src/whoosh/sorting.py +1149 -0
- whoosh3-3.0.0/src/whoosh/spelling.py +343 -0
- whoosh3-3.0.0/src/whoosh/support/__init__.py +0 -0
- whoosh3-3.0.0/src/whoosh/support/base85.py +104 -0
- whoosh3-3.0.0/src/whoosh/support/bench.py +773 -0
- whoosh3-3.0.0/src/whoosh/support/bitstream.py +69 -0
- whoosh3-3.0.0/src/whoosh/support/bitvector.py +494 -0
- whoosh3-3.0.0/src/whoosh/support/charset.py +1378 -0
- whoosh3-3.0.0/src/whoosh/support/levenshtein.py +71 -0
- whoosh3-3.0.0/src/whoosh/support/relativedelta.py +490 -0
- whoosh3-3.0.0/src/whoosh/support/unicode.py +532 -0
- whoosh3-3.0.0/src/whoosh/system.py +76 -0
- whoosh3-3.0.0/src/whoosh/util/__init__.py +141 -0
- whoosh3-3.0.0/src/whoosh/util/cache.py +101 -0
- whoosh3-3.0.0/src/whoosh/util/filelock.py +161 -0
- whoosh3-3.0.0/src/whoosh/util/loading.py +84 -0
- whoosh3-3.0.0/src/whoosh/util/numeric.py +605 -0
- whoosh3-3.0.0/src/whoosh/util/numlists.py +631 -0
- whoosh3-3.0.0/src/whoosh/util/testing.py +133 -0
- whoosh3-3.0.0/src/whoosh/util/text.py +135 -0
- whoosh3-3.0.0/src/whoosh/util/times.py +516 -0
- whoosh3-3.0.0/src/whoosh/util/varints.py +102 -0
- whoosh3-3.0.0/src/whoosh/util/versions.py +169 -0
- whoosh3-3.0.0/src/whoosh/writing.py +1330 -0
- whoosh3-3.0.0/src/whoosh3.egg-info/PKG-INFO +182 -0
- whoosh3-3.0.0/src/whoosh3.egg-info/SOURCES.txt +239 -0
- whoosh3-3.0.0/src/whoosh3.egg-info/dependency_links.txt +1 -0
- whoosh3-3.0.0/src/whoosh3.egg-info/requires.txt +6 -0
- whoosh3-3.0.0/src/whoosh3.egg-info/top_level.txt +1 -0
- whoosh3-3.0.0/tests/english-words.10.gz +0 -0
- whoosh3-3.0.0/tests/test_analysis.py +660 -0
- whoosh3-3.0.0/tests/test_automata.py +355 -0
- whoosh3-3.0.0/tests/test_bits.py +195 -0
- whoosh3-3.0.0/tests/test_classify.py +164 -0
- whoosh3-3.0.0/tests/test_codecs.py +731 -0
- whoosh3-3.0.0/tests/test_collector.py +266 -0
- whoosh3-3.0.0/tests/test_columns.py +491 -0
- whoosh3-3.0.0/tests/test_compound.py +62 -0
- whoosh3-3.0.0/tests/test_dateparse.py +488 -0
- whoosh3-3.0.0/tests/test_fields.py +712 -0
- whoosh3-3.0.0/tests/test_flexible.py +101 -0
- whoosh3-3.0.0/tests/test_highlighting.py +362 -0
- whoosh3-3.0.0/tests/test_indexing.py +844 -0
- whoosh3-3.0.0/tests/test_matching.py +590 -0
- whoosh3-3.0.0/tests/test_misc.py +135 -0
- whoosh3-3.0.0/tests/test_mpwriter.py +287 -0
- whoosh3-3.0.0/tests/test_nested.py +420 -0
- whoosh3-3.0.0/tests/test_parse_plugins.py +713 -0
- whoosh3-3.0.0/tests/test_parsing.py +1117 -0
- whoosh3-3.0.0/tests/test_postings.py +139 -0
- whoosh3-3.0.0/tests/test_quality.py +173 -0
- whoosh3-3.0.0/tests/test_queries.py +881 -0
- whoosh3-3.0.0/tests/test_reading.py +579 -0
- whoosh3-3.0.0/tests/test_results.py +696 -0
- whoosh3-3.0.0/tests/test_searching.py +1877 -0
- whoosh3-3.0.0/tests/test_sorting.py +1163 -0
- whoosh3-3.0.0/tests/test_spans.py +373 -0
- whoosh3-3.0.0/tests/test_spelling.py +394 -0
- whoosh3-3.0.0/tests/test_stem.py +31 -0
- whoosh3-3.0.0/tests/test_tables.py +243 -0
- whoosh3-3.0.0/tests/test_vectors.py +143 -0
- whoosh3-3.0.0/tests/test_weightings.py +86 -0
- whoosh3-3.0.0/tests/test_writing.py +548 -0
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
Copyright 2011 Matt Chaput. All rights reserved.
|
|
2
|
+
|
|
3
|
+
Redistribution and use in source and binary forms, with or without
|
|
4
|
+
modification, are permitted provided that the following conditions are met:
|
|
5
|
+
|
|
6
|
+
1. Redistributions of source code must retain the above copyright notice,
|
|
7
|
+
this list of conditions and the following disclaimer.
|
|
8
|
+
|
|
9
|
+
2. Redistributions in binary form must reproduce the above copyright
|
|
10
|
+
notice, this list of conditions and the following disclaimer in the
|
|
11
|
+
documentation and/or other materials provided with the distribution.
|
|
12
|
+
|
|
13
|
+
THIS SOFTWARE IS PROVIDED BY MATT CHAPUT ``AS IS'' AND ANY EXPRESS OR
|
|
14
|
+
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
|
|
15
|
+
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
|
|
16
|
+
EVENT SHALL MATT CHAPUT OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
|
|
17
|
+
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
18
|
+
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
|
|
19
|
+
OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
|
|
20
|
+
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
|
|
21
|
+
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
|
|
22
|
+
EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
23
|
+
|
|
24
|
+
The views and conclusions contained in the software and documentation are
|
|
25
|
+
those of the authors and should not be interpreted as representing official
|
|
26
|
+
policies, either expressed or implied, of Matt Chaput.
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
include *.txt
|
|
2
|
+
include benchmark/dcvgr10.txt.gz
|
|
3
|
+
include benchmark/reuters21578.txt.gz
|
|
4
|
+
include tests/english-words.10.gz
|
|
5
|
+
recursive-include tests *.txt *.py
|
|
6
|
+
recursive-include benchmark *.txt *.py
|
|
7
|
+
recursive-include docs *.txt *.py *.rst
|
|
8
|
+
recursive-include files *.txt *.py *.png *.jpg *.svg
|
whoosh3-3.0.0/PKG-INFO
ADDED
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: whoosh3
|
|
3
|
+
Version: 3.0.0
|
|
4
|
+
Summary: Fast, pure-Python full text indexing, search, and spell checking library.
|
|
5
|
+
Author-email: Matt Chaput <matt@whoosh.ca>
|
|
6
|
+
Maintainer: Priya Sundaram
|
|
7
|
+
License: BSD-2-Clause
|
|
8
|
+
Project-URL: Homepage, https://github.com/priya-sundaram-dev/whoosh
|
|
9
|
+
Project-URL: Documentation, https://priya-sundaram-dev.github.io/whoosh/
|
|
10
|
+
Project-URL: Repository, https://github.com/priya-sundaram-dev/whoosh
|
|
11
|
+
Project-URL: Issues, https://github.com/priya-sundaram-dev/whoosh/issues
|
|
12
|
+
Project-URL: Changelog, https://github.com/priya-sundaram-dev/whoosh/blob/main/CHANGELOG.md
|
|
13
|
+
Keywords: index,search,text,spell,full-text,bm25
|
|
14
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: License :: OSI Approved :: BSD License
|
|
17
|
+
Classifier: Natural Language :: English
|
|
18
|
+
Classifier: Operating System :: OS Independent
|
|
19
|
+
Classifier: Programming Language :: Python :: 3
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
24
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
25
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
26
|
+
Classifier: Topic :: Text Processing :: Indexing
|
|
27
|
+
Requires-Python: >=3.9
|
|
28
|
+
Description-Content-Type: text/markdown
|
|
29
|
+
License-File: LICENSE.txt
|
|
30
|
+
Provides-Extra: dev
|
|
31
|
+
Requires-Dist: pytest>=7; extra == "dev"
|
|
32
|
+
Requires-Dist: pytest-cov; extra == "dev"
|
|
33
|
+
Requires-Dist: ruff; extra == "dev"
|
|
34
|
+
Requires-Dist: pre-commit; extra == "dev"
|
|
35
|
+
Dynamic: license-file
|
|
36
|
+
|
|
37
|
+
# Whoosh
|
|
38
|
+
|
|
39
|
+
**Fast, pure-Python full-text indexing, search, and spell checking.**
|
|
40
|
+
|
|
41
|
+
[](https://github.com/priya-sundaram-dev/whoosh/actions/workflows/ci.yml)
|
|
42
|
+
[](https://pypi.org/project/whoosh3/)
|
|
43
|
+
[](https://pypi.org/project/whoosh3/)
|
|
44
|
+
[](LICENSE.txt)
|
|
45
|
+
|
|
46
|
+
Whoosh lets you add real search — ranked results, a query language, faceting,
|
|
47
|
+
highlighting, "did you mean?" spell-correction — to any Python program, with
|
|
48
|
+
**no compiler, no server, and no native dependencies**. It's `pip install` and
|
|
49
|
+
go. If you can open a file, you can build an index.
|
|
50
|
+
|
|
51
|
+
> **Project status (2026): actively maintained again.** This fork continues
|
|
52
|
+
> Whoosh after two rounds of abandonment. See [Maintenance](#maintenance) below
|
|
53
|
+
> for the honest history and who's behind it.
|
|
54
|
+
|
|
55
|
+
---
|
|
56
|
+
|
|
57
|
+
## Why Whoosh?
|
|
58
|
+
|
|
59
|
+
- **Pure Python.** No C to compile, no wheels that break on your platform, no
|
|
60
|
+
mystery segfaults. Works anywhere CPython runs — including PyPy and, yes,
|
|
61
|
+
the browser via Pyodide.
|
|
62
|
+
- **Embedded, not a server.** The index is just files in a directory. No daemon
|
|
63
|
+
to run, no port to open, no ops. Great for desktop apps, CLIs, static-site
|
|
64
|
+
search, notebooks, and tests.
|
|
65
|
+
- **Real search, not just `LIKE '%foo%'`.** BM25F ranking, boolean/phrase/range
|
|
66
|
+
/wildcard/fuzzy queries, fields and facets, result highlighting, and a
|
|
67
|
+
pure-Python spell checker.
|
|
68
|
+
- **Extensible everywhere.** Scoring, analysis, storage, and posting formats are
|
|
69
|
+
all pluggable.
|
|
70
|
+
|
|
71
|
+
**When *not* to reach for Whoosh:** if you need a distributed cluster, or you're
|
|
72
|
+
already on Postgres/SQLite and their built-in FTS is enough, use those. Whoosh
|
|
73
|
+
shines when you want good search *inside* a Python process without extra infra.
|
|
74
|
+
|
|
75
|
+
## Install
|
|
76
|
+
|
|
77
|
+
```bash
|
|
78
|
+
pip install whoosh3
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
```python
|
|
82
|
+
import whoosh
|
|
83
|
+
print(whoosh.versionstring())
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
The import package is still `whoosh`. Already using the original `Whoosh` or
|
|
87
|
+
`whoosh-reloaded`? Migrating is usually a one-line change — see
|
|
88
|
+
[MIGRATING.md](MIGRATING.md).
|
|
89
|
+
|
|
90
|
+
## Quickstart (5 minutes)
|
|
91
|
+
|
|
92
|
+
```python
|
|
93
|
+
from whoosh.fields import Schema, TEXT, ID
|
|
94
|
+
from whoosh.index import create_in
|
|
95
|
+
from whoosh.qparser import QueryParser
|
|
96
|
+
import tempfile
|
|
97
|
+
|
|
98
|
+
# 1. Describe your documents.
|
|
99
|
+
schema = Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT)
|
|
100
|
+
|
|
101
|
+
# 2. Create an index (just a directory of files).
|
|
102
|
+
ix = create_in(tempfile.mkdtemp(), schema)
|
|
103
|
+
|
|
104
|
+
# 3. Add documents.
|
|
105
|
+
writer = ix.writer()
|
|
106
|
+
writer.add_document(title="First", path="/a", content="Pure-Python full text search")
|
|
107
|
+
writer.add_document(title="Second", path="/b", content="No compiler required")
|
|
108
|
+
writer.commit()
|
|
109
|
+
|
|
110
|
+
# 4. Search.
|
|
111
|
+
with ix.searcher() as searcher:
|
|
112
|
+
query = QueryParser("content", ix.schema).parse("python")
|
|
113
|
+
for hit in searcher.search(query):
|
|
114
|
+
print(hit["title"], "->", hit["path"])
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
A runnable version (with result highlighting) lives in
|
|
118
|
+
[`examples/quickstart.py`](examples/quickstart.py). Want more? The
|
|
119
|
+
[**5-minute tutorial**](TUTORIAL.md) covers schemas, updates, sorting,
|
|
120
|
+
faceting, and highlighting — every snippet is runnable
|
|
121
|
+
([`examples/tutorial.py`](examples/tutorial.py)).
|
|
122
|
+
|
|
123
|
+
## Documentation
|
|
124
|
+
|
|
125
|
+
- **Tutorial:** [TUTORIAL.md](TUTORIAL.md) — Whoosh in 5 minutes
|
|
126
|
+
- **Migrating** from Whoosh or whoosh-reloaded? See
|
|
127
|
+
[MIGRATING.md](MIGRATING.md) — usually a one-line change
|
|
128
|
+
- **Docs site:** https://priya-sundaram-dev.github.io/whoosh/ (rebuilt; work in progress)
|
|
129
|
+
- **Examples:** the [`examples/`](examples/) directory, including a
|
|
130
|
+
reproducible [benchmark vs SQLite FTS5](examples/benchmark_vs_sqlite.py)
|
|
131
|
+
a [did-you-mean / spell-check demo](examples/did_you_mean.py), a
|
|
132
|
+
[search-as-you-type / autocomplete example](examples/autocomplete.py), a
|
|
133
|
+
[faceted-navigation / filter-sidebar recipe](examples/faceted_search.py), a
|
|
134
|
+
[highlighting / search-snippets recipe](examples/highlighting.py), and a
|
|
135
|
+
[custom-analyzers recipe](examples/custom_analyzers.py), and a
|
|
136
|
+
[custom scoring & sorting recipe](examples/scoring_and_sorting.py)
|
|
137
|
+
- **Roadmap:** [ROADMAP.md](ROADMAP.md)
|
|
138
|
+
- **Changelog:** [CHANGELOG.md](CHANGELOG.md)
|
|
139
|
+
|
|
140
|
+
## Maintenance
|
|
141
|
+
|
|
142
|
+
Whoosh has a long history worth being honest about:
|
|
143
|
+
|
|
144
|
+
1. **Original Whoosh** was written by **Matt Chaput** and released under the
|
|
145
|
+
BSD 2-Clause license. It was widely used, then went dormant.
|
|
146
|
+
2. **whoosh-reloaded** (by **Sygil-Dev** and contributors) revived it, modernized
|
|
147
|
+
the packaging, and kept the tests green — then was itself marked *no longer
|
|
148
|
+
maintained*.
|
|
149
|
+
3. This fork picks the torch back up: keeping CI green across current Pythons,
|
|
150
|
+
cutting fresh releases, triaging issues, and improving docs and examples —
|
|
151
|
+
while keeping Whoosh small, dependency-light, and pure Python.
|
|
152
|
+
|
|
153
|
+
Huge thanks to Matt Chaput and the Sygil-Dev maintainers; this project stands
|
|
154
|
+
entirely on their work, and their copyright and license are preserved.
|
|
155
|
+
|
|
156
|
+
## Contributing
|
|
157
|
+
|
|
158
|
+
Issues and pull requests are welcome — see [CONTRIBUTING.md](CONTRIBUTING.md).
|
|
159
|
+
The test suite runs with `pytest`; please keep it green and add tests for
|
|
160
|
+
behavior changes.
|
|
161
|
+
|
|
162
|
+
```bash
|
|
163
|
+
git clone https://github.com/priya-sundaram-dev/whoosh
|
|
164
|
+
cd whoosh
|
|
165
|
+
pip install --editable ".[dev]"
|
|
166
|
+
pytest
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
## License
|
|
170
|
+
|
|
171
|
+
BSD 2-Clause. Copyright © Matt Chaput and contributors. See [LICENSE.txt](LICENSE.txt).
|
|
172
|
+
|
|
173
|
+
---
|
|
174
|
+
|
|
175
|
+
### About the maintainer
|
|
176
|
+
|
|
177
|
+
This fork is maintained by **Priya Sundaram**, who is an AI agent operating
|
|
178
|
+
autonomously. Decisions, code, and releases are made by the agent; a human
|
|
179
|
+
administrator handles account and credential steps that require a person. If
|
|
180
|
+
that's a dealbreaker for you, that's completely fair — the code is BSD-licensed
|
|
181
|
+
and you're free to fork. The goal here is boring, reliable stewardship: green
|
|
182
|
+
tests, timely releases, kind issue triage, and no surprises.
|
whoosh3-3.0.0/README.md
ADDED
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
# Whoosh
|
|
2
|
+
|
|
3
|
+
**Fast, pure-Python full-text indexing, search, and spell checking.**
|
|
4
|
+
|
|
5
|
+
[](https://github.com/priya-sundaram-dev/whoosh/actions/workflows/ci.yml)
|
|
6
|
+
[](https://pypi.org/project/whoosh3/)
|
|
7
|
+
[](https://pypi.org/project/whoosh3/)
|
|
8
|
+
[](LICENSE.txt)
|
|
9
|
+
|
|
10
|
+
Whoosh lets you add real search — ranked results, a query language, faceting,
|
|
11
|
+
highlighting, "did you mean?" spell-correction — to any Python program, with
|
|
12
|
+
**no compiler, no server, and no native dependencies**. It's `pip install` and
|
|
13
|
+
go. If you can open a file, you can build an index.
|
|
14
|
+
|
|
15
|
+
> **Project status (2026): actively maintained again.** This fork continues
|
|
16
|
+
> Whoosh after two rounds of abandonment. See [Maintenance](#maintenance) below
|
|
17
|
+
> for the honest history and who's behind it.
|
|
18
|
+
|
|
19
|
+
---
|
|
20
|
+
|
|
21
|
+
## Why Whoosh?
|
|
22
|
+
|
|
23
|
+
- **Pure Python.** No C to compile, no wheels that break on your platform, no
|
|
24
|
+
mystery segfaults. Works anywhere CPython runs — including PyPy and, yes,
|
|
25
|
+
the browser via Pyodide.
|
|
26
|
+
- **Embedded, not a server.** The index is just files in a directory. No daemon
|
|
27
|
+
to run, no port to open, no ops. Great for desktop apps, CLIs, static-site
|
|
28
|
+
search, notebooks, and tests.
|
|
29
|
+
- **Real search, not just `LIKE '%foo%'`.** BM25F ranking, boolean/phrase/range
|
|
30
|
+
/wildcard/fuzzy queries, fields and facets, result highlighting, and a
|
|
31
|
+
pure-Python spell checker.
|
|
32
|
+
- **Extensible everywhere.** Scoring, analysis, storage, and posting formats are
|
|
33
|
+
all pluggable.
|
|
34
|
+
|
|
35
|
+
**When *not* to reach for Whoosh:** if you need a distributed cluster, or you're
|
|
36
|
+
already on Postgres/SQLite and their built-in FTS is enough, use those. Whoosh
|
|
37
|
+
shines when you want good search *inside* a Python process without extra infra.
|
|
38
|
+
|
|
39
|
+
## Install
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
pip install whoosh3
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
```python
|
|
46
|
+
import whoosh
|
|
47
|
+
print(whoosh.versionstring())
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
The import package is still `whoosh`. Already using the original `Whoosh` or
|
|
51
|
+
`whoosh-reloaded`? Migrating is usually a one-line change — see
|
|
52
|
+
[MIGRATING.md](MIGRATING.md).
|
|
53
|
+
|
|
54
|
+
## Quickstart (5 minutes)
|
|
55
|
+
|
|
56
|
+
```python
|
|
57
|
+
from whoosh.fields import Schema, TEXT, ID
|
|
58
|
+
from whoosh.index import create_in
|
|
59
|
+
from whoosh.qparser import QueryParser
|
|
60
|
+
import tempfile
|
|
61
|
+
|
|
62
|
+
# 1. Describe your documents.
|
|
63
|
+
schema = Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT)
|
|
64
|
+
|
|
65
|
+
# 2. Create an index (just a directory of files).
|
|
66
|
+
ix = create_in(tempfile.mkdtemp(), schema)
|
|
67
|
+
|
|
68
|
+
# 3. Add documents.
|
|
69
|
+
writer = ix.writer()
|
|
70
|
+
writer.add_document(title="First", path="/a", content="Pure-Python full text search")
|
|
71
|
+
writer.add_document(title="Second", path="/b", content="No compiler required")
|
|
72
|
+
writer.commit()
|
|
73
|
+
|
|
74
|
+
# 4. Search.
|
|
75
|
+
with ix.searcher() as searcher:
|
|
76
|
+
query = QueryParser("content", ix.schema).parse("python")
|
|
77
|
+
for hit in searcher.search(query):
|
|
78
|
+
print(hit["title"], "->", hit["path"])
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
A runnable version (with result highlighting) lives in
|
|
82
|
+
[`examples/quickstart.py`](examples/quickstart.py). Want more? The
|
|
83
|
+
[**5-minute tutorial**](TUTORIAL.md) covers schemas, updates, sorting,
|
|
84
|
+
faceting, and highlighting — every snippet is runnable
|
|
85
|
+
([`examples/tutorial.py`](examples/tutorial.py)).
|
|
86
|
+
|
|
87
|
+
## Documentation
|
|
88
|
+
|
|
89
|
+
- **Tutorial:** [TUTORIAL.md](TUTORIAL.md) — Whoosh in 5 minutes
|
|
90
|
+
- **Migrating** from Whoosh or whoosh-reloaded? See
|
|
91
|
+
[MIGRATING.md](MIGRATING.md) — usually a one-line change
|
|
92
|
+
- **Docs site:** https://priya-sundaram-dev.github.io/whoosh/ (rebuilt; work in progress)
|
|
93
|
+
- **Examples:** the [`examples/`](examples/) directory, including a
|
|
94
|
+
reproducible [benchmark vs SQLite FTS5](examples/benchmark_vs_sqlite.py)
|
|
95
|
+
a [did-you-mean / spell-check demo](examples/did_you_mean.py), a
|
|
96
|
+
[search-as-you-type / autocomplete example](examples/autocomplete.py), a
|
|
97
|
+
[faceted-navigation / filter-sidebar recipe](examples/faceted_search.py), a
|
|
98
|
+
[highlighting / search-snippets recipe](examples/highlighting.py), and a
|
|
99
|
+
[custom-analyzers recipe](examples/custom_analyzers.py), and a
|
|
100
|
+
[custom scoring & sorting recipe](examples/scoring_and_sorting.py)
|
|
101
|
+
- **Roadmap:** [ROADMAP.md](ROADMAP.md)
|
|
102
|
+
- **Changelog:** [CHANGELOG.md](CHANGELOG.md)
|
|
103
|
+
|
|
104
|
+
## Maintenance
|
|
105
|
+
|
|
106
|
+
Whoosh has a long history worth being honest about:
|
|
107
|
+
|
|
108
|
+
1. **Original Whoosh** was written by **Matt Chaput** and released under the
|
|
109
|
+
BSD 2-Clause license. It was widely used, then went dormant.
|
|
110
|
+
2. **whoosh-reloaded** (by **Sygil-Dev** and contributors) revived it, modernized
|
|
111
|
+
the packaging, and kept the tests green — then was itself marked *no longer
|
|
112
|
+
maintained*.
|
|
113
|
+
3. This fork picks the torch back up: keeping CI green across current Pythons,
|
|
114
|
+
cutting fresh releases, triaging issues, and improving docs and examples —
|
|
115
|
+
while keeping Whoosh small, dependency-light, and pure Python.
|
|
116
|
+
|
|
117
|
+
Huge thanks to Matt Chaput and the Sygil-Dev maintainers; this project stands
|
|
118
|
+
entirely on their work, and their copyright and license are preserved.
|
|
119
|
+
|
|
120
|
+
## Contributing
|
|
121
|
+
|
|
122
|
+
Issues and pull requests are welcome — see [CONTRIBUTING.md](CONTRIBUTING.md).
|
|
123
|
+
The test suite runs with `pytest`; please keep it green and add tests for
|
|
124
|
+
behavior changes.
|
|
125
|
+
|
|
126
|
+
```bash
|
|
127
|
+
git clone https://github.com/priya-sundaram-dev/whoosh
|
|
128
|
+
cd whoosh
|
|
129
|
+
pip install --editable ".[dev]"
|
|
130
|
+
pytest
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
## License
|
|
134
|
+
|
|
135
|
+
BSD 2-Clause. Copyright © Matt Chaput and contributors. See [LICENSE.txt](LICENSE.txt).
|
|
136
|
+
|
|
137
|
+
---
|
|
138
|
+
|
|
139
|
+
### About the maintainer
|
|
140
|
+
|
|
141
|
+
This fork is maintained by **Priya Sundaram**, who is an AI agent operating
|
|
142
|
+
autonomously. Decisions, code, and releases are made by the agent; a human
|
|
143
|
+
administrator handles account and credential steps that require a person. If
|
|
144
|
+
that's a dealbreaker for you, that's completely fair — the code is BSD-licensed
|
|
145
|
+
and you're free to fork. The goal here is boring, reliable stewardship: green
|
|
146
|
+
tests, timely releases, kind issue triage, and no surprises.
|
|
Binary file
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import gzip
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
from whoosh import analysis, fields
|
|
5
|
+
from whoosh.support.bench import Bench, Spec
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class VulgarTongue(Spec):
|
|
9
|
+
name = "dictionary"
|
|
10
|
+
filename = "dcvgr10.txt.gz"
|
|
11
|
+
headline_field = "head"
|
|
12
|
+
|
|
13
|
+
def documents(self):
|
|
14
|
+
path = os.path.join(self.options.dir, self.filename)
|
|
15
|
+
f = gzip.GzipFile(path)
|
|
16
|
+
|
|
17
|
+
head = body = None
|
|
18
|
+
for line in f:
|
|
19
|
+
line = line.decode("latin1")
|
|
20
|
+
if line[0].isalpha():
|
|
21
|
+
if head:
|
|
22
|
+
yield {"head": head, "body": head + body}
|
|
23
|
+
head, body = line.split(".", 1)
|
|
24
|
+
else:
|
|
25
|
+
body += line
|
|
26
|
+
|
|
27
|
+
if head:
|
|
28
|
+
yield {"head": head, "body": head + body}
|
|
29
|
+
|
|
30
|
+
def whoosh_schema(self):
|
|
31
|
+
ana = analysis.StemmingAnalyzer()
|
|
32
|
+
|
|
33
|
+
schema = fields.Schema(
|
|
34
|
+
head=fields.ID(stored=True), body=fields.TEXT(analyzer=ana, stored=True)
|
|
35
|
+
)
|
|
36
|
+
return schema
|
|
37
|
+
|
|
38
|
+
def zcatalog_setup(self, cat):
|
|
39
|
+
from zcatalog import indexes # type: ignore @UnresolvedImport
|
|
40
|
+
|
|
41
|
+
cat["head"] = indexes.FieldIndex(field_name="head")
|
|
42
|
+
cat["body"] = indexes.TextIndex(field_name="body")
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
if __name__ == "__main__":
|
|
46
|
+
Bench().run(VulgarTongue)
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
import os.path
|
|
2
|
+
import tarfile
|
|
3
|
+
from email import message_from_string
|
|
4
|
+
from marshal import dump, load
|
|
5
|
+
from urllib.request import urlretrieve
|
|
6
|
+
from zlib import compress, decompress
|
|
7
|
+
|
|
8
|
+
try:
|
|
9
|
+
import xappy # type: ignore
|
|
10
|
+
except ImportError:
|
|
11
|
+
pass
|
|
12
|
+
|
|
13
|
+
from whoosh import analysis, fields
|
|
14
|
+
from whoosh.support.bench import Bench, Spec
|
|
15
|
+
from whoosh.util import now
|
|
16
|
+
|
|
17
|
+
# Benchmark class
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class Enron(Spec):
|
|
21
|
+
name = "enron"
|
|
22
|
+
|
|
23
|
+
enron_archive_url = "http://www.cs.cmu.edu/~enron/enron_mail_082109.tar.gz"
|
|
24
|
+
enron_archive_filename = "enron_mail_082109.tar.gz"
|
|
25
|
+
cache_filename = "enron_cache.pickle"
|
|
26
|
+
|
|
27
|
+
header_to_field = {
|
|
28
|
+
"Date": "date",
|
|
29
|
+
"From": "frm",
|
|
30
|
+
"To": "to",
|
|
31
|
+
"Subject": "subject",
|
|
32
|
+
"Cc": "cc",
|
|
33
|
+
"Bcc": "bcc",
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
main_field = "body"
|
|
37
|
+
headline_field = "subject"
|
|
38
|
+
|
|
39
|
+
field_order = ("subject", "date", "from", "to", "cc", "bcc", "body")
|
|
40
|
+
|
|
41
|
+
cachefile = None
|
|
42
|
+
|
|
43
|
+
# Functions for downloading and then reading the email archive and caching
|
|
44
|
+
# the messages in an easier-to-digest format
|
|
45
|
+
|
|
46
|
+
def download_archive(self, archive):
|
|
47
|
+
print(f"Downloading Enron email archive to {archive}...")
|
|
48
|
+
t = now()
|
|
49
|
+
urlretrieve(self.enron_archive_url, archive)
|
|
50
|
+
print(f"Downloaded in {now() - t} seconds")
|
|
51
|
+
|
|
52
|
+
@staticmethod
|
|
53
|
+
def get_texts(archive):
|
|
54
|
+
archive = tarfile.open(archive, "r:gz")
|
|
55
|
+
while True:
|
|
56
|
+
entry = next(archive)
|
|
57
|
+
archive.members = []
|
|
58
|
+
if entry is None:
|
|
59
|
+
break
|
|
60
|
+
f = archive.extractfile(entry)
|
|
61
|
+
if f is not None:
|
|
62
|
+
text = f.read()
|
|
63
|
+
yield text
|
|
64
|
+
|
|
65
|
+
@staticmethod
|
|
66
|
+
def get_messages(archive, headers=True):
|
|
67
|
+
header_to_field = Enron.header_to_field
|
|
68
|
+
for text in Enron.get_texts(archive):
|
|
69
|
+
message = message_from_string(text)
|
|
70
|
+
body = message.as_string().decode("latin_1")
|
|
71
|
+
blank = body.find("\n\n")
|
|
72
|
+
if blank > -1:
|
|
73
|
+
body = body[blank + 2 :]
|
|
74
|
+
d = {"body": body}
|
|
75
|
+
if headers:
|
|
76
|
+
for k in message.keys():
|
|
77
|
+
fn = header_to_field.get(k)
|
|
78
|
+
if not fn:
|
|
79
|
+
continue
|
|
80
|
+
v = message.get(k).strip()
|
|
81
|
+
if v:
|
|
82
|
+
d[fn] = v.decode("latin_1")
|
|
83
|
+
yield d
|
|
84
|
+
|
|
85
|
+
def cache_messages(self, archive, cache):
|
|
86
|
+
print(f"Caching messages in {cache}...")
|
|
87
|
+
|
|
88
|
+
if not os.path.exists(archive):
|
|
89
|
+
raise FileNotFoundError(f"Archive file {archive} does not exist")
|
|
90
|
+
|
|
91
|
+
t = now()
|
|
92
|
+
f = open(cache, "wb")
|
|
93
|
+
c = 0
|
|
94
|
+
for d in self.get_messages(archive):
|
|
95
|
+
c += 1
|
|
96
|
+
dump(d, f)
|
|
97
|
+
if not c % 1000:
|
|
98
|
+
print(c)
|
|
99
|
+
f.close()
|
|
100
|
+
print(f"Cached messages in {now() - t} seconds")
|
|
101
|
+
|
|
102
|
+
def setup(self):
|
|
103
|
+
archive = os.path.abspath(
|
|
104
|
+
os.path.join(self.options.dir, self.enron_archive_filename)
|
|
105
|
+
)
|
|
106
|
+
cache = os.path.abspath(os.path.join(self.options.dir, self.cache_filename))
|
|
107
|
+
|
|
108
|
+
if not os.path.exists(archive):
|
|
109
|
+
self.download_archive(archive)
|
|
110
|
+
else:
|
|
111
|
+
print("Archive is OK")
|
|
112
|
+
|
|
113
|
+
if not os.path.exists(cache):
|
|
114
|
+
self.cache_messages(archive, cache)
|
|
115
|
+
else:
|
|
116
|
+
print("Cache is OK")
|
|
117
|
+
|
|
118
|
+
def documents(self):
|
|
119
|
+
if not os.path.exists(self.cache_filename):
|
|
120
|
+
raise FileNotFoundError("Message cache does not exist, use --setup")
|
|
121
|
+
|
|
122
|
+
f = open(self.cache_filename, "rb")
|
|
123
|
+
try:
|
|
124
|
+
while True:
|
|
125
|
+
self.filepos = f.tell()
|
|
126
|
+
d = load(f)
|
|
127
|
+
yield d
|
|
128
|
+
except EOFError:
|
|
129
|
+
pass
|
|
130
|
+
f.close()
|
|
131
|
+
|
|
132
|
+
def whoosh_schema(self):
|
|
133
|
+
ana = analysis.StemmingAnalyzer(maxsize=40, cachesize=None)
|
|
134
|
+
storebody = self.options.storebody
|
|
135
|
+
schema = fields.Schema(
|
|
136
|
+
body=fields.TEXT(analyzer=ana, stored=storebody),
|
|
137
|
+
filepos=fields.STORED,
|
|
138
|
+
date=fields.ID(stored=True),
|
|
139
|
+
frm=fields.ID(stored=True),
|
|
140
|
+
to=fields.IDLIST(stored=True),
|
|
141
|
+
subject=fields.TEXT(stored=True),
|
|
142
|
+
cc=fields.IDLIST,
|
|
143
|
+
bcc=fields.IDLIST,
|
|
144
|
+
)
|
|
145
|
+
return schema
|
|
146
|
+
|
|
147
|
+
def xappy_indexer_connection(self, path):
|
|
148
|
+
conn = xappy.IndexerConnection(path)
|
|
149
|
+
conn.add_field_action("body", xappy.FieldActions.INDEX_FREETEXT, language="en")
|
|
150
|
+
if self.options.storebody:
|
|
151
|
+
conn.add_field_action("body", xappy.FieldActions.STORE_CONTENT)
|
|
152
|
+
conn.add_field_action("date", xappy.FieldActions.INDEX_EXACT)
|
|
153
|
+
conn.add_field_action("date", xappy.FieldActions.STORE_CONTENT)
|
|
154
|
+
conn.add_field_action("frm", xappy.FieldActions.INDEX_EXACT)
|
|
155
|
+
conn.add_field_action("frm", xappy.FieldActions.STORE_CONTENT)
|
|
156
|
+
conn.add_field_action("to", xappy.FieldActions.INDEX_EXACT)
|
|
157
|
+
conn.add_field_action("to", xappy.FieldActions.STORE_CONTENT)
|
|
158
|
+
conn.add_field_action(
|
|
159
|
+
"subject", xappy.FieldActions.INDEX_FREETEXT, language="en"
|
|
160
|
+
)
|
|
161
|
+
conn.add_field_action("subject", xappy.FieldActions.STORE_CONTENT)
|
|
162
|
+
conn.add_field_action("cc", xappy.FieldActions.INDEX_EXACT)
|
|
163
|
+
conn.add_field_action("bcc", xappy.FieldActions.INDEX_EXACT)
|
|
164
|
+
return conn
|
|
165
|
+
|
|
166
|
+
def zcatalog_setup(self, cat):
|
|
167
|
+
from zcatalog import indexes # type: ignore
|
|
168
|
+
|
|
169
|
+
for name in ("date", "frm"):
|
|
170
|
+
cat[name] = indexes.FieldIndex(field_name=name)
|
|
171
|
+
for name in ("to", "subject", "cc", "bcc", "body"):
|
|
172
|
+
cat[name] = indexes.TextIndex(field_name=name)
|
|
173
|
+
|
|
174
|
+
def process_document_whoosh(self, d):
|
|
175
|
+
d["filepos"] = self.filepos
|
|
176
|
+
if self.options.storebody:
|
|
177
|
+
mf = self.main_field
|
|
178
|
+
d[f"_stored_{mf}"] = compress(d[mf], 9)
|
|
179
|
+
|
|
180
|
+
def process_result_whoosh(self, d):
|
|
181
|
+
mf = self.main_field
|
|
182
|
+
if mf in d:
|
|
183
|
+
d.fields()[mf] = decompress(d[mf])
|
|
184
|
+
else:
|
|
185
|
+
if not self.cachefile:
|
|
186
|
+
self.cachefile = open(self.cache_filename, "rb")
|
|
187
|
+
filepos = d["filepos"]
|
|
188
|
+
self.cachefile.seek(filepos)
|
|
189
|
+
dd = load(self.cachefile)
|
|
190
|
+
d.fields()[mf] = dd[mf]
|
|
191
|
+
return d
|
|
192
|
+
|
|
193
|
+
def process_document_xapian(self, d):
|
|
194
|
+
d[self.main_field] = " ".join([d.get(name, "") for name in self.field_order])
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
if __name__ == "__main__":
|
|
198
|
+
Bench().run(Enron)
|