whoosh-ng 1.0.0__tar.gz

whoosh_ng-1.0.0/LICENSE.txt

@@ -0,0 +1,26 @@
+Copyright 2011 Matt Chaput. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+   1. Redistributions of source code must retain the above copyright notice,
+      this list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in the
+      documentation and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY MATT CHAPUT ``AS IS'' AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
+EVENT SHALL MATT CHAPUT OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
+OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+The views and conclusions contained in the software and documentation are
+those of the authors and should not be interpreted as representing official
+policies, either expressed or implied, of Matt Chaput.

whoosh_ng-1.0.0/MANIFEST.in

@@ -0,0 +1,8 @@
+include *.txt
+include benchmark/dcvgr10.txt.gz
+include benchmark/reuters21578.txt.gz
+include tests/english-words.10.gz
+recursive-include tests *.py
+recursive-include benchmark *.py
+recursive-include docs *.txt *.py *.rst
+recursive-include files *.png *.svg

whoosh_ng-1.0.0/PKG-INFO

@@ -0,0 +1,307 @@
+Metadata-Version: 2.4
+Name: whoosh-ng
+Version: 1.0.0
+Summary: Fast, pure-Python full-text indexing, search, and spell checking library.
+Author-email: David Orel <d.orel@free.fr>
+Maintainer-email: dorel14 <d.orel@free.fr>
+License-Expression: BSD-2-Clause
+Project-URL: Documentation, https://dorel14.github.io/whoosh-ng/
+Project-URL: Homepage, https://github.com/dorel14/whoosh-ng
+Project-URL: Issues, https://github.com/dorel14/whoosh-ng/issues
+Project-URL: Repository, https://github.com/dorel14/whoosh-ng.git
+Project-URL: Changelog, https://github.com/dorel14/whoosh-ng/releases
+Keywords: index,search,text,spell,whoosh
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Text Processing :: Indexing
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE.txt
+Requires-Dist: cached-property>=2.0.1
+Provides-Extra: models
+Requires-Dist: pydantic>=2; extra == "models"
+Requires-Dist: msgspec>=0.19; extra == "models"
+Provides-Extra: vector
+Requires-Dist: numpy>=2; extra == "vector"
+Provides-Extra: hnsw
+Requires-Dist: hnswlib; extra == "hnsw"
+Provides-Extra: faiss
+Requires-Dist: faiss-cpu; extra == "faiss"
+Provides-Extra: embeddings
+Requires-Dist: sentence-transformers; extra == "embeddings"
+Provides-Extra: fastembed
+Requires-Dist: fastembed; extra == "fastembed"
+Provides-Extra: async
+Provides-Extra: sqlite
+Provides-Extra: postgres
+Requires-Dist: psycopg[binary]>=3; extra == "postgres"
+Requires-Dist: asyncpg; extra == "postgres"
+Provides-Extra: s3
+Requires-Dist: boto3; extra == "s3"
+Provides-Extra: watch
+Requires-Dist: watchdog; extra == "watch"
+Provides-Extra: fuzzy
+Requires-Dist: rapidfuzz; extra == "fuzzy"
+Provides-Extra: phonetic
+Requires-Dist: metaphone; extra == "phonetic"
+Provides-Extra: api
+Requires-Dist: fastapi; extra == "api"
+Requires-Dist: uvicorn[standard]; extra == "api"
+Requires-Dist: orjson; extra == "api"
+Provides-Extra: metrics
+Requires-Dist: prometheus-client; extra == "metrics"
+Provides-Extra: dev
+Requires-Dist: ruff>=0.11; extra == "dev"
+Requires-Dist: mypy>=1.15; extra == "dev"
+Requires-Dist: pyright>=1.1; extra == "dev"
+Requires-Dist: pytest>=9; extra == "dev"
+Requires-Dist: pytest-cov>=6; extra == "dev"
+Requires-Dist: pytest-asyncio>=1; extra == "dev"
+Requires-Dist: pre-commit>=4; extra == "dev"
+Requires-Dist: build>=1.2; extra == "dev"
+Requires-Dist: twine>=6; extra == "dev"
+Requires-Dist: python-semantic-release>=10.5; extra == "dev"
+Requires-Dist: isort>=8.0.1; extra == "dev"
+Requires-Dist: nose>=1.3.7; extra == "dev"
+Requires-Dist: pre-commit>=4.6.0; extra == "dev"
+Provides-Extra: benchmark
+Dynamic: license-file
+
+[![PyPI](https://img.shields.io/pypi/v/whoosh-ng.svg)](https://pypi.org/project/whoosh-ng/)
+[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/whoosh-ng.svg)](https://pypi.org/project/whoosh-ng/)
+[![License](https://img.shields.io/pypi/l/whoosh-ng.svg)](https://pypi.org/project/whoosh-ng/)
+[![Documentation](https://img.shields.io/badge/docs-yes-blue.svg)](https://dorel14.github.io/whoosh-ng/)
+
+# Whoosh-NG
+
+**Whoosh-Reloaded** is a modern, pure-Python full-text indexing and search library. Version 4.0 brings a complete modernization with Python 3.11+ support, strict type annotations, optional feature profiles, and automated semantic releases.
+
+## Quick Start
+
+```bash
+pip install whoosh-ng
+```
+
+## Features
+
+- **Pure Python** - No native dependencies, works everywhere Python runs
+- **Full-text search** - BM25/BM25F scoring with phrase queries
+- **Fielded documents** - Structured indexing with typed fields
+- **Query parsing** - Flexible parser with boosting and syntax options
+- **Facets & sorting** - Group and sort results by any field
+- **Highlighting** - Snippet extraction with customizable formatters
+- **Spell checking** - Built-in spelling correction
+- **Event-driven architecture** - Plugin system with hooks and middleware
+- **Optional extensions** - Vector search, async, FastAPI, metrics, and more
+
+## Installation
+
+### Core Installation
+
+```bash
+pip install whoosh-ng
+```
+
+### Optional Profiles
+
+```bash
+# Vector search with NumPy
+pip install "whoosh-ng[vector]"
+
+# Async wrappers
+pip install "whoosh-ng[async]"
+
+# FastAPI REST API integration
+pip install "whoosh-ng[api]"
+
+# Prometheus metrics
+pip install "whoosh-ng[metrics]"
+
+# PostgreSQL backend
+pip install "whoosh-ng[postgres]"
+
+# Fuzzy matching
+pip install "whoosh-ng[fuzzy]"
+
+# Phonetic search
+pip install "whoosh-ng[phonetic]"
+
+# All optional features
+pip install "whoosh-ng[vector,async,api,metrics,postgres,fuzzy,phonetic]"
+```
+
+### Development Installation
+
+```bash
+pip install "whoosh-ng[dev]"
+```
+
+## Documentation
+
+- **[API Reference](/whoosh-ng/en/api/overview/)** - Complete module documentation
+- **[User Guides](/whoosh-ng/en/guides/)** - Tutorials and best practices
+- **[Examples](/whoosh-ng/en/examples/)** - Runnable code examples
+- **[French Documentation](/whoosh-ng/fr/)** - Documentation en français
+
+## Recent Changes in 4.0.0.dev0
+
+### Added
+
+- **Plugin System** (`whoosh.plugins`): `Plugin` base class and `PluginManager` with entry-point auto-discovery, version validation, conflict detection, enable/disable, and dependency management
+- **Registry System** (`whoosh.registry`): Generic registry plus `StorageRegistry`, `AnalyzerRegistry`, `RankingRegistry`, `SuggestRegistry`, `VectorRegistry`, `AutocompleteRegistry`, and `BackendRegistry`
+- **Middleware Pipeline** (`whoosh.middleware`): `Middleware` base class (sync + async), `MiddlewareContext`, `MiddlewareChain`, `MiddlewareRegistry`, with official `MetricsMiddleware`, `CacheMiddleware`, `CompressionMiddleware`, `EncryptionMiddleware`, and `PrometheusMiddleware`
+- **Event Bus** (`whoosh.event_bus`): `EventBus` with subscribe/publish/clear
+- **Hook System** (`whoosh.hooks`): `hookimpl`, `register_hook`, `call_hook`
+- **Backends**: `Backend` ABC with lifecycle hooks, `FileBackend`, and `SQLiteBackend`
+- **Provider Architecture**: `VectorProvider`/`VectorField`, `NumpyProvider` for vector similarity search
+- **Autocomplete Plugin** (`whoosh_modern.autocomplete`): Inverted index and edge-ngram autocomplete
+- **FastAPI Plugin** (`whoosh_fastapi`): REST endpoints for search, autocomplete, vector search, and health checks
+- **Admin UI Plugin** (`whoosh_admin`): Dashboard for index administration
+- **Entry Points**: Auto-loaded plugins under `whoosh.plugins` group
+
+### Changed
+
+- Distribution renamed from `whoosh-reloaded` to **`whoosh-ng`** (import namespace remains `whoosh`)
+- Documentation site moved to GitHub Pages: https://dorel14.github.io/whoosh-ng/
+- **Python 3.11+ required** (dropped Python 3.9/3.10 support)
+- Packaging cleaned: consolidated extras in `pyproject.toml`
+- Type annotations modernized: `mypy src/whoosh` reports 0 errors, `py.typed` marker included
+
+## Example: Simple Search
+
+```python
+from whoosh import index
+from whoosh.fields import Schema, TEXT, ID
+from whoosh.qparser import QueryParser
+
+# Define schema
+schema = Schema(
+    id=ID(stored=True, unique=True),
+    title=TEXT(stored=True),
+    content=TEXT,
+)
+
+# Create index
+ix = index.create_in("my_index", schema)
+
+# Index documents
+with ix.writer() as w:
+    w.add_document(id="1", title="Hello World", content="Welcome to Whoosh-NG")
+    w.add_document(id="2", title="Python Search", content="Fast text search library")
+
+# Search
+with ix.searcher() as s:
+    qp = QueryParser("content", ix.schema)
+    q = qp.parse("search library")
+    results = s.search(q)
+    for hit in results:
+        print(hit["title"], hit.score)
+```
+
+## Example: FastAPI Integration
+
+```python
+from fastapi import FastAPI
+from whoosh import index
+from whoosh.fields import Schema, TEXT, ID
+from whoosh_fastapi import create_app
+
+schema = Schema(id=ID(), title=TEXT(), content=TEXT())
+ix = index.create_in("docs", schema)
+
+# Create FastAPI app with Whoosh-NG endpoints
+app = create_app(ix, prefix="/api/v1")
+
+# Endpoints available:
+# GET  /api/v1/health          - Health check
+# POST /api/v1/search          - Full-text search
+# GET  /api/v1/autocomplete?q= - Autocomplete suggestions
+```
+
+## Example: Vector Search
+
+```bash
+pip install "whoosh-ng[vector]" numpy
+```
+
+```python
+from whoosh.fields import Schema, TEXT, ID, VECTOR
+from whoosh.vector import VectorField
+from whoosh_modern.vector.plugin import VectorPlugin
+from whoosh.plugins.manager import PluginManager
+import numpy as np
+
+# Create index with vector field
+schema = Schema(
+    id=ID(stored=True),
+    title=TEXT(stored=True),
+    embedding=VECTOR(dim=384),
+)
+
+# Register vector plugin
+VectorPlugin().register(PluginManager())
+
+# Index with embeddings
+ix = index.create_in("vector_db", schema)
+with ix.writer() as w:
+    w.add_document(
+        id="doc1",
+        title="Python tutorial",
+        embedding=np.random.rand(384).astype(np.float32).tobytes()
+    )
+```
+
+## Quality Gates
+
+The 4.0 quality policy requires every merged PR to pass:
+
+```bash
+uv run ruff check .
+uv run ruff format --check .
+uv run mypy src/whoosh
+uv run pytest tests
+uv run python -m build
+uv run twine check dist/*
+```
+
+CI validates these checks on Windows and Linux before merge.
+
+## Development
+
+```bash
+# Install uv (if not already)
+curl -LsSf https://astral.sh/uv/install.sh | sh
+
+# Setup
+uv python install 3.11
+uv venv --python 3.11
+uv sync --extra dev
+
+# Run tests
+uv run pytest
+```
+
+## Contributing
+
+Commits follow [Conventional Commits](https://www.conventionalcommits.org/) so `python-semantic-release` can generate versions, changelogs, tags, GitHub releases, and PyPI publications automatically.
+
+See [CONTRIBUTING.md](CONTRIBUTING.md) for development guidelines.
+
+## Maintainers
+
+- [Sygil-Dev Organization](https://github.com/Sygil-Dev)
+- Matt Chaput (original Whoosh author)
+
+## License
+
+BSD-2-Clause. See [LICENSE](LICENSE) for details.

whoosh_ng-1.0.0/README.md

@@ -0,0 +1,228 @@
+[![PyPI](https://img.shields.io/pypi/v/whoosh-ng.svg)](https://pypi.org/project/whoosh-ng/)
+[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/whoosh-ng.svg)](https://pypi.org/project/whoosh-ng/)
+[![License](https://img.shields.io/pypi/l/whoosh-ng.svg)](https://pypi.org/project/whoosh-ng/)
+[![Documentation](https://img.shields.io/badge/docs-yes-blue.svg)](https://dorel14.github.io/whoosh-ng/)
+
+# Whoosh-NG
+
+**Whoosh-Reloaded** is a modern, pure-Python full-text indexing and search library. Version 4.0 brings a complete modernization with Python 3.11+ support, strict type annotations, optional feature profiles, and automated semantic releases.
+
+## Quick Start
+
+```bash
+pip install whoosh-ng
+```
+
+## Features
+
+- **Pure Python** - No native dependencies, works everywhere Python runs
+- **Full-text search** - BM25/BM25F scoring with phrase queries
+- **Fielded documents** - Structured indexing with typed fields
+- **Query parsing** - Flexible parser with boosting and syntax options
+- **Facets & sorting** - Group and sort results by any field
+- **Highlighting** - Snippet extraction with customizable formatters
+- **Spell checking** - Built-in spelling correction
+- **Event-driven architecture** - Plugin system with hooks and middleware
+- **Optional extensions** - Vector search, async, FastAPI, metrics, and more
+
+## Installation
+
+### Core Installation
+
+```bash
+pip install whoosh-ng
+```
+
+### Optional Profiles
+
+```bash
+# Vector search with NumPy
+pip install "whoosh-ng[vector]"
+
+# Async wrappers
+pip install "whoosh-ng[async]"
+
+# FastAPI REST API integration
+pip install "whoosh-ng[api]"
+
+# Prometheus metrics
+pip install "whoosh-ng[metrics]"
+
+# PostgreSQL backend
+pip install "whoosh-ng[postgres]"
+
+# Fuzzy matching
+pip install "whoosh-ng[fuzzy]"
+
+# Phonetic search
+pip install "whoosh-ng[phonetic]"
+
+# All optional features
+pip install "whoosh-ng[vector,async,api,metrics,postgres,fuzzy,phonetic]"
+```
+
+### Development Installation
+
+```bash
+pip install "whoosh-ng[dev]"
+```
+
+## Documentation
+
+- **[API Reference](/whoosh-ng/en/api/overview/)** - Complete module documentation
+- **[User Guides](/whoosh-ng/en/guides/)** - Tutorials and best practices
+- **[Examples](/whoosh-ng/en/examples/)** - Runnable code examples
+- **[French Documentation](/whoosh-ng/fr/)** - Documentation en français
+
+## Recent Changes in 4.0.0.dev0
+
+### Added
+
+- **Plugin System** (`whoosh.plugins`): `Plugin` base class and `PluginManager` with entry-point auto-discovery, version validation, conflict detection, enable/disable, and dependency management
+- **Registry System** (`whoosh.registry`): Generic registry plus `StorageRegistry`, `AnalyzerRegistry`, `RankingRegistry`, `SuggestRegistry`, `VectorRegistry`, `AutocompleteRegistry`, and `BackendRegistry`
+- **Middleware Pipeline** (`whoosh.middleware`): `Middleware` base class (sync + async), `MiddlewareContext`, `MiddlewareChain`, `MiddlewareRegistry`, with official `MetricsMiddleware`, `CacheMiddleware`, `CompressionMiddleware`, `EncryptionMiddleware`, and `PrometheusMiddleware`
+- **Event Bus** (`whoosh.event_bus`): `EventBus` with subscribe/publish/clear
+- **Hook System** (`whoosh.hooks`): `hookimpl`, `register_hook`, `call_hook`
+- **Backends**: `Backend` ABC with lifecycle hooks, `FileBackend`, and `SQLiteBackend`
+- **Provider Architecture**: `VectorProvider`/`VectorField`, `NumpyProvider` for vector similarity search
+- **Autocomplete Plugin** (`whoosh_modern.autocomplete`): Inverted index and edge-ngram autocomplete
+- **FastAPI Plugin** (`whoosh_fastapi`): REST endpoints for search, autocomplete, vector search, and health checks
+- **Admin UI Plugin** (`whoosh_admin`): Dashboard for index administration
+- **Entry Points**: Auto-loaded plugins under `whoosh.plugins` group
+
+### Changed
+
+- Distribution renamed from `whoosh-reloaded` to **`whoosh-ng`** (import namespace remains `whoosh`)
+- Documentation site moved to GitHub Pages: https://dorel14.github.io/whoosh-ng/
+- **Python 3.11+ required** (dropped Python 3.9/3.10 support)
+- Packaging cleaned: consolidated extras in `pyproject.toml`
+- Type annotations modernized: `mypy src/whoosh` reports 0 errors, `py.typed` marker included
+
+## Example: Simple Search
+
+```python
+from whoosh import index
+from whoosh.fields import Schema, TEXT, ID
+from whoosh.qparser import QueryParser
+
+# Define schema
+schema = Schema(
+    id=ID(stored=True, unique=True),
+    title=TEXT(stored=True),
+    content=TEXT,
+)
+
+# Create index
+ix = index.create_in("my_index", schema)
+
+# Index documents
+with ix.writer() as w:
+    w.add_document(id="1", title="Hello World", content="Welcome to Whoosh-NG")
+    w.add_document(id="2", title="Python Search", content="Fast text search library")
+
+# Search
+with ix.searcher() as s:
+    qp = QueryParser("content", ix.schema)
+    q = qp.parse("search library")
+    results = s.search(q)
+    for hit in results:
+        print(hit["title"], hit.score)
+```
+
+## Example: FastAPI Integration
+
+```python
+from fastapi import FastAPI
+from whoosh import index
+from whoosh.fields import Schema, TEXT, ID
+from whoosh_fastapi import create_app
+
+schema = Schema(id=ID(), title=TEXT(), content=TEXT())
+ix = index.create_in("docs", schema)
+
+# Create FastAPI app with Whoosh-NG endpoints
+app = create_app(ix, prefix="/api/v1")
+
+# Endpoints available:
+# GET  /api/v1/health          - Health check
+# POST /api/v1/search          - Full-text search
+# GET  /api/v1/autocomplete?q= - Autocomplete suggestions
+```
+
+## Example: Vector Search
+
+```bash
+pip install "whoosh-ng[vector]" numpy
+```
+
+```python
+from whoosh.fields import Schema, TEXT, ID, VECTOR
+from whoosh.vector import VectorField
+from whoosh_modern.vector.plugin import VectorPlugin
+from whoosh.plugins.manager import PluginManager
+import numpy as np
+
+# Create index with vector field
+schema = Schema(
+    id=ID(stored=True),
+    title=TEXT(stored=True),
+    embedding=VECTOR(dim=384),
+)
+
+# Register vector plugin
+VectorPlugin().register(PluginManager())
+
+# Index with embeddings
+ix = index.create_in("vector_db", schema)
+with ix.writer() as w:
+    w.add_document(
+        id="doc1",
+        title="Python tutorial",
+        embedding=np.random.rand(384).astype(np.float32).tobytes()
+    )
+```
+
+## Quality Gates
+
+The 4.0 quality policy requires every merged PR to pass:
+
+```bash
+uv run ruff check .
+uv run ruff format --check .
+uv run mypy src/whoosh
+uv run pytest tests
+uv run python -m build
+uv run twine check dist/*
+```
+
+CI validates these checks on Windows and Linux before merge.
+
+## Development
+
+```bash
+# Install uv (if not already)
+curl -LsSf https://astral.sh/uv/install.sh | sh
+
+# Setup
+uv python install 3.11
+uv venv --python 3.11
+uv sync --extra dev
+
+# Run tests
+uv run pytest
+```
+
+## Contributing
+
+Commits follow [Conventional Commits](https://www.conventionalcommits.org/) so `python-semantic-release` can generate versions, changelogs, tags, GitHub releases, and PyPI publications automatically.
+
+See [CONTRIBUTING.md](CONTRIBUTING.md) for development guidelines.
+
+## Maintainers
+
+- [Sygil-Dev Organization](https://github.com/Sygil-Dev)
+- Matt Chaput (original Whoosh author)
+
+## License
+
+BSD-2-Clause. See [LICENSE](LICENSE) for details.

whoosh_ng-1.0.0/benchmark/dictionary.py

@@ -0,0 +1,46 @@
+import gzip
+import os
+
+from whoosh import analysis, fields
+from whoosh.support.bench import Bench, Spec
+
+
+class VulgarTongue(Spec):
+    name = "dictionary"
+    filename = "dcvgr10.txt.gz"
+    headline_field = "head"
+
+    def documents(self):
+        path = os.path.join(self.options.dir, self.filename)
+        f = gzip.GzipFile(path)
+
+        head = body = None
+        for line in f:
+            line = line.decode("latin1")
+            if line[0].isalpha():
+                if head:
+                    yield {"head": head, "body": head + body}
+                head, body = line.split(".", 1)
+            else:
+                body += line
+
+        if head:
+            yield {"head": head, "body": head + body}
+
+    def whoosh_schema(self):
+        ana = analysis.StemmingAnalyzer()
+
+        schema = fields.Schema(
+            head=fields.ID(stored=True), body=fields.TEXT(analyzer=ana, stored=True)
+        )
+        return schema
+
+    def zcatalog_setup(self, cat):
+        from zcatalog import indexes  # type: ignore
+
+        cat["head"] = indexes.FieldIndex(field_name="head")
+        cat["body"] = indexes.TextIndex(field_name="body")
+
+
+if __name__ == "__main__":
+    Bench().run(VulgarTongue)