fromscratch-pysearch 0.3.0__py3-none-any.whl

fromscratch_pysearch-0.3.0.dist-info/METADATA

@@ -0,0 +1,188 @@
+Metadata-Version: 2.5
+Name: fromscratch-pysearch
+Version: 0.3.0
+Summary: An educational full-text search engine built from scratch in Python
+Project-URL: Homepage, https://github.com/robcox2910/pysearch
+Project-URL: Repository, https://github.com/robcox2910/pysearch
+Project-URL: Issues, https://github.com/robcox2910/pysearch/issues
+Project-URL: Documentation, https://robcox2910.github.io/pysearch/
+Author: Rob Cox
+License: MIT
+License-File: LICENSE
+Keywords: education,inverted-index,search,tfidf
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Education
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Education
+Classifier: Topic :: Text Processing :: Indexing
+Classifier: Typing :: Typed
+Requires-Python: >=3.14
+Provides-Extra: dev
+Requires-Dist: commitizen>=4.4.0; extra == 'dev'
+Requires-Dist: hypothesis>=6.170; extra == 'dev'
+Requires-Dist: pip-audit>=2.10.0; extra == 'dev'
+Requires-Dist: pre-commit>=4.0.1; extra == 'dev'
+Requires-Dist: pyright>=1.1.400; extra == 'dev'
+Requires-Dist: pytest-cov>=6.0.0; extra == 'dev'
+Requires-Dist: pytest>=8.3.4; extra == 'dev'
+Requires-Dist: ruff>=0.9.3; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# PySearch
+
+**An educational full-text search engine built from scratch in Python.**
+
+## The Librarian Who Read Every Book
+
+Imagine a librarian who has read every single book in the library. When you ask
+her "Which books mention *dinosaurs*?", she doesn't need to open a single book.
+She already knows the answer because she kept notes while reading.
+
+For every word she encountered, she wrote down which book it was in and how many
+times it appeared. Her notebook is called an **inverted index**, and it lets her
+answer your question in the blink of an eye.
+
+But what if ten books mention dinosaurs? Which one should you read first? The
+librarian uses a clever trick called **TF-IDF** -- she ranks the books so the
+ones that talk about dinosaurs *the most* (and about other things *the least*)
+float to the top.
+
+PySearch teaches you how to build this librarian from scratch.
+
+## Examples
+
+### Adding Documents and Searching
+
+```python
+from pysearch.engine import SearchEngine
+
+# Create a search engine and add some documents
+engine = SearchEngine()
+engine.add("doc1", "Python is a great programming language")
+engine.add("doc2", "Java is also a popular language")
+engine.add("doc3", "Python and Java are both used for web development")
+
+# Search for a single term -- results ranked by relevance
+results = engine.search("python")
+for doc_id, score in results:
+    print(f"{doc_id}: {score:.4f}")
+
+# Boolean queries
+results = engine.search("python AND java")   # documents with both
+results = engine.search("python OR java")    # documents with either
+```
+
+### Using the Tokenizer Directly
+
+```python
+from pysearch.tokenizer import tokenize, simple_stem
+
+tokens = tokenize("The quick brown foxes are running!")
+print(tokens)  # ['quick', 'brown', 'fox', 'runn']
+
+print(simple_stem("running"))  # 'runn'
+print(simple_stem("played"))   # 'play'
+print(simple_stem("quickly"))  # 'quick'
+```
+
+### Building an Inverted Index
+
+```python
+from pysearch.index import InvertedIndex
+
+index = InvertedIndex()
+index.add_document("doc1", "cats and dogs")
+index.add_document("doc2", "cats and birds")
+index.add_document("doc3", "dogs and birds")
+
+print(index.search("cat"))            # ['doc1', 'doc2']
+print(index.search_and(["cat", "dog"]))  # ['doc1']
+print(index.search_or(["cat", "dog"]))   # ['doc1', 'doc2', 'doc3']
+```
+
+## Features
+
+- **Tokenizer** -- Split text into searchable words, remove stop words, and
+  apply simple stemming
+- **Inverted Index** -- Map every word to the documents that contain it for
+  instant lookups
+- **TF-IDF Ranking** -- Score and rank results so the most relevant documents
+  come first
+- **Boolean Queries** -- Combine search terms with AND and OR operators
+- **100% Typed** -- Full type annotations with strict Pyright checking
+
+## Quick Start
+
+```bash
+# Install from source
+git clone https://github.com/robcox2910/pysearch.git
+cd pysearch
+uv sync --all-extras
+
+# Run the tests
+uv run pytest
+
+# Try it in a Python shell
+uv run python -c "
+from pysearch.engine import SearchEngine
+engine = SearchEngine()
+engine.add('doc1', 'hello world')
+print(engine.search('hello'))
+"
+```
+
+## Concepts
+
+Saving an engine to disk and loading it back: [Saving Your Index](docs/concepts/saving.md).
+
+<!-- docs:start -->
+## Documentation
+
+Full docs (with kid-friendly explanations of every idea): https://robcox2910.github.io/pysearch/
+<!-- docs:end -->
+
+## Releasing
+
+Other projects in the series ([PyStack](https://github.com/robcox2910/pystack)) use a pinned release of this one,
+not whatever is on `main`. To publish a change for them:
+
+1. Merge the change to `main`.
+2. Tag it: `git tag -a vX.Y.Z -m vX.Y.Z && git push origin vX.Y.Z`.
+   A change to the public API bumps the minor version (`v0.1.0` → `v0.2.0`);
+   anything else bumps the patch (`v0.1.0` → `v0.1.1`).
+3. In each project that uses it, edit the tag in `[tool.uv.sources]`, run
+   `uv lock --upgrade-package <name>`, and open a pull request so CI checks
+   the pair together.
+
+<!-- series:start -->
+## Related Projects
+
+This project is one of a series that builds real systems "from scratch in Python" to teach how they work. See the [series home](https://robcox2910.github.io/) for the full story.
+
+| Project | What it teaches | Repository | Docs |
+| --- | --- | --- | --- |
+| PyOS | Operating systems | [py-os](https://github.com/robcox2910/py-os) | [Docs](https://robcox2910.github.io/py-os/) |
+| Pebble | Compilers and programming languages | [pebble-lang](https://github.com/robcox2910/pebble-lang) | [Docs](https://robcox2910.github.io/pebble-lang/) |
+| PyDB | Relational databases | [pydb](https://github.com/robcox2910/pydb) | [Docs](https://robcox2910.github.io/pydb/) |
+| PyKV | Key-value stores | [pykv](https://github.com/robcox2910/pykv) | [Docs](https://robcox2910.github.io/pykv/) |
+| PyDocDB | Document databases | [pydoc-db](https://github.com/robcox2910/pydoc-db) | [Docs](https://robcox2910.github.io/pydoc-db/) |
+| PyGraphDB | Graph databases | [pygraphdb](https://github.com/robcox2910/pygraphdb) | [Docs](https://robcox2910.github.io/pygraphdb/) |
+| PyTSDB | Time-series databases | [pytsdb](https://github.com/robcox2910/pytsdb) | [Docs](https://robcox2910.github.io/pytsdb/) |
+| PyVecDB | Vector databases | [pyvecdb](https://github.com/robcox2910/pyvecdb) | [Docs](https://robcox2910.github.io/pyvecdb/) |
+| PyColDB | Column-family databases | [pycoldb](https://github.com/robcox2910/pycoldb) | [Docs](https://robcox2910.github.io/pycoldb/) |
+| PyCrypt | Cryptography | [pycrypt](https://github.com/robcox2910/pycrypt) | [Docs](https://robcox2910.github.io/pycrypt/) |
+| PyNet | Computer networking | [pynet](https://github.com/robcox2910/pynet) | [Docs](https://robcox2910.github.io/pynet/) |
+| PyWeb | HTTP web servers | [pyweb](https://github.com/robcox2910/pyweb) | [Docs](https://robcox2910.github.io/pyweb/) |
+| PyGit | Version control | [pygit](https://github.com/robcox2910/pygit) | [Docs](https://robcox2910.github.io/pygit/) |
+| **PySearch** | **Search engines** | [pysearch](https://github.com/robcox2910/pysearch) | [Docs](https://robcox2910.github.io/pysearch/) |
+| PyMQ | Message queues | [pymq](https://github.com/robcox2910/pymq) | [Docs](https://robcox2910.github.io/pymq/) |
+| PyLLM | Language models (the brain) | [pyllm](https://github.com/robcox2910/pyllm) | [Docs](https://robcox2910.github.io/pyllm/) |
+| PyScheduler | Job scheduling | [pyscheduler](https://github.com/robcox2910/pyscheduler) | [Docs](https://robcox2910.github.io/pyscheduler/) |
+| PyStack | Full-stack integration | [pystack](https://github.com/robcox2910/pystack) | [Docs](https://robcox2910.github.io/pystack/) |
+<!-- series:end -->
+
+## License
+
+MIT -- see [LICENSE](LICENSE) for details.

fromscratch_pysearch-0.3.0.dist-info/RECORD

@@ -0,0 +1,15 @@
+pysearch/__init__.py,sha256=itjA-esHRCj8DdeWHzJWRv6kob7yqJiVnZEbEHqbuC0,84
+pysearch/_version.py,sha256=RCY04NY0kBCyn6ttH6lnVL6Gz2gceOrt45f0yDOOU8A,520
+pysearch/atomic.py,sha256=QLLPGZqpA-TgVV5lDEKCLB7vVr05bxj97_Fwykz_gi4,2345
+pysearch/engine.py,sha256=FiGX5g8HoSag2WnEYEABV2KfOuS5b4qnalBuparjRco,5914
+pysearch/errors.py,sha256=H5xkapTBr1ww7UqMKhUUfIJe5R0kp5xT_DBESkCNf4Y,607
+pysearch/index.py,sha256=cdHZP4dhaRNJ7E7dI-3xNnqxzHtAWYJVNxRvgQr6w5Q,4107
+pysearch/persistence.py,sha256=yCs5K-ZTV7MJ74Qhz6AQak51yOZEYS2XS1F7KZEokxw,2265
+pysearch/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+pysearch/query.py,sha256=zkPpgLt9ikC1tsBddv_y1BWsaagp9-4k710Cb1ldfMs,3658
+pysearch/tfidf.py,sha256=y4K5sG5GO-YQG0qIM6uqnvoJ0_OUuBMM7tAFBOVAUi4,3114
+pysearch/tokenizer.py,sha256=Iq-W7g0sLHZaKAGwpqdBhzeI5oBWUOfj5aG_UYZjRCg,2686
+fromscratch_pysearch-0.3.0.dist-info/METADATA,sha256=4BssHRVeJWfC7XEyhmtoM301Vu34zxa6ejwoUOiqu1Q,7998
+fromscratch_pysearch-0.3.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+fromscratch_pysearch-0.3.0.dist-info/licenses/LICENSE,sha256=NtHM2ak--oE71VCcY50kcCoC1cKS0LRUsxq_xA89RNc,1064
+fromscratch_pysearch-0.3.0.dist-info/RECORD,,

fromscratch_pysearch-0.3.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.4
+Root-Is-Purelib: true
+Tag: py3-none-any

fromscratch_pysearch-0.3.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Rob Cox
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

pysearch/__init__.py

@@ -0,0 +1 @@
+"""PySearch -- educational full-text search engine built from scratch in Python."""

pysearch/_version.py

@@ -0,0 +1,24 @@
+# file generated by vcs-versioning
+# don't change, don't track in version control
+from __future__ import annotations
+
+__all__ = [
+    "__version__",
+    "__version_tuple__",
+    "version",
+    "version_tuple",
+    "__commit_id__",
+    "commit_id",
+]
+
+version: str
+__version__: str
+__version_tuple__: tuple[int | str, ...]
+version_tuple: tuple[int | str, ...]
+commit_id: str | None
+__commit_id__: str | None
+
+__version__ = version = '0.3.0'
+__version_tuple__ = version_tuple = (0, 3, 0)
+
+__commit_id__ = commit_id = None

pysearch/atomic.py

@@ -0,0 +1,66 @@
+"""Safe saving -- never leave a half-written file behind.
+
+The usual way to save a file is: open it, wipe it, write the new
+contents. If the computer crashes after the wipe but before the writing
+finishes, your data is gone -- the old version was wiped and the new one
+never arrived.
+
+So we save like a careful student copying out their homework:
+
+1. Write the new version on a **fresh sheet** (a temporary file sitting
+   right next to the real one).
+2. Make sure the **ink is dry** -- ``fsync`` forces the bytes out of the
+   computer's memory and onto the disk.
+3. **Swap** the fresh sheet in for the old one in a single instant step
+   (``Path.replace``). There is never a moment with no complete file: you
+   see either the old version or the new one, never half of each.
+
+Real databases (SQLite, PostgreSQL, Redis) all use this same trick.
+"""
+
+import contextlib
+import os
+import stat
+import tempfile
+from pathlib import Path
+
+# Permissions for a brand-new file: owner can read/write, others can read.
+_NEW_FILE_MODE = 0o644
+
+
+def atomic_write_text(path: Path, text: str) -> None:
+    """Replace the file at *path* with *text*, so a crash can never leave it half-written.
+
+    Args:
+        path: The file to create or replace (its folder must already exist).
+        text: The new contents, saved as UTF-8.
+
+    """
+    mode = stat.S_IMODE(path.stat().st_mode) if path.exists() else _NEW_FILE_MODE
+    fd, temp_name = tempfile.mkstemp(dir=path.parent, prefix=f".{path.name}.", suffix=".tmp")
+    temp_path = Path(temp_name)
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as temp_file:
+            temp_file.write(text)
+            temp_file.flush()
+            os.fsync(temp_file.fileno())
+        temp_path.chmod(mode)
+        temp_path.replace(path)
+    except BaseException:
+        temp_path.unlink(missing_ok=True)  # don't leave the scrap sheet lying around
+        raise
+    _sync_folder(path.parent)
+
+
+def _sync_folder(folder: Path) -> None:
+    """Make the swap itself durable by syncing the folder's list of files.
+
+    Some systems (like Windows) can't sync a folder; there the swap is
+    still safe, just not guaranteed to survive a power cut.
+    """
+    with contextlib.suppress(OSError):
+        fd = os.open(folder, os.O_RDONLY)
+        try:
+            os.fsync(fd)
+        finally:
+            os.close(fd)

pysearch/engine.py

@@ -0,0 +1,162 @@
+"""Tie every component together into a complete search engine.
+
+The search engine is the librarian herself. She knows how to file new books
+(add documents), understand your question (parse queries), look things up
+(search the index), and tell you which results matter most (rank by TF-IDF).
+"""
+
+from typing import Self, cast
+
+from pysearch.index import InvertedIndex
+from pysearch.query import AndQuery, SearchQuery, TermQuery, parse_query
+from pysearch.tfidf import rank_results
+
+# Bump this if the saved layout ever changes, so old files are refused, not misread.
+SAVE_FORMAT = 1
+
+
+class SearchEngine:
+    """A full-text search engine with TF-IDF ranking and boolean queries."""
+
+    def __init__(self) -> None:
+        """Initialise an empty search engine."""
+        self._index = InvertedIndex()
+        # The original text of every book, in the order it arrived, so the
+        # engine can be saved and rebuilt exactly.
+        self._documents: dict[str, str] = {}
+
+    def add(self, doc_id: str, text: str) -> None:
+        """Hand the librarian a new book to file away.
+
+        Args:
+            doc_id: A unique identifier for the document.
+            text: The raw document text.
+
+        """
+        self._index.add_document(doc_id, text)
+        self._documents[doc_id] = text
+
+    def to_dict(self) -> dict[str, object]:
+        """Return everything needed to rebuild this engine, as plain data.
+
+        Returns:
+            ``{"format": 1, "documents": {doc_id: text, ...}}``
+
+        """
+        return {"format": SAVE_FORMAT, "documents": dict(self._documents)}
+
+    @classmethod
+    def from_dict(cls, data: dict[str, object]) -> Self:
+        """Rebuild an engine from :meth:`to_dict` data by re-filing every book.
+
+        Args:
+            data: The saved data.
+
+        Returns:
+            A new engine holding the same documents.
+
+        Raises:
+            ValueError: If *data* is from an unknown format version.
+            TypeError: If *data* doesn't have the expected shape.
+
+        """
+        version = data.get("format")
+        # ``True == 1`` and ``1.0 == 1`` in Python, so insist on a real int.
+        if type(version) is not int or version != SAVE_FORMAT:
+            msg = f"unknown save format {version!r} (expected the integer {SAVE_FORMAT})"
+            raise ValueError(msg)
+        documents = data["documents"]
+        if not isinstance(documents, dict):
+            msg = "'documents' must be an object mapping ids to text"
+            raise TypeError(msg)
+        engine = cls()
+        for doc_id, text in cast("dict[object, object]", documents).items():
+            if not isinstance(doc_id, str) or not isinstance(text, str):
+                msg = "every document id and text must be a string"
+                raise TypeError(msg)
+            engine.add(doc_id, text)
+        return engine
+
+    def search(self, query_text: str) -> list[tuple[str, float]]:
+        """Ask the librarian a question and get back the best-matching books, best first.
+
+        Args:
+            query_text: A query string such as ``"cats"``, ``"cats AND dogs"``,
+                or ``"cats OR dogs"``.
+
+        Returns:
+            A list of ``(doc_id, score)`` tuples sorted by relevance.
+
+        """
+        query = parse_query(query_text)
+        doc_ids = self._execute(query)
+
+        if not doc_ids:
+            return []
+
+        # Rank by ALL terms in the query so every word counts.
+        all_terms = self._collect_terms(query)
+        return rank_results(all_terms, doc_ids, self._index)
+
+    @property
+    def document_count(self) -> int:
+        """Return the total number of indexed documents."""
+        return self._index.document_count
+
+    def _execute(self, query: SearchQuery) -> list[str]:
+        """Work out which documents match a parsed query tree.
+
+        A long question makes a tall tree (2,000 words, 2,000 levels), too
+        tall for a function that calls itself once per level -- Python stops
+        at about 1,000 calls deep. So instead of calling itself, this keeps
+        its own to-do pile: each branch is visited once on the way down
+        (to queue its two halves) and once on the way back up (to join
+        their answers).
+
+        Args:
+            query: A parsed ``SearchQuery`` tree.
+
+        Returns:
+            A list of matching document IDs.
+
+        """
+        to_do: list[tuple[SearchQuery, bool]] = [(query, False)]
+        answers: list[set[str]] = []
+        while to_do:
+            node, halves_done = to_do.pop()
+            if isinstance(node, TermQuery):
+                answers.append(set(self._index.search(node.term)))
+            elif not halves_done:
+                # Come back to this branch once both halves are answered.
+                to_do.extend([(node, True), (node.right, False), (node.left, False)])
+            else:
+                right = answers.pop()
+                left = answers.pop()
+                answers.append(left & right if isinstance(node, AndQuery) else left | right)
+        return list(answers[0])
+
+    @staticmethod
+    def _collect_terms(query: SearchQuery) -> list[str]:
+        """Collect every term from a query tree for ranking, left to right.
+
+        Walk the whole tree and gather every leaf -- like picking every
+        apple off a tree instead of just the first one you see. Like
+        :meth:`_execute`, it keeps its own to-do pile so a tall tree can't
+        make it crash.
+
+        Args:
+            query: A parsed ``SearchQuery`` tree.
+
+        Returns:
+            A list of all term strings in the query.
+
+        """
+        terms: list[str] = []
+        to_do: list[SearchQuery] = [query]
+        while to_do:
+            node = to_do.pop()
+            if isinstance(node, TermQuery):
+                terms.append(node.term)
+            else:
+                to_do.extend([node.right, node.left])  # left comes off the pile first
+        return terms

pysearch/errors.py

@@ -0,0 +1,22 @@
+"""Search engine error types."""
+
+
+class PySearchError(Exception):
+    """Base exception for all PySearch errors."""
+
+
+class QueryParseError(PySearchError):
+    """Raise when a search query cannot be parsed."""
+
+
+class DuplicateDocumentError(PySearchError):
+    """Raise when a document id is added to the index a second time."""
+
+
+class PersistenceError(PySearchError):
+    """Raise when a saved search engine can't be read back.
+
+    Like opening the librarian's notebook and finding the pages torn or
+    scribbled on -- the file is missing, isn't valid JSON, or isn't a saved
+    engine at all.
+    """

pysearch/index.py

@@ -0,0 +1,127 @@
+"""Build and query an inverted index.
+
+An inverted index is like the librarian's memory. Instead of reading every
+book to find a word, the librarian keeps a notebook: for each word, she writes
+down which books contain it and how many times it appears. When you ask her
+about a word, she just flips to that page in her notebook -- instant answer!
+"""
+
+from collections import defaultdict
+
+from pysearch.errors import DuplicateDocumentError
+from pysearch.tokenizer import simple_stem, tokenize
+
+
+class InvertedIndex:
+    """Map terms to the documents that contain them.
+
+    The index stores two things for every (term, document) pair:
+    - which documents contain the term (for searching)
+    - how many times the term appears in each document (for ranking)
+    """
+
+    def __init__(self) -> None:
+        """Initialise an empty inverted index."""
+        self._index: dict[str, dict[str, int]] = defaultdict(lambda: defaultdict(int))
+        self._doc_ids: set[str] = set()
+
+    def add_document(self, doc_id: str, text: str) -> None:
+        """File a new book away -- read its words and note which ones live in this document.
+
+        Args:
+            doc_id: A unique identifier for the document.
+            text: The raw document text.
+
+        Raises:
+            DuplicateDocumentError: If a document with this id was already
+                added -- filing the same book twice would make every word in
+                it look more common than it really is.
+
+        """
+        if doc_id in self._doc_ids:
+            msg = f"Document {doc_id!r} is already in the index"
+            raise DuplicateDocumentError(msg)
+        self._doc_ids.add(doc_id)
+        tokens = tokenize(text)
+        for token in tokens:
+            self._index[token][doc_id] += 1
+
+    def search(self, term: str) -> list[str]:
+        """Ask the librarian which books contain a word -- she flips to that page and reads the list.
+
+        Args:
+            term: A single search term (will be stemmed).
+
+        Returns:
+            A list of matching document IDs.
+
+        """
+        stemmed = simple_stem(term.lower())
+        return list(self._index.get(stemmed, {}))
+
+    def search_and(self, terms: list[str]) -> list[str]:
+        """Find books that mention EVERY word you asked for -- both "cats" and "dogs", not just one.
+
+        Args:
+            terms: A list of search terms.
+
+        Returns:
+            Document IDs present in every term's posting list (intersection).
+
+        """
+        if not terms:
+            return []
+        sets = [set(self.search(t)) for t in terms]
+        result = sets[0]
+        for s in sets[1:]:
+            result &= s
+        return list(result)
+
+    def search_or(self, terms: list[str]) -> list[str]:
+        """Find books that mention ANY of your words -- "cats" or "dogs" or both will do.
+
+        Args:
+            terms: A list of search terms.
+
+        Returns:
+            Document IDs present in at least one term's posting list (union).
+
+        """
+        if not terms:
+            return []
+        result: set[str] = set()
+        for t in terms:
+            result |= set(self.search(t))
+        return list(result)
+
+    @property
+    def document_count(self) -> int:
+        """Return the total number of indexed documents."""
+        return len(self._doc_ids)
+
+    def get_term_frequency(self, term: str, doc_id: str) -> int:
+        """Return how many times *term* appears in *doc_id*.
+
+        Args:
+            term: The search term (will be stemmed).
+            doc_id: The document identifier.
+
+        Returns:
+            The number of occurrences, or 0 if not found.
+
+        """
+        stemmed = simple_stem(term.lower())
+        return self._index.get(stemmed, {}).get(doc_id, 0)
+
+    def get_document_frequency(self, term: str) -> int:
+        """Return how many documents contain *term*.
+
+        Args:
+            term: The search term (will be stemmed).
+
+        Returns:
+            The number of documents containing the term.
+
+        """
+        stemmed = simple_stem(term.lower())
+        return len(self._index.get(stemmed, {}))

pysearch/persistence.py

@@ -0,0 +1,65 @@
+"""Save a search engine to disk and load it back.
+
+Closing the library for the night? The librarian writes every book's text
+into a notebook (a JSON file). Tomorrow she reads the notebook and files
+every book again, so every search gives the same answers as before.
+
+Saving uses :func:`pysearch.atomic.atomic_write_text`, so a crash halfway
+through can never leave a half-written notebook.
+"""
+
+import json
+from pathlib import Path
+from typing import cast
+
+from pysearch.atomic import atomic_write_text
+from pysearch.engine import SearchEngine
+from pysearch.errors import PersistenceError
+
+
+def save_engine(engine: SearchEngine, path: str | Path) -> None:
+    """Save *engine* to a JSON file at *path*.
+
+    Args:
+        engine: The engine to save.
+        path: Where to write it (its folder must already exist).
+
+    """
+    atomic_write_text(Path(path), json.dumps(engine.to_dict(), indent=2))
+
+
+def load_engine(path: str | Path) -> SearchEngine:
+    """Load an engine saved by :func:`save_engine`.
+
+    Args:
+        path: The file to read.
+
+    Returns:
+        A new engine holding the saved documents.
+
+    Raises:
+        PersistenceError: If the file is missing, unreadable, not text, not
+            valid JSON, or not a saved engine.
+
+    """
+    file_path = Path(path)
+    try:
+        raw = file_path.read_text(encoding="utf-8")
+    except (OSError, UnicodeDecodeError) as exc:  # missing, unreadable, or not text at all
+        msg = f"Failed to load search engine from {file_path}: {exc}"
+        raise PersistenceError(msg) from exc
+    # Not JSON (JSONDecodeError is a kind of ValueError), a number too long
+    # for Python to read (ValueError), or lists nested too deep (RecursionError).
+    try:
+        data = json.loads(raw)
+    except (ValueError, RecursionError) as exc:
+        msg = f"Invalid JSON in {file_path}: {exc}"
+        raise PersistenceError(msg) from exc
+    if not isinstance(data, dict):
+        msg = f"{file_path} is not a saved search engine"
+        raise PersistenceError(msg)
+    try:
+        return SearchEngine.from_dict(cast("dict[str, object]", data))
+    except (KeyError, TypeError, ValueError) as exc:
+        msg = f"{file_path} is not a saved search engine: {exc}"
+        raise PersistenceError(msg) from exc

pysearch/query.py

@@ -0,0 +1,118 @@
+"""Parse search queries into a structured representation.
+
+Understanding your question is the first step to answering it. The query
+parser reads what you typed -- like ``cats AND dogs`` -- and turns it into a
+tree that the search engine can execute step by step.
+"""
+
+from dataclasses import dataclass
+
+from pysearch.errors import QueryParseError
+from pysearch.tokenizer import split_words
+
+
+@dataclass
+class TermQuery:
+    """A query for a single search term."""
+
+    term: str
+
+
+@dataclass
+class AndQuery:
+    """A query requiring both the left and right sub-queries to match."""
+
+    left: TermQuery | AndQuery | OrQuery
+    right: TermQuery | AndQuery | OrQuery
+
+
+@dataclass
+class OrQuery:
+    """A query requiring either the left or right sub-query to match."""
+
+    left: TermQuery | AndQuery | OrQuery
+    right: TermQuery | AndQuery | OrQuery
+
+
+SearchQuery = TermQuery | AndQuery | OrQuery
+
+
+_OPERATORS = frozenset({"AND", "OR"})
+
+
+def parse_query(text: str) -> SearchQuery:
+    """Read what you typed -- like "cats AND dogs" -- and turn it into a plan the engine can follow.
+
+    Supported formats:
+
+    - ``"word"`` -- a single term
+    - ``"a AND b"`` / ``"a OR b"`` -- both / either must match
+    - ``"a b"`` -- words next to each other must *all* match (an implied AND)
+    - longer chains like ``"a AND b OR c"`` are read left to right:
+      ``(a AND b) OR c``
+
+    Each word is cleaned up exactly like document text (see
+    :func:`~pysearch.tokenizer.split_words`): punctuation and stop words go,
+    and a word that splits into pieces (``"state-of-the-art"``) needs all of
+    its pieces.
+
+    Args:
+        text: The raw query string.
+
+    Returns:
+        A ``SearchQuery`` representing the parsed query.
+
+    Raises:
+        QueryParseError: If the query is empty, has nothing searchable, or
+            has an operator without a word on both sides.
+
+    """
+    words = text.split()
+    if not words:
+        msg = "Query must not be empty."
+        raise QueryParseError(msg)
+
+    result: SearchQuery | None = None
+    pending_operator = "AND"
+    expecting_term = True
+    for word in words:
+        operator = word.upper()
+        if operator in _OPERATORS:
+            if expecting_term:
+                msg = f"The {operator} operator needs a word on both sides: {text!r}"
+                raise QueryParseError(msg)
+            pending_operator = operator
+            expecting_term = True
+            continue
+        term = _word_query(word)
+        if term is None:
+            continue  # a stop word or bare punctuation -- nothing to search for
+        result = term if result is None else _combine(result, pending_operator, term)
+        pending_operator = "AND"
+        expecting_term = False
+
+    if expecting_term and result is not None:
+        msg = f"The {pending_operator} operator needs a word on both sides: {text!r}"
+        raise QueryParseError(msg)
+    if result is None:
+        msg = f"Query has no searchable words (only stop words?): {text!r}"
+        raise QueryParseError(msg)
+    return result
+
+
+def _word_query(word: str) -> SearchQuery | None:
+    """Turn one typed word into a query (several pieces must all match)."""
+    pieces = [TermQuery(term=piece) for piece in split_words(word)]
+    if not pieces:
+        return None
+    query: SearchQuery = pieces[0]
+    for piece in pieces[1:]:
+        query = AndQuery(left=query, right=piece)
+    return query
+
+
+def _combine(left: SearchQuery, operator: str, right: SearchQuery) -> SearchQuery:
+    """Join two queries with AND or OR."""
+    if operator == "OR":
+        return OrQuery(left=left, right=right)
+    return AndQuery(left=left, right=right)

pysearch/tfidf.py

@@ -0,0 +1,94 @@
+"""Compute TF-IDF relevance scores.
+
+TF-IDF stands for Term Frequency - Inverse Document Frequency. It answers the
+question: "How important is this word to this particular document?" A word
+that appears many times in one document but rarely in others gets a high
+score -- it is probably what that document is *about*.
+"""
+
+import math
+
+from pysearch.index import InvertedIndex
+
+
+def compute_tf(term: str, doc_id: str, index: InvertedIndex) -> float:
+    """Tally how many times a word shows up in one document -- more mentions, more relevant.
+
+    Args:
+        term: The search term.
+        doc_id: The document identifier.
+        index: The inverted index to query.
+
+    Returns:
+        The raw count of *term* in *doc_id* as a float.
+
+    """
+    return float(index.get_term_frequency(term, doc_id))
+
+
+def compute_idf(term: str, index: InvertedIndex) -> float:
+    """Figure out how special a word is -- common words like "the" score near zero, rare words like "dinosaur" score high.
+
+    Uses the formula ``log(N / df)`` where *N* is the total number of
+    documents and *df* is the number of documents containing *term*. Note that a
+    word appearing in *every* document (or the only document) scores exactly
+    ``0.0`` -- a fun exercise is to smooth this to ``log(N / df) + 1`` so even a
+    lone match keeps a small positive score.
+
+    Args:
+        term: The search term.
+        index: The inverted index to query.
+
+    Returns:
+        The IDF value, or 0.0 if the term appears in no documents.
+
+    """
+    df = index.get_document_frequency(term)
+    if df == 0:
+        return 0.0
+    return math.log(index.document_count / df)
+
+
+def compute_tfidf(term: str, doc_id: str, index: InvertedIndex) -> float:
+    """Blend "how often" with "how special" into one relevance score for a word in a document.
+
+    Args:
+        term: The search term.
+        doc_id: The document identifier.
+        index: The inverted index to query.
+
+    Returns:
+        The product of TF and IDF.
+
+    """
+    return compute_tf(term, doc_id, index) * compute_idf(term, index)
+
+
+def rank_results(
+    terms: str | list[str],
+    doc_ids: list[str],
+    index: InvertedIndex,
+) -> list[tuple[str, float]]:
+    """Rank documents by their TF-IDF score across all *terms*, highest first.
+
+    Think of it like a school report card: each subject (term) gives a grade
+    (TF-IDF score), and we add them all up to get the final grade. A document
+    that scores well on *every* search word floats to the top.
+
+    Args:
+        terms: A single search term or a list of terms.
+        doc_ids: The candidate document IDs to rank.
+        index: The inverted index to query.
+
+    Returns:
+        A list of ``(doc_id, score)`` tuples sorted by score descending.
+
+    """
+    term_list = [terms] if isinstance(terms, str) else terms
+    scored = [
+        (doc_id, sum(compute_tfidf(t, doc_id, index) for t in term_list)) for doc_id in doc_ids
+    ]
+    # Best score first; equal scores are listed by document id so the
+    # order never depends on luck.
+    scored.sort(key=lambda pair: (-pair[1], pair[0]))
+    return scored

pysearch/tokenizer.py

@@ -0,0 +1,96 @@
+"""Tokenize text into searchable words.
+
+Think of a tokenizer like cutting sentences into word cards. You take a long
+sentence, snip it into individual words, throw away boring words like "the"
+and "is," and keep the interesting ones. Each card can then be filed in the
+index so you can find it later.
+"""
+
+import re
+
+STOP_WORDS: frozenset[str] = frozenset(
+    {
+        "the",
+        "a",
+        "an",
+        "is",
+        "are",
+        "was",
+        "were",
+        "in",
+        "on",
+        "at",
+        "to",
+        "for",
+        "of",
+        "and",
+        "or",
+        "but",
+        "not",
+        "it",
+        "this",
+        "that",
+        "with",
+        "from",
+        "by",
+    }
+)
+
+_SUFFIX_ORDER: tuple[str, ...] = ("ing", "ed", "ly", "er", "est", "es", "s")
+_MIN_STEM_LENGTH = 3
+
+
+def split_words(text: str) -> list[str]:
+    """Cut text into clean words -- lowercased, punctuation gone, boring words tossed.
+
+    This is the first half of :func:`tokenize`, without the stemming. Search
+    queries use it too, so a query word is cleaned up *exactly* the way the
+    documents were: ``"fox,"`` becomes ``"fox"``, and ``"state-of-the-art"``
+    becomes ``["state", "art"]``.
+
+    Args:
+        text: The raw text to split.
+
+    Returns:
+        A list of lowercase words with stop words removed (not yet stemmed).
+
+    """
+    words = re.split(r"[^a-zA-Z0-9]+", text.lower())
+    return [w for w in words if w and w not in STOP_WORDS]
+
+
+def tokenize(text: str) -> list[str]:
+    """Cut a sentence into clean word cards -- lowercased, punctuation gone, boring words tossed.
+
+    Args:
+        text: The raw text to tokenize.
+
+    Returns:
+        A list of stemmed, lowercase tokens with stop words removed.
+
+    """
+    return [simple_stem(w) for w in split_words(text)]
+
+
+def simple_stem(word: str) -> str:
+    """Trim a word down to its root so "foxes" and "fox" count as the same word.
+
+    We only chop an ending off if enough of the word is left over (at least 3
+    letters), so tiny words like "bed" stay whole. Plural "-ies" becomes "-y"
+    ("berries" -> "berry"), plural "-es" is dropped ("foxes" -> "fox"), and the
+    other endings checked in order are: "ing", "ed", "ly", "er", "est", "s".
+
+    Args:
+        word: The word to stem.
+
+    Returns:
+        The stemmed word.
+
+    """
+    # Plural "-ies" -> "-y" so "berries" and "berry" collapse to the same stem.
+    if word.endswith("ies") and len(word) - len("ies") >= _MIN_STEM_LENGTH - 1:
+        return word[:-3] + "y"
+    for suffix in _SUFFIX_ORDER:
+        if word.endswith(suffix) and len(word) - len(suffix) >= _MIN_STEM_LENGTH:
+            return word[: -len(suffix)]
+    return word