passageway 0.3.0__py3-none-any.whl

passageway/chunk.py

@@ -0,0 +1,68 @@
+"""Turns an item into the pieces that are embedded and returned as passages."""
+
+import re
+from dataclasses import dataclass
+
+from passageway.protocol import Item
+
+MAX_CHARS = 1500  # a section over this is split; the embedding model cuts at a few hundred tokens anyway
+
+
+@dataclass(frozen=True)
+class Chunk:
+    heading: str
+    text: str
+    ordinal: int
+
+
+def _cut_words(paragraph: str, cap: int) -> list[str]:
+    out, line = [], ""
+    for word in paragraph.split():
+        while len(word) > cap:
+            if line:
+                out.append(line)
+                line = ""
+            out.append(word[:cap])
+            word = word[cap:]
+        if line and len(line) + 1 + len(word) > cap:
+            out.append(line)
+            line = word
+        else:
+            line = f"{line} {word}" if line else word
+    if line:
+        out.append(line)
+    return out
+
+
+def split_text(text: str, cap: int = MAX_CHARS) -> list[str]:
+    """Pieces of at most `cap` characters, cut at blank lines first, then between words."""
+    text = text.strip()
+    if len(text) <= cap:
+        return [text] if text else []
+    pieces, current = [], ""
+    for paragraph in re.split(r"\n\s*\n", text):
+        paragraph = paragraph.strip()
+        for part in [paragraph] if len(paragraph) <= cap else _cut_words(paragraph, cap):
+            if not part:
+                continue
+            if current and len(current) + 2 + len(part) > cap:
+                pieces.append(current)
+                current = part
+            else:
+                current = f"{current}\n\n{part}" if current else part
+    if current:
+        pieces.append(current)
+    return pieces
+
+
+def chunks(item: Item) -> list[Chunk]:
+    out: list[Chunk] = []
+    for section in item.sections:
+        for piece in split_text(section.text):
+            out.append(Chunk(section.heading or "", piece, len(out)))
+    return out
+
+
+def passage_text(item: Item, chunk: Chunk) -> str:
+    trail = " › ".join([*item.path, *([chunk.heading] if chunk.heading else [])])
+    return f"{item.title} › {trail}: {chunk.text}" if trail else f"{item.title}: {chunk.text}"

passageway/config.py

@@ -0,0 +1,315 @@
+"""Configuration: environment variables for the core, one TOML file for connectors (docs/install.md)."""
+
+import math
+import re
+import sys
+import threading
+import tomllib
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from mcp.server.auth.routes import validate_issuer_url
+from pydantic import AnyHttpUrl
+
+from passageway.protocol import MAX_LINE_BYTES, MAX_OUTPUT_BYTES
+
+
+class ConfigError(Exception):
+    pass
+
+
+DEFAULT_TOOL_DESCRIPTION = (
+    "Search the user's indexed sources (documents, notes, decisions, how their systems and tools actually "
+    "behave) and get up to 3 short passages with links. Call it on your own before a task the sources may "
+    "cover, when you hit an unfamiliar error, host or service, and before making a decision they may already "
+    "have made. Returns nothing-relevant when there is no match. Open the link for the full document.")
+
+DEFAULT_INSTRUCTIONS = (
+    "This server searches the user's indexed sources: their documents, notes, decisions and how their systems "
+    "and tools actually behave. Call `recall` before starting a task the sources may cover, when you hit an "
+    "unfamiliar error, host or service, and before making a decision they may already have made, even when you "
+    "think you know the answer: the sources hold what is true for this user. A hit is a pointer: open its link "
+    "for the full document before relying on it. If the context already holds passages headed \"Recall results\", "
+    "those came from this server; don't search for the same thing again.")
+
+_NAME = re.compile(r"[a-z0-9][a-z0-9_-]*")
+
+
+@dataclass(frozen=True)
+class ConnectorConfig:
+    name: str
+    command: tuple[str, ...]
+    env: dict[str, str]
+    secrets: dict[str, str]  # already resolved from env:NAME references
+    timeout: float = 300.0
+    max_line_bytes: int = MAX_LINE_BYTES
+    max_output_bytes: int = MAX_OUTPUT_BYTES
+    max_delete_fraction: float = 0.3  # a reconcile that would delete more than this share of a source is refused
+
+
+EMBEDDER_KINDS = ("onnx", "command")
+_ONNX_KEYS = {"kind", "min_similarity", "model", "model_dir", "device", "download"}
+_COMMAND_KEYS = {"kind", "command", "env", "secrets", "min_similarity", "batch_size", "timeout"}
+
+
+@dataclass(frozen=True)
+class EmbedderConfig:
+    """How texts become vectors: the built-in ONNX model, or any program speaking the embedder protocol."""
+    kind: str = "onnx"
+    command: tuple[str, ...] = ()
+    env: dict[str, str] = field(default_factory=dict)
+    secrets: dict[str, str] = field(default_factory=dict)  # already resolved from env:NAME references
+    min_similarity: float | None = None  # the relevance gate of this model; None keeps the built-in default
+    batch_size: int = 32  # texts per request
+    timeout: float = 60.0  # seconds per request
+    model_dir: Path | None = None  # onnx only; None: <data dir>/models/<model>
+    model: str = "multilingual-e5-small"  # onnx only: a preset of passageway.models
+    device: str = "auto"  # onnx only: auto | cpu | coreml
+    download: bool = True  # onnx only: fetch the pinned, checksummed files when they are missing
+
+
+@dataclass
+class Config:
+    password: str  # PASSAGEWAY_LOGIN_PASSWORD: static Bearer token and the OAuth login page's password
+    connectors: list[ConnectorConfig]
+    data_dir: Path
+    model_dir: Path
+    base_url: str | None = None  # public URL; set, it turns on OAuth for /mcp
+    host: str = "127.0.0.1"
+    port: int = 8082
+    poll_seconds: int = 60
+    reconcile_seconds: int = 1800
+    log_level: str = "INFO"
+    tool_description: str = DEFAULT_TOOL_DESCRIPTION
+    instructions: str = DEFAULT_INSTRUCTIONS
+    instance: str | None = None  # PASSAGEWAY_INSTANCE: one of several servers on this machine
+    models_root: Path | None = None  # where the built-in models live; None: data_dir (instances share the default)
+    min_similarity: float | None = None  # the relevance gate; None keeps the embedder preset's value
+    embedder: EmbedderConfig = field(default_factory=EmbedderConfig)  # the one the server uses
+    embedders: dict[str, EmbedderConfig] = field(default_factory=dict)  # candidates for `passageway eval compare`
+
+
+_INSTANCE = r"[a-z0-9][a-z0-9-]*"
+
+
+def instance_name(env: Mapping[str, str]) -> str | None:
+    """PASSAGEWAY_INSTANCE names one of several servers on this machine; unset, the one and only."""
+    name = env.get("PASSAGEWAY_INSTANCE", "").strip()
+    if name and not re.fullmatch(_INSTANCE, name):
+        raise ConfigError(f"PASSAGEWAY_INSTANCE must be lowercase letters, digits and hyphens, got {name!r}")
+    return name or None
+
+
+def take_instance_argument(argv: list[str]) -> tuple[list[str], str | None]:
+    """`--instance NAME` or `--instance=NAME`, anywhere in the command line: the rest of it and the name."""
+    rest, name, args = [], None, iter(argv)
+    for arg in args:
+        if arg == "--instance":
+            name = next(args, "")
+        elif arg.startswith("--instance="):
+            name = arg.split("=", 1)[1]
+        else:
+            rest.append(arg)
+            continue
+        if not re.fullmatch(_INSTANCE, name):
+            raise ConfigError(f"--instance needs a name of lowercase letters, digits and hyphens, got {name!r}")
+    return rest, name
+
+
+def existing_instances() -> list[str]:
+    return sorted(p.parent.name.removeprefix("passageway-") for p in (Path.home() / ".config").glob("passageway-*/passageway.toml"))
+
+
+def instance_problem(env: Mapping[str, str]) -> str | None:
+    """Why this command would run against an instance that is not there, or None. A named instance must have
+    been created by init; with none named and no default config, the named ones are what the user meant."""
+    if env.get("PASSAGEWAY_CONFIG"):
+        return None
+    name, known = instance_name(env), existing_instances()
+    listed = ", ".join(known) or "none"
+    if name and not default_config_path(name).exists():
+        return (f"there is no passageway instance {name!r} (instances on this machine: {listed}); "
+                f"create it with: passageway --instance {name} init")
+    if not name and known and not default_config_path().exists():
+        return f"this machine has named passageway instances ({listed}): pick one with --instance NAME"
+    return None
+
+
+def _suffix(instance: str | None) -> str:
+    return f"-{instance}" if instance else ""
+
+
+def default_config_path(instance: str | None = None) -> Path:
+    return Path.home() / ".config" / f"passageway{_suffix(instance)}" / "passageway.toml"
+
+
+def default_data_dir(instance: str | None = None) -> Path:
+    """The data folder; an instance gets its own next to the shared one, which keeps the built-in models."""
+    base = Path.home() / "Library" / "Application Support" if sys.platform == "darwin" else Path.home() / ".local" / "share"
+    return base / f"passageway{_suffix(instance)}"
+
+
+def _int(env: Mapping[str, str], key: str, default: int) -> int:
+    raw = env.get(key)
+    if not raw:
+        return default
+    try:
+        return int(raw)
+    except ValueError:
+        raise ConfigError(f"{key} must be an integer, got {raw!r}") from None
+
+
+def _resolve(connector: str, ref: object, env: Mapping[str, str], what: str = "connector") -> str:
+    if not isinstance(ref, str) or not ref.startswith("env:") or len(ref) == 4:
+        raise ConfigError(f"{what} {connector!r}: a secret must be a reference like \"env:NAME\" (in 'secrets')")
+    var = ref[4:]
+    if not env.get(var):
+        raise ConfigError(f"{what} {connector!r}: the environment variable {var} is not set")
+    return env[var]
+
+
+def _table(connector: str, raw: dict, key: str, what: str = "connector") -> dict[str, str]:
+    table = raw.get(key, {})
+    if not isinstance(table, dict) or not all(isinstance(v, str) for v in table.values()):
+        raise ConfigError(f"{what} {connector!r}: '{key}' must be a table of strings")
+    return table
+
+
+def _embedder(name: str, raw: object, env: Mapping[str, str]) -> EmbedderConfig:
+    where = f"embedder {name!r}"
+    if not isinstance(raw, dict):
+        raise ConfigError(f"{where}: must be a table")
+    kind = raw.get("kind", "onnx")
+    if kind not in EMBEDDER_KINDS:
+        raise ConfigError(f"{where}: 'kind' must be one of {', '.join(EMBEDDER_KINDS)}, got {kind!r}")
+    allowed = _ONNX_KEYS if kind == "onnx" else _COMMAND_KEYS
+    for key in raw:
+        if key not in allowed:
+            raise ConfigError(f"{where}: unknown key {key!r} for kind {kind!r} (known: {', '.join(sorted(allowed))})")
+    gate = raw.get("min_similarity")
+    if gate is not None:
+        if isinstance(gate, bool) or not isinstance(gate, (int, float)) or not 0 <= gate <= 1:
+            raise ConfigError(f"{where}: 'min_similarity' must be a number from 0 to 1")
+        gate = float(gate)
+    if kind == "onnx":
+        from passageway.models import DEVICES, PRESETS
+
+        model_dir = raw.get("model_dir")
+        if model_dir is not None and not isinstance(model_dir, str):
+            raise ConfigError(f"{where}: 'model_dir' must be a path")
+        model = raw.get("model", "multilingual-e5-small")
+        if not isinstance(model, str) or model not in PRESETS:
+            raise ConfigError(f"{where}: 'model' must be one of the presets {', '.join(PRESETS)}, got {model!r} "
+                              "(another model runs as kind = \"command\")")
+        device = raw.get("device", "auto")
+        if device not in DEVICES:
+            raise ConfigError(f"{where}: 'device' must be one of {', '.join(DEVICES)}, got {device!r}")
+        auto = raw.get("download", True)
+        if not isinstance(auto, bool):
+            raise ConfigError(f"{where}: 'download' must be true or false")
+        return EmbedderConfig("onnx", min_similarity=gate, model_dir=Path(model_dir) if model_dir else None, model=model,
+                              device=device, download=auto)
+    command = raw.get("command")
+    if not isinstance(command, list) or not command or not all(isinstance(c, str) for c in command):
+        raise ConfigError(f"{where}: 'command' must be a non-empty list of strings")
+    secrets = {k: _resolve(name, v, env, "embedder") for k, v in _table(name, raw, "secrets", "embedder").items()}
+    batch = raw.get("batch_size", 32)
+    if isinstance(batch, bool) or not isinstance(batch, int) or not 1 <= batch <= 1024:
+        raise ConfigError(f"{where}: 'batch_size' must be a whole number from 1 to 1024")
+    timeout = raw.get("timeout", 60.0)
+    if (isinstance(timeout, bool) or not isinstance(timeout, (int, float)) or not math.isfinite(timeout) or timeout <= 0
+            or timeout > threading.TIMEOUT_MAX):
+        raise ConfigError(f"{where}: 'timeout' must be a number of seconds greater than 0 (and not absurdly large)")
+    return EmbedderConfig("command", tuple(command), _table(name, raw, "env", "embedder"), secrets, gate, batch, float(timeout))
+
+
+def _number(connector: str, raw: dict, key: str, default, *, kind=float, low=None, high=None):
+    value = raw.get(key, default)
+    ok = (isinstance(value, (int, float)) and not isinstance(value, bool) and (kind is float or isinstance(value, int))
+          and math.isfinite(value))
+    if not ok or (low is not None and value <= low) or (high is not None and value > high):
+        bound = f" (greater than {low}{'' if high is None else f' and at most {high}'})" if low is not None else ""
+        raise ConfigError(f"connector {connector!r}: '{key}' must be a number{bound}")
+    return kind(value)
+
+
+def _load_toml(path: Path, env: Mapping[str, str]) -> tuple[list[ConnectorConfig], EmbedderConfig, dict[str, EmbedderConfig]]:
+    try:
+        data = tomllib.loads(path.read_text())
+    except FileNotFoundError:
+        raise ConfigError(f"{path} not found: list your connectors in a passageway.toml (see docs/install.md)") from None
+    except tomllib.TOMLDecodeError as e:
+        raise ConfigError(f"{path}: {e}") from None
+    connectors: list[ConnectorConfig] = []
+    entries = data.get("connector", [])
+    if not isinstance(entries, list) or not all(isinstance(e, dict) for e in entries):
+        raise ConfigError(f"{path}: connectors are written as [[connector]] tables, one per source")
+    for raw in entries:
+        name = raw.get("name", "")
+        if not isinstance(name, str) or not _NAME.fullmatch(name):
+            raise ConfigError(f"connector name {name!r} must match [a-z0-9][a-z0-9_-]* (it names the index file)")
+        if any(c.name == name for c in connectors):
+            raise ConfigError(f"duplicate connector name {name!r}")
+        command = raw.get("command")
+        if not isinstance(command, list) or not command or not all(isinstance(c, str) for c in command):
+            raise ConfigError(f"connector {name!r}: 'command' must be a non-empty list of strings")
+        secrets = {k: _resolve(name, v, env) for k, v in _table(name, raw, "secrets").items()}
+        connectors.append(ConnectorConfig(
+            name, tuple(command), _table(name, raw, "env"), secrets,
+            _number(name, raw, "timeout", 300.0, low=0, high=threading.TIMEOUT_MAX),
+            _number(name, raw, "max_line_bytes", MAX_LINE_BYTES, kind=int, low=0),
+            _number(name, raw, "max_output_bytes", MAX_OUTPUT_BYTES, kind=int, low=0),
+            _number(name, raw, "max_delete_fraction", 0.3, low=0, high=1)))
+    if not connectors:
+        raise ConfigError(f"no connectors in {path}: add at least one [[connector]]")
+    embedder = _embedder("default", data.get("embedder", {}), env)
+    candidates = data.get("embedders", {})
+    if not isinstance(candidates, dict):
+        raise ConfigError(f"{path}: 'embedders' holds the candidates for `passageway eval compare`, one [embedders.NAME] table each")
+    embedders: dict[str, EmbedderConfig] = {}
+    for name, raw in candidates.items():
+        if not _NAME.fullmatch(name):
+            raise ConfigError(f"embedder name {name!r} must match [a-z0-9][a-z0-9_-]*")
+        if name == "default":
+            raise ConfigError("the embedder name 'default' is taken: it stands for the [embedder] the server uses")
+        embedders[name] = _embedder(name, raw, env)
+    return connectors, embedder, embedders
+
+
+def load_config(env: Mapping[str, str], *, require_password: bool = True) -> Config:
+    """require_password=False is for commands that do not serve (the evaluation): nobody logs in to them."""
+    password = env.get("PASSAGEWAY_LOGIN_PASSWORD", "").strip()
+    password_file = env.get("PASSAGEWAY_LOGIN_PASSWORD_FILE", "").strip()
+    if not password and password_file:  # a file the service manager can point at (passageway init writes one)
+        try:
+            password = Path(password_file).read_text().strip()
+        except OSError as e:
+            raise ConfigError(f"PASSAGEWAY_LOGIN_PASSWORD_FILE={password_file}: {e.strerror or e}") from None
+        if not password:
+            raise ConfigError(f"PASSAGEWAY_LOGIN_PASSWORD_FILE={password_file} is empty")
+    if require_password and not password:
+        raise ConfigError("PASSAGEWAY_LOGIN_PASSWORD is not set (nor PASSAGEWAY_LOGIN_PASSWORD_FILE): clients authenticate "
+                          "with it, so the engine refuses to start without it; `passageway init` generates one")
+    base_url = env.get("PASSAGEWAY_BASE_URL", "").strip().rstrip("/") or None
+    if base_url:
+        try:
+            validate_issuer_url(AnyHttpUrl(base_url))
+        except ValueError as e:
+            raise ConfigError(f"PASSAGEWAY_BASE_URL={base_url!r} is not a valid OAuth issuer "
+                              f"(https, no query or fragment; plain http only for localhost): {e}") from None
+    instance = instance_name(env)
+    config_path = Path(env.get("PASSAGEWAY_CONFIG") or default_config_path(instance))
+    connectors, embedder, embedders = _load_toml(config_path, env)
+    data_dir = Path(env.get("PASSAGEWAY_DATA_DIR") or default_data_dir(instance))
+    models_root = Path(env.get("PASSAGEWAY_DATA_DIR") or default_data_dir())
+    return Config(
+        password=password, connectors=connectors, data_dir=data_dir, instance=instance, models_root=models_root,
+        model_dir=Path(env.get("PASSAGEWAY_MODEL_DIR") or models_root / "models" / "e5-small"),
+        base_url=base_url, host=env.get("PASSAGEWAY_HOST", "127.0.0.1"),
+        port=_int(env, "PASSAGEWAY_PORT", 8082), poll_seconds=_int(env, "PASSAGEWAY_POLL_SECONDS", 60),
+        reconcile_seconds=_int(env, "PASSAGEWAY_RECONCILE_SECONDS", 1800),
+        log_level=env.get("PASSAGEWAY_LOG_LEVEL", "INFO"),
+        tool_description=env.get("PASSAGEWAY_TOOL_DESCRIPTION", "").strip() or DEFAULT_TOOL_DESCRIPTION,
+        instructions=env.get("PASSAGEWAY_INSTRUCTIONS", "").strip() or DEFAULT_INSTRUCTIONS,
+        min_similarity=embedder.min_similarity, embedder=embedder, embedders=embedders)

passageway/embed.py

@@ -0,0 +1,213 @@
+"""Text embedders: the built-in ONNX model, any program speaking the embedder protocol, and a hashing stand-in for tests."""
+
+import hashlib
+import logging
+import re
+from pathlib import Path
+from typing import Protocol
+
+import numpy as np
+
+from passageway.config import ConfigError, EmbedderConfig
+from passageway.models import DEFAULT_PRESET, PRESETS, ModelError, download, files_missing, model_folder_for, verify
+
+log = logging.getLogger(__name__)
+
+
+class EmbedderError(Exception):
+    """An embedder failed: a program of its own died or answered badly, or the model raised."""
+
+
+class EmbedderStartError(ConfigError):
+    """An embedder could not start. `public` is what an open endpoint may show; the message is for the log."""
+
+    def __init__(self, message: str, public: str):
+        super().__init__(message)
+        self.public = public
+
+
+class Embedder(Protocol):
+    dim: int
+    name: str
+
+    def embed_queries(self, texts: list[str]) -> np.ndarray: ...
+
+    def embed_passages(self, texts: list[str]) -> np.ndarray: ...
+
+
+def _normalise(v: np.ndarray) -> np.ndarray:
+    norms = np.linalg.norm(v, axis=1, keepdims=True)
+    norms[norms == 0] = 1
+    return (v / norms).astype(np.float32)
+
+
+def token_batches(lengths: list[int], budget: int) -> list[list[int]]:
+    """Group indices so each batch's padded size (count × longest) stays within budget.
+
+    Memory at inference grows with the padded batch, so long passages go alone or in pairs
+    while short ones share a batch. Sorting by length keeps padding small.
+    """
+    batches: list[list[int]] = []
+    current: list[int] = []
+    for i in sorted(range(len(lengths)), key=lambda i: lengths[i]):
+        if current and (len(current) + 1) * lengths[i] > budget:
+            batches.append(current)
+            current = []
+        current.append(i)
+    if current:
+        batches.append(current)
+    return batches
+
+
+class OnnxEmbedder:
+    """A built-in model: an ONNX encoder and its tokenizer from a preset, mean pooling over the attention mask, L2-normalised."""
+
+    max_tokens = 512
+    # Padded tokens per inference call: 1024 keeps peak RSS near 310 MB for e5-small (8 × 512 reached 550 MB).
+    token_budget = 1024
+
+    def __init__(self, model_dir: Path, preset=None, device: str = "auto", threads: int = 4):
+        import onnxruntime as ort
+
+        from passageway.models import PRESETS, choose_device
+
+        self.preset = preset or PRESETS["multilingual-e5-small"]
+        self.name, self.dim, self.min_similarity = self.preset.label, self.preset.dim, self.preset.min_similarity
+        self.max_tokens = self.preset.max_tokens
+        options = ort.SessionOptions()
+        options.intra_op_num_threads = threads
+        options.inter_op_num_threads = 1
+        # The arena would keep the largest batch's peak reserved for the life of the process.
+        options.enable_cpu_mem_arena = False
+        providers = choose_device(device, ort.get_available_providers())
+        self._session = ort.InferenceSession(str(Path(model_dir) / "model.onnx"), options, providers=providers)
+        self._inputs = {i.name for i in self._session.get_inputs()}
+        self._encode_batch = self._tokenizer(self.preset, Path(model_dir))
+
+    @staticmethod
+    def _tokenizer(preset, model_dir: Path):
+        """A function from texts to (ids, mask) arrays, padded to the longest text and cut at max_tokens."""
+        if preset.tokenizer == "sentencepiece":
+            import sentencepiece as spm
+
+            # The SentencePiece model gives the same ids as the HF tokenizer.json at ~60 MB instead of ~280 MB.
+            sp = spm.SentencePieceProcessor(model_file=str(model_dir / "sentencepiece.bpe.model"))
+
+            def encode(text: str) -> list[int]:
+                # XLM-R ids: fairseq offset of 1 over SentencePiece, its <unk> (0) is 3, wrapped in <s> (0) … </s> (2).
+                ids = [3 if i == 0 else i + 1 for i in sp.encode(text)][: preset.max_tokens - 2]
+                return [0, *ids, 2]
+
+            pad = 1
+        else:
+            from tokenizers import Tokenizer
+
+            tok = Tokenizer.from_file(str(model_dir / "tokenizer.json"))
+            tok.enable_truncation(preset.max_tokens)
+            tok.no_padding()
+            pad = next((i for i in (tok.token_to_id(t) for t in ("[PAD]", "<pad>")) if i is not None), 0)
+
+            def encode(text: str) -> list[int]:
+                return tok.encode(text).ids
+
+        def batch(texts: list[str]) -> tuple[np.ndarray, np.ndarray]:
+            encoded = [encode(t) for t in texts]
+            width = max(len(e) for e in encoded)
+            ids = np.full((len(encoded), width), pad, dtype=np.int64)
+            mask = np.zeros((len(encoded), width), dtype=np.int64)
+            for row, e in enumerate(encoded):
+                ids[row, : len(e)] = e
+                mask[row, : len(e)] = 1
+            return ids, mask
+
+        return batch
+
+    def _embed(self, texts: list[str]) -> np.ndarray:
+        if not texts:
+            return np.zeros((0, self.dim), dtype=np.float32)
+        lengths = [len(self._encode_batch([t])[0][0]) for t in texts]
+        out = np.zeros((len(texts), self.dim), dtype=np.float32)
+        for batch in token_batches(lengths, self.token_budget):
+            ids, mask = self._encode_batch([texts[i] for i in batch])
+            feed = {"input_ids": ids, "attention_mask": mask}
+            if "token_type_ids" in self._inputs:
+                feed["token_type_ids"] = np.zeros_like(ids)
+            hidden = self._session.run(None, feed)[0]
+            weights = mask[..., None].astype(np.float32)
+            out[batch] = (hidden * weights).sum(axis=1) / np.maximum(weights.sum(axis=1), 1e-9)
+        return _normalise(out)
+
+    def embed_queries(self, texts: list[str]) -> np.ndarray:
+        return self._embed([self.preset.query_prefix + t for t in texts])
+
+    def embed_passages(self, texts: list[str]) -> np.ndarray:
+        return self._embed([self.preset.passage_prefix + t for t in texts])
+
+
+class HashEmbedder:
+    """Bag of lower-cased words hashed into buckets: overlap in words means similarity. Tests only."""
+
+    name = "hash-test"
+
+    def __init__(self, dim: int = 384):
+        self.dim = dim
+
+    def _embed(self, texts: list[str]) -> np.ndarray:
+        v = np.zeros((len(texts), self.dim), dtype=np.float32)
+        for row, text in enumerate(texts):
+            for word in re.findall(r"\w+", text.lower()):
+                bucket = int.from_bytes(hashlib.blake2b(word.encode(), digest_size=4).digest(), "big")
+                v[row, bucket % self.dim] += 1
+        return _normalise(v)
+
+    embed_queries = _embed
+    embed_passages = _embed
+
+
+def model_folder(cfg: EmbedderConfig, config) -> Path:
+    """Where a preset's files live: one rule with `passageway model` (models.model_folder_for); config.model_dir is the
+    server's resolved PASSAGEWAY_MODEL_DIR for the default preset."""
+    if cfg.model_dir:
+        return cfg.model_dir
+    if cfg.model == DEFAULT_PRESET:
+        return config.model_dir
+    return model_folder_for(cfg.model, config.models_root or config.data_dir, None)
+
+
+def build_embedder(cfg: EmbedderConfig, config, label: str = "default") -> Embedder:
+    """The embedder a configuration describes. A model or program that is not there is a ConfigError, so that a server
+    can start in its error mode and say what is wrong instead of crashing."""
+    if cfg.kind == "command":
+        from passageway.embed_command import CommandEmbedder
+
+        if cfg.min_similarity is None:
+            log.warning("embedder %r has no min_similarity: the relevance gate of the built-in model applies, which is "
+                        "probably wrong for this one; find a value with `passageway eval --sweep`", label)
+        try:
+            return CommandEmbedder(label, cfg.command, env=cfg.env, secrets=cfg.secrets, batch_size=cfg.batch_size,
+                                   timeout=cfg.timeout)
+        except EmbedderError as e:  # its output can hold hosts and paths: only the log gets it
+            raise EmbedderStartError(str(e), public=f"the embedder {label!r} could not start; see the server log") from None
+    preset = PRESETS[cfg.model]
+    folder = model_folder(cfg, config)
+    missing, wrong = files_missing(preset, folder), verify(preset, folder)
+    if missing or wrong:
+        if not cfg.download:
+            what = f"missing {', '.join(missing)}" if missing else f"a wrong checksum for {', '.join(wrong)}"
+            raise ConfigError(f"the model {cfg.model!r} in {folder} has {what}: run `passageway model download {cfg.model}` "
+                              "(download = true in [embedder] does it at start)")
+        log.info("fetching the model %s (%d MB) from huggingface.co into %s, pinned and checksummed; the server answers "
+                 "once it is there", cfg.model, preset.size_mb, folder)
+        try:
+            download(preset, folder, report=lambda m: log.info("%s", m))
+        except (ModelError, OSError) as e:
+            raise ConfigError(f"could not fetch the model {cfg.model!r}: {e}") from None
+    try:
+        embedder = OnnxEmbedder(folder, preset, cfg.device)
+    except ModelError as e:
+        raise ConfigError(str(e)) from None
+    if cfg.min_similarity is None and getattr(embedder, "min_similarity", None) is None:
+        log.warning("the model %r has no measured min_similarity yet, so the default gate of the built-in model (0.845) "
+                    "applies, and on another similarity scale almost nothing will pass it: set min_similarity under "
+                    "[embedder] from a run of `passageway eval --sweep`", cfg.model)
+    return embedder