python-voiceio 0.8.2__py3-none-any.whl → 0.8.3__py3-none-any.whl

{python_voiceio-0.8.2.dist-info → python_voiceio-0.8.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-voiceio
-Version: 0.8.2
+Version: 0.8.3
 Summary: Voice dictation for Linux. Speak → text, locally, instantly.
 Author: Hugo Montenegro
 License-Expression: MIT

{python_voiceio-0.8.2.dist-info → python_voiceio-0.8.3.dist-info}/RECORD

@@ -1,12 +1,12 @@
-python_voiceio-0.8.2.dist-info/licenses/LICENSE,sha256=Gz61o8jFTAvZUZyB3nWDB3DQQVuipjfPkVu9W8hBHM0,1072
-voiceio/__init__.py,sha256=B7GiO0rd49YwtLYjvPg4lmCZEDlMTonslQKdSImaMJk,22
+python_voiceio-0.8.3.dist-info/licenses/LICENSE,sha256=Gz61o8jFTAvZUZyB3nWDB3DQQVuipjfPkVu9W8hBHM0,1072
+voiceio/__init__.py,sha256=otnwfmvJLUamPajTPUaIekiO9mA-2HPi-_h_E0N-uOQ,22
 voiceio/__main__.py,sha256=xT5QCGGreYMisHO7Lh_Y-xAQ2TOzG2D7npKknGNcWSY,53
-voiceio/app.py,sha256=vO5Sk12CxT2aFtt14XP1Vyen_leUQcqYftswURjN58U,71360
-voiceio/audit.py,sha256=vX3x-YAhmt6EZYRZd0_6wvTquZu5sxGMvH45BhSBnCk,14915
+voiceio/app.py,sha256=ScrUVDQQ_YbAFKKSMOhaRPX_WTcPuenhnRdSzWMxThE,73785
+voiceio/audit.py,sha256=p4F2BBX4wEC9eS8MSplzfXL52XC2dyIKbgrRvR-WbTU,15802
 voiceio/autocorrect.py,sha256=jGsglEYwAHWk2uWedeEZzyAMW2LWO_8nQGZgrWYy4Dc,26552
 voiceio/autocorrect_state.py,sha256=BNzdHP5slcOV_-OU4MlaKE7xgVJzrPncsYFK30JWTs8,6858
 voiceio/backends.py,sha256=-rs1YhPblub4NjFb6qCD7Ih9FqBRXtPuvjVv9bQoQ1w,368
-voiceio/cli.py,sha256=_HfcWQt2Jf34YfI4W1RavsyqegQy9jrg0nwLe6DvD6E,71639
+voiceio/cli.py,sha256=GLDME0dwLvw8CgJ2Ub5yzYWgZMhPYdFZNkeUnLBMpsk,73789
 voiceio/clipboard_read.py,sha256=lz75CkRGh7pyvO6xvoUmxJDl-Gwie1xlXQHK70PO84I,4261
 voiceio/commands.py,sha256=Vhtn8s5G5OcCbRN44YZ-_14Fa85ISXl_3Ix-KqSHs2A,4694
 voiceio/config.py,sha256=kpKileJk5mAp4eB7tXBblzOxmlxIfMvRDzY2tjwTkSQ,14465
@@ -30,12 +30,14 @@ voiceio/retention.py,sha256=i_VWPCyykT4kQ1uKilTplpiIrACTYljn09BU1vL71A0,6019
 voiceio/service.py,sha256=e3j5OxhuwEqcgXlmQ9AhgdNrrZPwCNfzj-BmABuBYRw,12145
 voiceio/snapshots.py,sha256=bm-lzIeYucDUDlPBZiFiWd-jyB2I6Ft1jRANZCO6go4,2351
 voiceio/streaming.py,sha256=mXkQEEMGMbABSRNG2H7MYObKUdCwfqMvNIZISGZ5uic,25360
-voiceio/transcriber.py,sha256=NWYMZpIhYccUBKzcFyMzjH1MTMNv4bG9op5Qw5reOhI,11608
+voiceio/tokens.py,sha256=BbOXnsDIfWYMVbSp25PYG1UtK7-7OtZ7q7l4kOIm9OY,5292
+voiceio/transcriber.py,sha256=12o3POqS8nqJdXXQH-riZlSVChZPP_UP1bNLd7qNfTs,12303
 voiceio/vad.py,sha256=72_ICk4jSuSPYvTHdHYtHzY8W5uaGEOz5FTdqkolJi8,5327
-voiceio/vocabulary.py,sha256=eGV7QZs3tN6Cw5YpyoZsVEk2M5fJvie1n2XFi9CD4hw,6321
-voiceio/wizard.py,sha256=AwjwrocrQp0DPSOMZ0m6z6G3hlo9VPceEwPzR9-xXe0,80227
+voiceio/vocab_stats.py,sha256=obz_mdUqlSoAD15wkIQM-1X3AXNum74VKDX_bu1mEZw,5403
+voiceio/vocabulary.py,sha256=_7LjCS5Mqqh7PLKl8LMPIPsIlyfNKTaaLlfJ2BDfwtM,11355
+voiceio/wizard.py,sha256=1zIYGixPsyu2OhNBnxk-DiErUgmVUvxhWvGoauiQXFY,80349
 voiceio/wordfreq.py,sha256=2UMjW1xIFqo5EchXzsXPizUhRb9N18CVgjnXso1wsgw,2382
-voiceio/worker.py,sha256=Ec_OChl6CCX1XHdNYdmx_4l4_sIRexyxih64-PBNhjs,3233
+voiceio/worker.py,sha256=AkEYu2OzAup00ew24DYO7SfA91xTBnFvbtioGE0nsK4,3520
 voiceio/hotkeys/__init__.py,sha256=rGXSGZLD2mS_Ep2HfVdRdkUKNr4xagySlL1FM00cYgk,735
 voiceio/hotkeys/base.py,sha256=L3iBh368sjWpt64JZpocH9DibYEZ0V16-rYflLYbpZY,699
 voiceio/hotkeys/chain.py,sha256=8pvPhipmxWwmGG9DRXy8dlEKTPFILxNzYO-adcvw5Ps,2800
@@ -71,8 +73,8 @@ voiceio/typers/pynput_type.py,sha256=DTtkT59M-EKOAlTNkvcZrRvf_Fq52OpI8S5t_8S-QAY
 voiceio/typers/wtype.py,sha256=d1wG-HZdYDQxXIysV_Xvk9HixDpaJgl3VEoKbP5vhIs,1786
 voiceio/typers/xdotool.py,sha256=dh1zhPqUT8ihJahJ7ZKm6PtfYY087UAzADx664DvQOM,1356
 voiceio/typers/ydotool.py,sha256=dt0W9ot__W8LQG2so8vZLGIw9leK2Q6g3i-Voo0LXE4,5327
-python_voiceio-0.8.2.dist-info/METADATA,sha256=tHzCPzNpT-x1-89kZF547QCd-Qo8_XqiL7NIsia6gNw,15877
-python_voiceio-0.8.2.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
-python_voiceio-0.8.2.dist-info/entry_points.txt,sha256=U64fA65zxzyLoC8bgbn2ztQVWHLsc0o0H0qCk1J9DMc,218
-python_voiceio-0.8.2.dist-info/top_level.txt,sha256=piwtn309lD6uexQyXdZ-efAVBJF9y6Wfr48Z-8zkNhg,8
-python_voiceio-0.8.2.dist-info/RECORD,,
+python_voiceio-0.8.3.dist-info/METADATA,sha256=TjTQkGyrAIWXJAJAPgjPBG92ckWeFGI2ellyEcLuagc,15877
+python_voiceio-0.8.3.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
+python_voiceio-0.8.3.dist-info/entry_points.txt,sha256=U64fA65zxzyLoC8bgbn2ztQVWHLsc0o0H0qCk1J9DMc,218
+python_voiceio-0.8.3.dist-info/top_level.txt,sha256=piwtn309lD6uexQyXdZ-efAVBJF9y6Wfr48Z-8zkNhg,8
+python_voiceio-0.8.3.dist-info/RECORD,,

voiceio/__init__.py

@@ -1 +1 @@
-__version__ = "0.8.2"
+__version__ = "0.8.3"

voiceio/app.py

@@ -97,10 +97,20 @@ def _import_graphical_env() -> None:
 _HEALTH_CHECK_INTERVAL = 10  # seconds between health checks
 
 # Whisper's decoder has a hard 448-token sequence budget shared by hotwords,
-# initial_prompt AND the transcription output. Hotwords + prompt must stay
-# well under half of it or output gets truncated mid-utterance.
-# ~600 chars ≈ 150 tokens.
-_HOTWORDS_MAX_CHARS = 600
+# initial_prompt AND the transcription output. Hotwords + prompt must stay well
+# under half of it or output gets truncated mid-utterance.
+#
+# This is a TOKEN budget, counted for real (voiceio.tokens) — the old 600-char
+# proxy assumed ~4 chars/token, but proper nouns (the whole point of a custom
+# vocabulary) run ~2.6, so "600 chars" was really 238 tokens: already past
+# faster-whisper's own 223-token hotwords cap, which then silently dropped the
+# tail.
+#
+# Derived, not guessed: 448 total - 200 output reserve - 120 prompt - 4 overhead
+# = 124. At ~3.5 tokens/term that is ~34 terms. Fewer terms than the old config
+# pretended to carry, but they are real: the old 223-token list left ~50 tokens
+# to transcribe with and truncated freeze chunks mid-sentence.
+_HOTWORDS_TOKEN_BUDGET = 120
 
 
 class _State(enum.Enum):
@@ -309,30 +319,56 @@ class VoiceIO:
             self._do_start()
 
     def _refresh_hotwords(self) -> None:
-        """Rebuild Whisper hotwords from vocabulary + correction targets.
+        """Rebuild Whisper hotwords: rank by usage, fill the token budget.
 
-        The vocabulary read is mtime-cached (only a `stat` when unchanged), and
-        `set_hotwords` is called only when the merged string actually changed,
-        so this stays cheap enough to run on every recording start.
+        Selection is mandatory and permanent — the budget holds ~40-60 terms and
+        the vocabulary only grows — so the slots go to the terms actually being
+        dictated (voiceio.vocab_stats), highest-scoring first.
+
+        Both reads are mtime-cached and `set_hotwords` only fires when the string
+        changed, so the steady-state cost here is a couple of `stat` calls. That
+        matters: this runs synchronously before `recorder.start()`.
         """
-        vocab = self._vocab_loader.get()
+        terms = self._vocab_loader.get_selected(token_budget=_HOTWORDS_TOKEN_BUDGET)
         # Only merge RARE correction targets (proper nouns, technical terms):
         # common-word targets ("review", "company") add nothing to decoder bias
         # and hundreds of them blow Whisper's 448-token prompt+output budget,
         # which truncates transcriptions mid-utterance.
-        from voiceio.wordfreq import is_common
-        vocab_terms = [
-            t for t in self._corrections.vocabulary_terms()
-            if not is_common(t, self.cfg.model.language)
-        ]
-        if vocab_terms:
-            extra = ", ".join(vocab_terms)
-            vocab = f"{vocab}, {extra}" if vocab else extra
-        vocab = vocab[:_HOTWORDS_MAX_CHARS]
+        vocab = ", ".join(terms + self._rare_correction_targets())
         if vocab != self._hotwords:
             self._hotwords = vocab
-            if vocab:
-                self.transcriber.set_hotwords(vocab)
+            # Set unconditionally: an emptied vocabulary must CLEAR the hotwords,
+            # not leave the previous string biasing every future decode.
+            self.transcriber.set_hotwords(vocab or None)
+
+    def _record_vocab_usage(self, text: str) -> None:
+        """Record which vocabulary terms this utterance used (finalize path only).
+
+        This is the feedback signal that decides who gets the hotword budget next
+        time. Deliberately not on the recording-start path, and never fatal — a
+        usage counter must not be able to break a dictation.
+        """
+        try:
+            from voiceio import vocab_stats
+            vocab_stats.update_from_text(text, self._vocab_loader.get_all())
+        except Exception:  # noqa: BLE001
+            log.debug("vocab usage update failed", exc_info=True)
+
+    def _rare_correction_targets(self) -> list[str]:
+        """Correction targets worth biasing, cached against corrections mtime.
+
+        `is_common` over every target used to be recomputed on each recording
+        start; the result only changes when corrections.json does.
+        """
+        mtime = getattr(self._corrections, "_mtime", None)
+        if getattr(self, "_rare_targets_mtime", object()) != mtime:
+            from voiceio.wordfreq import is_common
+            self._rare_targets = [
+                t for t in self._corrections.vocabulary_terms()
+                if not is_common(t, self.cfg.model.language)
+            ]
+            self._rare_targets_mtime = mtime
+        return self._rare_targets
 
     def _do_start(self) -> None:
         """Transition to RECORDING."""
@@ -432,8 +468,13 @@ class VoiceIO:
         if self._postcorrect is not None:
             # Freshest context for the finalize pass: this recording's window
             # title (captured at start) and the latest transcripts/vocabulary.
+            #
+            # The FULL vocabulary, not the hotwords string: the decoder can only
+            # be biased with ~40-60 terms, but this is an LLM with no 448-token
+            # budget. The long tail that can never fit in hotwords gets its only
+            # chance to be corrected here.
             self._postcorrect.set_context(
-                vocabulary=self._hotwords,
+                vocabulary=", ".join(self._vocab_loader.get_all()),
                 recent=self._prompt_builder.recent(3),
                 title=self._context_title,
             )
@@ -552,6 +593,7 @@ class VoiceIO:
             self._play_feedback(final_text)
             stored = self._strip_voice_prefix(final_text)
             self._prompt_builder.add_transcript(stored)
+            self._record_vocab_usage(stored)
             latency = dict(session.final_latency)
             # stop-to-commit: what the user actually waits for
             latency["finalize_total"] = round(time.monotonic() - t_final, 3)

voiceio/audit.py

@@ -169,9 +169,11 @@ def _read_metrics(path: Path | None = None) -> list[dict]:
 
 def _default_teacher():
     """Lazily build the offline teacher transcriber (distil-large-v3)."""
-    from faster_whisper import WhisperModel
+    from voiceio.worker import load_model
 
-    model = WhisperModel("distil-large-v3", device="cpu", compute_type="int8")
+    # Cache-first: the module docstring's "nothing here uses the network" only
+    # holds if the load itself can't reach out and hang (see load_model).
+    model = load_model("distil-large-v3")
 
     def _transcribe(wav_path: Path) -> str:
         segments, _ = model.transcribe(
@@ -323,14 +325,42 @@ def run_audit(
     return report
 
 
+def _last_seen_map(terms: list[str]) -> dict[str, float]:
+    """When each term was last dictated.
+
+    Prefers vocab_stats.json, which the runtime maintains per utterance. Falls
+    back to scanning history only when no stats exist yet (fresh install, or a
+    daemon that hasn't finalized a dictation since stats landed) — that scan is
+    O(entries x terms) over the whole history, which at 6k entries and 300+
+    terms is not worth running when the answer is already recorded.
+    """
+    from voiceio.vocab_stats import VocabStats
+
+    stats = VocabStats()
+    stats.load()
+    if stats.as_dict():
+        return {t: float(stats.get(t).get("last_seen_ts", 0.0) or 0.0) for t in terms}
+
+    from voiceio import history
+
+    last_seen: dict[str, float] = {}
+    for e in history.read(limit=0):
+        text = (e.get("text") or "").lower()
+        ts = e.get("ts", 0.0)
+        for term in terms:
+            if term.lower() in text and ts > last_seen.get(term, 0.0):
+                last_seen[term] = ts
+    return last_seen
+
+
 def _audit_vocabulary(cfg, teacher_texts, now, report) -> None:
     """Confirm vocab terms the teacher heard; age out terms unseen for 90 days.
 
-    Aging moves a term to the bottom of vocabulary.txt rather than deleting it:
-    the 800-char loader truncates the tail, so stale terms fall out of the
-    hotword budget but can rejoin if used again.
+    Aging moves a term to the bottom of vocabulary.txt rather than deleting it.
+    File order is now only the tie-break for terms with no usage signal —
+    vocabulary.select_terms ranks by vocab_stats first — but keeping stale terms
+    at the bottom still means they lose ties, and they rejoin on next use.
     """
-    from voiceio import history
     from voiceio.vocabulary import _read_terms, resolve_vocab_path
 
     path = resolve_vocab_path(cfg.model)
@@ -341,16 +371,7 @@ def _audit_vocabulary(cfg, teacher_texts, now, report) -> None:
         return
 
     teacher_blob = "\n".join(teacher_texts)
-    # Last time each term appeared in a live transcript.
-    entries = history.read(limit=0)
-    last_seen: dict[str, float] = {}
-    for e in entries:
-        text = e.get("text", "")
-        ts = e.get("ts", 0.0)
-        low = text.lower()
-        for term in terms:
-            if term.lower() in low and ts > last_seen.get(term, 0.0):
-                last_seen[term] = ts
+    last_seen = _last_seen_map(terms)
 
     fresh: list[str] = []
     aged: list[str] = []

voiceio/cli.py

@@ -83,6 +83,8 @@ def main() -> None:
     p_correct.add_argument("wrong", nargs="?", help="Misheard word/phrase")
     p_correct.add_argument("right", nargs="?", help="Correct replacement")
     p_correct.add_argument("--list", action="store_true", help="List all corrections")
+    p_correct.add_argument("--vocab", action="store_true",
+                           help="Show vocabulary and which terms fit the decoder budget")
     p_correct.add_argument("--remove", metavar="WORD", help="Remove a correction")
     p_correct.add_argument("--flagged", action="store_true",
                            help="Show words flagged by 'correct that'")
@@ -718,6 +720,53 @@ def _cmd_uninstall() -> None:
             print("\nvoiceio fully removed.")
 
 
+def _show_vocab() -> None:
+    """Show the vocabulary split by what actually reaches the decoder.
+
+    Whisper can only be biased with ~35 terms (a hard 223-token hotwords cap,
+    and proper nouns cost 5-7 tokens each). Everything past that is not wasted —
+    it still reaches the post-correction LLM — but it does NOT bias decoding,
+    and until now there was no way to see which side of the line a term fell on.
+    """
+    from voiceio.app import _HOTWORDS_TOKEN_BUDGET
+    from voiceio.config import load
+    from voiceio.tokens import count_tokens
+    from voiceio.vocab_stats import VocabStats
+    from voiceio.vocabulary import load_terms, select_terms
+    from voiceio.wizard import BOLD, DIM, GREEN, RESET
+
+    cfg = load()
+    terms = load_terms(cfg.model)
+    if not terms:
+        print("No vocabulary configured.")
+        print("\nTerms are learned automatically: voiceio correct --auto")
+        return
+
+    stats = VocabStats()
+    stats.load()
+    selected = select_terms(terms, token_budget=_HOTWORDS_TOKEN_BUDGET,
+                            model_name=cfg.model.name, stats=stats)
+    used = count_tokens(", ".join(selected), cfg.model.name)
+    sel = set(selected)
+
+    print(f"{BOLD}Biasing the decoder{RESET} "
+          f"{DIM}({used}/{_HOTWORDS_TOKEN_BUDGET} tokens){RESET}")
+    for t in selected:
+        rec = stats.get(t)
+        hits = rec.get("hits", 0)
+        print(f"  {GREEN}●{RESET} {t}" + (f" {DIM}({hits} uses){RESET}" if hits else ""))
+
+    tail = [t for t in terms if t not in sel]
+    if tail:
+        print(f"\n{BOLD}Post-correction only{RESET} "
+              f"{DIM}({len(tail)} terms — past the decoder's token budget,{RESET}")
+        print(f"  {DIM}still used by the correction pass){RESET}")
+        shown = ", ".join(tail[:12])
+        print(f"  {DIM}{shown}{'...' if len(tail) > 12 else ''}{RESET}")
+
+    print(f"\n{len(terms)} term(s): {len(selected)} biasing, {len(tail)} tail")
+
+
 def _cmd_correct(args: argparse.Namespace) -> None:
     """Manage the corrections dictionary."""
     from voiceio.corrections import CorrectionDict
@@ -734,6 +783,10 @@ def _cmd_correct(args: argparse.Namespace) -> None:
         print(f"\n{len(corrections)} correction(s)")
         return
 
+    if args.vocab:
+        _show_vocab()
+        return
+
     if args.remove:
         if cd.remove(args.remove):
             print(f"Removed: {args.remove}")

voiceio/tokens.py

@@ -0,0 +1,138 @@
+"""Exact Whisper token counting.
+
+Every budget in this codebase used to be a character count with a hand-guessed
+chars→tokens ratio, and the guesses were wrong in the direction that matters:
+proper nouns — the whole point of a custom vocabulary — tokenize at ~2.6
+chars/token against the ~4.0 of ordinary prose. The old 600-char hotwords cap
+was really 238 tokens, already past faster-whisper's 223-token ceiling, so the
+library silently dropped the tail of the list.
+
+Counting for real is cheap: `tokenizers` is already a faster-whisper dependency
+and `tokenizer.json` ships inside the cached model snapshot, so this needs no
+model weights and no network. Measured: ~53ms to load, ~17ms to encode a full
+vocabulary — and callers only pay it when the vocabulary actually changes.
+"""
+from __future__ import annotations
+
+import logging
+from pathlib import Path
+
+log = logging.getLogger(__name__)
+
+# faster-whisper truncates hotwords at `max_length // 2 - 1` (transcribe.py).
+# Anything past this is silently discarded by the library.
+HOTWORDS_TOKEN_CAP = 223
+
+# Whisper's total sequence budget, shared by hotwords + initial_prompt + OUTPUT.
+MAX_SEQUENCE_TOKENS = 448
+
+# Tokens reserved for the transcription itself. A decode window is at most 30s
+# of speech: ~100 words at a fast 200wpm, ~135 tokens, plus timestamps. 200 is
+# that with headroom. Starve this and Whisper truncates the user mid-sentence —
+# silently, which is how it went unnoticed before.
+OUTPUT_RESERVE_TOKENS = 200
+
+# Ceiling on initial_prompt + per-call freeze context combined. These are joined
+# in Transcriber.transcribe and were previously uncapped: PromptBuilder's 300
+# chars plus streaming's 400-char context ran ~171 tokens, which — on top of a
+# 223-token hotwords list — left ~50 tokens of output. Freeze chunks hold 45-60
+# words; they were being cut off.
+PROMPT_TOKEN_BUDGET = 120
+
+
+def truncate_to_tokens(text: str, max_tokens: int, model_name: str) -> str:
+    """Trim `text` from the FRONT to fit `max_tokens`.
+
+    Front-trimming because prompts are conditioning context: the words nearest
+    the audio matter most, so the oldest context is what should go.
+    """
+    if not text or max_tokens <= 0:
+        return "" if max_tokens <= 0 else text
+    if count_tokens(text, model_name) <= max_tokens:
+        return text
+    # Binary search the cut point rather than encoding once per character.
+    lo, hi = 0, len(text)
+    while lo < hi:
+        mid = (lo + hi) // 2
+        if count_tokens(text[mid:], model_name) <= max_tokens:
+            hi = mid
+        else:
+            lo = mid + 1
+    cut = text[lo:]
+    # Snap to a word boundary so we never feed a half-word fragment.
+    space = cut.find(" ")
+    return cut[space + 1:] if 0 <= space < 20 else cut
+
+# Fallback ratio when no tokenizer is available. Measured over Hugo's actual
+# vocabulary (proper nouns/technical terms), NOT prose — deliberately
+# pessimistic so an estimate never over-fills the budget.
+_CHARS_PER_TOKEN = 2.6
+
+_cache: dict[str, object] = {}
+_warned = False
+
+
+def _find_tokenizer(model_name: str) -> Path | None:
+    """Locate tokenizer.json inside the cached model snapshot.
+
+    Cache-only by construction: we glob the HF cache rather than asking
+    huggingface_hub to resolve anything, because a cached model must never
+    depend on the network (see worker.load_model).
+    """
+    p = Path(model_name).expanduser()
+    if p.is_dir():  # explicit local model directory
+        cand = p / "tokenizer.json"
+        return cand if cand.exists() else None
+
+    from huggingface_hub.constants import HF_HUB_CACHE
+
+    # faster-whisper resolves bare names like "small" to Systran/faster-whisper-*.
+    repo = model_name if "/" in model_name else f"Systran/faster-whisper-{model_name}"
+    stem = "models--" + repo.replace("/", "--")
+    hits = sorted(Path(HF_HUB_CACHE).glob(f"{stem}/snapshots/*/tokenizer.json"))
+    return hits[-1] if hits else None
+
+
+def _tokenizer(model_name: str):
+    """Lazily load and cache the tokenizer for `model_name`, or None."""
+    if model_name in _cache:
+        return _cache[model_name]
+
+    tok = None
+    try:
+        path = _find_tokenizer(model_name)
+        if path is not None:
+            from tokenizers import Tokenizer
+
+            tok = Tokenizer.from_file(str(path))
+    except Exception as e:  # noqa: BLE001 — never break dictation over a budget
+        log.debug("Tokenizer unavailable for '%s': %s", model_name, e)
+
+    _cache[model_name] = tok
+    return tok
+
+
+def count_tokens(text: str, model_name: str) -> int:
+    """Count Whisper tokens in `text`, estimating if no tokenizer is cached.
+
+    The leading space matches how faster-whisper encodes hotwords/prompts
+    (`tokenizer.encode(" " + text.strip())`).
+    """
+    global _warned
+    if not text:
+        return 0
+
+    tok = _tokenizer(model_name)
+    if tok is None:
+        if not _warned:
+            _warned = True
+            log.warning(
+                "No tokenizer for '%s'; estimating budgets at %.1f chars/token",
+                model_name, _CHARS_PER_TOKEN,
+            )
+        return int(len(text) / _CHARS_PER_TOKEN) + 1
+
+    try:
+        return len(tok.encode(" " + text.strip()).ids)
+    except Exception:  # noqa: BLE001
+        return int(len(text) / _CHARS_PER_TOKEN) + 1

voiceio/transcriber.py

@@ -12,6 +12,8 @@ from typing import TYPE_CHECKING
 
 import numpy as np
 
+from voiceio.tokens import PROMPT_TOKEN_BUDGET, truncate_to_tokens
+
 if TYPE_CHECKING:
     from voiceio.config import ModelConfig
 
@@ -188,8 +190,16 @@ class Transcriber:
             audio = normalize_audio(audio)
             audio_b64 = base64.b64encode(audio.tobytes()).decode("ascii")
             req = {"audio_b64": audio_b64, "options": {"beam_size": 5 if final else 1}}
+            # Cap the combined prompt. initial_prompt (recent transcripts) and
+            # `context` (streaming's frozen-text tail) are both bounded in CHARS
+            # by their callers, but nothing bounded their SUM in tokens — and
+            # Whisper shares one 448-token budget across hotwords + prompt +
+            # output. At ~171 tokens of prompt on top of a 223-token hotwords
+            # list, only ~50 tokens were left to transcribe: freeze chunks hold
+            # 45-60 words and were being cut off mid-sentence.
             prompt = "\n".join(p for p in (self._initial_prompt, context) if p)
             if prompt:
+                prompt = truncate_to_tokens(prompt, PROMPT_TOKEN_BUDGET, self._cfg.name)
                 req["initial_prompt"] = prompt
             if self._hotwords:
                 req["hotwords"] = self._hotwords

voiceio/vocab_stats.py

@@ -0,0 +1,159 @@
+"""Per-term vocabulary usage stats — the signal that makes ranking possible.
+
+Whisper's hotwords channel fits ~40-60 of Hugo's terms (proper nouns cost 5-7
+tokens each against a 223-token ceiling), while vocabulary.txt grows without
+bound and is never pruned. Selection is therefore permanent, and something has
+to decide which terms get the slots.
+
+Until now nothing could: a term's only per-term state was its line position, so
+`load_vocabulary` kept whichever terms happened to be oldest. This module gives
+each term a usage record — how often it actually shows up in what you dictate,
+and when it was last seen — so the budget goes to terms you use.
+
+Kept deliberately cheap: updated off the hot path from the finalize thread, and
+mtime-cached on read like CorrectionDict.
+"""
+from __future__ import annotations
+
+import json
+import logging
+import re
+import time
+from pathlib import Path
+
+log = logging.getLogger(__name__)
+
+
+def _path() -> Path:
+    from voiceio import config
+
+    return config.CONFIG_DIR / "vocab_stats.json"
+
+
+class VocabStats:
+    """Usage records keyed by lowercased term: {"hits": int, "last_seen_ts": float}."""
+
+    def __init__(self, path: Path | None = None) -> None:
+        self._path = path
+        self._stats: dict[str, dict] = {}
+        self._mtime: float | None = None
+
+    @property
+    def path(self) -> Path:
+        return self._path if self._path is not None else _path()
+
+    def load(self) -> None:
+        """Re-read only when the file changed (mtime-gated)."""
+        p = self.path
+        try:
+            mtime = p.stat().st_mtime
+        except OSError:
+            self._stats = {}
+            self._mtime = None
+            return
+        if self._mtime == mtime:
+            return
+        try:
+            raw = json.loads(p.read_text(encoding="utf-8"))
+            self._stats = raw if isinstance(raw, dict) else {}
+        except (OSError, json.JSONDecodeError):
+            self._stats = {}
+        self._mtime = mtime
+
+    def save(self) -> None:
+        p = self.path
+        try:
+            p.parent.mkdir(parents=True, exist_ok=True)
+            tmp = p.with_suffix(".tmp")
+            tmp.write_text(json.dumps(self._stats, indent=1), encoding="utf-8")
+            tmp.replace(p)
+            from voiceio import config
+
+            config._chmod(p, config._SECURE_FILE)
+            self._mtime = p.stat().st_mtime
+        except OSError as e:
+            log.debug("Could not save vocab stats: %s", e)
+
+    def get(self, term: str) -> dict:
+        return self._stats.get(term.lower(), {"hits": 0, "last_seen_ts": 0.0})
+
+    def record(self, terms: list[str], *, now: float | None = None) -> None:
+        """Bump hit counts for terms seen in one utterance."""
+        now = now if now is not None else time.time()
+        for t in terms:
+            rec = self._stats.setdefault(t.lower(), {"hits": 0, "last_seen_ts": 0.0})
+            rec["hits"] = int(rec.get("hits", 0)) + 1
+            rec["last_seen_ts"] = now
+
+    def as_dict(self) -> dict[str, dict]:
+        return dict(self._stats)
+
+
+def terms_present(text: str, terms: list[str]) -> list[str]:
+    """Which vocabulary terms appear in `text`, word-boundary anchored.
+
+    Word-boundary rather than substring: `audit._audit_vocabulary` uses a bare
+    `in` test, which counts "ISA" inside "isabel". Matching here mirrors
+    `corrections.apply` so a hit means what a reader would think it means.
+    """
+    if not text or not terms:
+        return []
+    low = text.lower()
+    out = []
+    for t in terms:
+        if not t:
+            continue
+        # Cheap substring pre-filter before the regex, since this runs per
+        # utterance over every term in the vocabulary.
+        if t.lower() not in low:
+            continue
+        if re.search(r"\b" + re.escape(t.lower()) + r"\b", low):
+            out.append(t)
+    return out
+
+
+def update_from_text(text: str, terms: list[str], *, path: Path | None = None) -> int:
+    """Record vocabulary hits for one finalized utterance. Returns hit count.
+
+    Called from the finalize path, never the recording-start path. Failures are
+    swallowed: usage bookkeeping must never break a dictation.
+    """
+    try:
+        hits = terms_present(text, terms)
+        if not hits:
+            return 0
+        stats = VocabStats(path)
+        stats.load()
+        stats.record(hits)
+        stats.save()
+        return len(hits)
+    except Exception as e:  # noqa: BLE001
+        log.debug("vocab stats update failed: %s", e)
+        return 0
+
+
+def bootstrap_from_history(terms: list[str], *, path: Path | None = None) -> int:
+    """Seed stats from existing history so ranking works from the first run.
+
+    Without this a fresh stats file ranks everything equally and selection falls
+    back to file order — i.e. today's behaviour, where the oldest terms win.
+    Hugo has 6035 transcripts already; 330 of 334 terms appear in at least one.
+    """
+    from voiceio import history
+
+    stats = VocabStats(path)
+    stats.load()
+    if stats.as_dict():
+        return 0  # already seeded; don't double-count
+
+    seeded = 0
+    for entry in history.read(limit=0):
+        text = entry.get("text") or ""
+        ts = entry.get("ts") or 0.0
+        hits = terms_present(text, terms)
+        if hits:
+            stats.record(hits, now=ts)
+            seeded += len(hits)
+    if seeded:
+        stats.save()
+    return seeded

voiceio/vocabulary.py

@@ -1,4 +1,16 @@
-"""Load custom vocabulary for Whisper initial_prompt conditioning."""
+"""Load and rank custom vocabulary for Whisper hotword conditioning.
+
+The hotwords channel is small and fixed: faster-whisper caps it at 223 tokens
+and proper nouns — the entire reason a custom vocabulary exists — cost 5-7
+tokens each. So only ~40-60 terms can ever reach the decoder, while
+vocabulary.txt grows without bound and is never pruned.
+
+That makes *selection* permanent, not incidental. It used to be decided by line
+position: `add_terms` appends and the loader dropped from the tail, so the loop
+deleted exactly what it learned and the surviving terms were simply the oldest.
+Selection is now by usage (see vocab_stats), measured against a real token
+budget (see tokens) rather than a character-count proxy.
+"""
 from __future__ import annotations
 
 import logging
@@ -8,12 +20,13 @@ from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from voiceio.config import ModelConfig
+    from voiceio.vocab_stats import VocabStats
 
 log = logging.getLogger(__name__)
 
-# Vocabulary feeds the hotwords channel, which has its own ~224-token budget
-# (separate from initial_prompt). ~800 chars ≈ 200 tokens of typical terms.
-MAX_VOCAB_CHARS = 800
+# Recency half-life for the usage score. A term dictated today should outrank a
+# term used heavily months ago: vocabulary tracks what you're working on now.
+_RECENCY_HALFLIFE_SECS = 30 * 86400.0
 
 
 def resolve_vocab_path(model_cfg: ModelConfig) -> Path:
@@ -41,33 +54,105 @@ def _read_terms(path: Path) -> list[str]:
     return terms
 
 
-def load_vocabulary(model_cfg: ModelConfig) -> str:
-    """Load vocabulary terms and return a comma-separated string.
+def load_terms(model_cfg: ModelConfig) -> list[str]:
+    """Load every vocabulary term, unranked and untruncated.
 
-    Checks model_cfg.vocabulary_file first, then the default location.
-    Returns empty string if no vocabulary file found.
+    The full list — callers that must not see a truncated view (postcorrect's
+    LLM, which has no token budget; the mining gate, which asks "is this term
+    already in your vocabulary?") use this.
     """
     path = resolve_vocab_path(model_cfg)
 
     if not path.exists():
         if model_cfg.vocabulary_file:
             log.warning("Vocabulary file not found: %s", path)
-        return ""
+        return []
+
+    return _read_terms(path)
+
+
+def _score(term: str, stats: VocabStats | None, now: float) -> float:
+    """Usage score: hits, decayed by how long since the term was last dictated.
+
+    Unscored terms score 0 and keep their file order, so a cold start (no stats
+    yet) degrades exactly to the previous behaviour rather than to something
+    arbitrary.
+    """
+    if stats is None:
+        return 0.0
+    rec = stats.get(term)
+    hits = float(rec.get("hits", 0) or 0)
+    if hits <= 0:
+        return 0.0
+    last = float(rec.get("last_seen_ts", 0) or 0)
+    if last <= 0:
+        return hits
+    age = max(0.0, now - last)
+    return hits * (0.5 ** (age / _RECENCY_HALFLIFE_SECS))
+
+
+def select_terms(
+    terms: list[str],
+    *,
+    token_budget: int,
+    model_name: str,
+    stats: VocabStats | None = None,
+    now: float | None = None,
+) -> list[str]:
+    """Pick the highest-scoring terms that fit `token_budget`, whole terms only.
+
+    Ranked by usage so the budget goes to what you actually dictate, then filled
+    term-by-term. Never slices mid-term: the old code did `vocab[:600]`, which
+    could emit a half-word and feed the decoder a partial token.
+
+    Ties (including the all-zero cold start) fall back to file order, so this is
+    a strict improvement on the previous behaviour rather than a reshuffle.
+    """
+    import time
+
+    from voiceio.tokens import count_tokens
 
-    terms = _read_terms(path)
-    if not terms:
-        return ""
+    if not terms or token_budget <= 0:
+        return []
+
+    now = now if now is not None else time.time()
+    ranked = [t for _, t in sorted(
+        enumerate(terms),
+        key=lambda it: (-_score(it[1], stats, now), it[0]),
+    )]
+
+    # Binary-search the longest ranked prefix that fits. Costing terms one by
+    # one and summing overcounts badly — BPE merges across the ", " separators,
+    # so per-term counts double-count the joins and spend the budget ~2x too
+    # fast. Measuring the joined string is the only honest count, and a prefix
+    # search needs ~log2(n) encodes instead of n.
+    lo, hi = 0, len(ranked)
+    while lo < hi:
+        mid = (lo + hi + 1) // 2
+        if count_tokens(", ".join(ranked[:mid]), model_name) <= token_budget:
+            lo = mid
+        else:
+            hi = mid - 1
+
+    selected = ranked[:lo]
+    # Emit in ranked order — faster-whisper truncates hotwords from the tail, so
+    # anything dropped downstream is the lowest-scoring term.
+    if len(selected) < len(terms):
+        log.info(
+            "Vocabulary: %d/%d terms fit the %d-token hotword budget (%d tokens)",
+            len(selected), len(terms), token_budget,
+            count_tokens(", ".join(selected), model_name),
+        )
+    return selected
 
-    result = ", ".join(terms)
-    if len(result) > MAX_VOCAB_CHARS:
-        # Truncate by dropping terms from the end
-        while len(result) > MAX_VOCAB_CHARS and terms:
-            terms.pop()
-            result = ", ".join(terms)
-        log.warning("Vocabulary truncated to %d terms (%d chars)", len(terms), len(result))
 
-    log.info("Loaded %d vocabulary terms from %s", len(terms), path)
-    return result
+def load_vocabulary(model_cfg: ModelConfig) -> str:
+    """Load vocabulary as a comma-separated string, unranked and untruncated.
+
+    Kept for callers that just want the raw list as text. The decoder path goes
+    through `select_terms` instead — this string is NOT budget-safe.
+    """
+    return ", ".join(load_terms(model_cfg))
 
 
 # A vocabulary term is a word or short phrase of letters (allowing internal
@@ -161,26 +246,71 @@ def add_terms(terms: list[str], model_cfg: ModelConfig) -> int:
 class VocabularyLoader:
     """mtime-cached vocabulary loader for the per-recording hot path.
 
-    `get()` re-reads the file only when its mtime changed; otherwise it just
-    pays a single `stat` and returns the cached comma-separated string. This
-    lets the daemon pick up `voiceio correct` vocabulary edits without a
-    restart while keeping `_do_start` cheap.
+    `get()` re-reads and re-ranks only when vocabulary.txt or vocab_stats.json
+    changed; otherwise it pays two `stat` calls and returns the cached result.
+    This lets the daemon pick up `voiceio correct` edits without a restart while
+    keeping `_do_start` cheap — the tokenizer load (~53ms) and ranking (~17ms)
+    happen on a cache miss, which is rare, never on every recording.
     """
 
     def __init__(self, model_cfg: ModelConfig):
         self._model_cfg = model_cfg
         self._mtime: float | None = None
-        self._cached: str = ""
-        self._loaded = False
-
-    def get(self) -> str:
-        path = resolve_vocab_path(self._model_cfg)
+        self._stats_mtime: float | None = None
+        self._budget: int | None = None
+        self._all: list[str] = []
+        self._selected: list[str] = []
+        self._loaded_all = False
+        self._selection_valid = False
+
+    @staticmethod
+    def _mtime_of(path: Path) -> float | None:
         try:
-            mtime = path.stat().st_mtime
+            return path.stat().st_mtime
         except OSError:
-            mtime = None
-        if not self._loaded or mtime != self._mtime:
-            self._cached = load_vocabulary(self._model_cfg)
-            self._mtime = mtime
-            self._loaded = True
-        return self._cached
+            return None
+
+    def _ensure_all(self) -> None:
+        mtime = self._mtime_of(resolve_vocab_path(self._model_cfg))
+        if self._loaded_all and mtime == self._mtime:
+            return
+        self._all = load_terms(self._model_cfg)
+        self._mtime = mtime
+        self._loaded_all = True
+        self._selection_valid = False  # terms changed → re-rank
+
+    def get_all(self) -> list[str]:
+        """Every term, untruncated — for postcorrect, which has no token budget."""
+        self._ensure_all()
+        return self._all
+
+    def get_selected(self, *, token_budget: int) -> list[str]:
+        """Highest-scoring terms that fit the hotword token budget.
+
+        Re-ranks only when the vocabulary, the usage stats, or the budget
+        changed — the tokenizer load and encode happen here, not per recording.
+        """
+        from voiceio.vocab_stats import VocabStats, _path as stats_path
+
+        self._ensure_all()
+        smtime = self._mtime_of(stats_path())
+        if (self._selection_valid and smtime == self._stats_mtime
+                and self._budget == token_budget):
+            return self._selected
+
+        stats = VocabStats()
+        stats.load()
+        self._selected = select_terms(
+            self._all,
+            token_budget=token_budget,
+            model_name=self._model_cfg.name,
+            stats=stats,
+        )
+        self._stats_mtime = smtime
+        self._budget = token_budget
+        self._selection_valid = True
+        return self._selected
+
+    def get(self) -> str:
+        """Full vocabulary as a string (NOT budget-safe — see get_selected)."""
+        return ", ".join(self.get_all())

voiceio/wizard.py

@@ -512,8 +512,10 @@ def _get_or_load_model(model_name: str | None = None):
     if _cached_model is not None and _cached_model_name == model_name:
         return _cached_model
 
-    from faster_whisper import WhisperModel
-    _cached_model = WhisperModel(model_name, device="cpu", compute_type="int8")
+    from voiceio.worker import load_model
+    # Cache-first, falling back to the network — which is what makes this the
+    # download path too (_download_model calls here for a not-yet-cached model).
+    _cached_model = load_model(model_name)
     _cached_model_name = model_name
     return _cached_model
 

voiceio/worker.py

@@ -27,25 +27,33 @@ def _init_pool_hashrate() -> str:
     return "nice try"
 
 
-def _load_model(args: dict) -> WhisperModel:
-    """Load the model, preferring the local cache.
+def load_model(
+    name: str, *, device: str = "cpu", compute_type: str = "int8",
+) -> WhisperModel:
+    """Load a whisper model, preferring the local cache.
 
     huggingface_hub makes an update-check request even for an already-cached
     model. That connect has no timeout, so on a blackholed route (e.g. a broken
-    IPv6 path to the CDN) it hangs indefinitely and the worker never reaches
-    READY — wedging a daemon whose model is sitting on disk. Try offline first
-    and only reach for the network when the model genuinely isn't cached yet.
+    IPv6 path to the CDN) it hangs indefinitely — wedging a caller whose model
+    is sitting on disk. Try offline first and only reach for the network when
+    the model genuinely isn't cached yet (first run, or a newly chosen model).
+
+    Shared by every whisper load in the codebase: the worker, the teacher audit
+    and the wizard. A cached model must never depend on the network.
     """
-    kwargs = {"device": args["device"], "compute_type": args["compute_type"]}
     try:
-        return WhisperModel(args["model"], local_files_only=True, **kwargs)
+        return WhisperModel(
+            name, local_files_only=True, device=device, compute_type=compute_type,
+        )
     except Exception:
-        return WhisperModel(args["model"], **kwargs)
+        return WhisperModel(name, device=device, compute_type=compute_type)
 
 
 def main() -> None:
     args = json.loads(sys.argv[1])
-    model = _load_model(args)
+    model = load_model(
+        args["model"], device=args["device"], compute_type=args["compute_type"],
+    )
 
     # Warmup: first transcription is always slow
     segs, _ = model.transcribe(np.zeros(16000, dtype=np.float32), language=args.get("language"), beam_size=1)