python-voiceio 1.8.1__tar.gz → 1.8.3__tar.gz

{python_voiceio-1.8.1/python_voiceio.egg-info → python_voiceio-1.8.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-voiceio
-Version: 1.8.1
+Version: 1.8.3
 Summary: Voice dictation for Linux. Speak → text, locally, instantly.
 Author: Hugo Montenegro
 License-Expression: MIT

{python_voiceio-1.8.1 → python_voiceio-1.8.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "python-voiceio"
-version = "1.8.1"
+version = "1.8.3"
 description = "Voice dictation for Linux. Speak → text, locally, instantly."
 readme = "README.md"
 license = "MIT"

{python_voiceio-1.8.1 → python_voiceio-1.8.3/python_voiceio.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-voiceio
-Version: 1.8.1
+Version: 1.8.3
 Summary: Voice dictation for Linux. Speak → text, locally, instantly.
 Author: Hugo Montenegro
 License-Expression: MIT

{python_voiceio-1.8.1 → python_voiceio-1.8.3}/python_voiceio.egg-info/SOURCES.txt

@@ -41,6 +41,7 @@ tests/test_ibus_typer.py
 tests/test_idle_audio.py
 tests/test_learn_clean.py
 tests/test_learn_import.py
+tests/test_learn_language.py
 tests/test_learn_metrics.py
 tests/test_learn_pipeline.py
 tests/test_learn_quiet.py

{python_voiceio-1.8.1 → python_voiceio-1.8.3}/tests/test_cancel.py

@@ -156,14 +156,17 @@ def test_cancel_too_late_changes_nothing(tmp_path):
     session = MagicMock()
     session.cancel.return_value = False          # final text already typed
     session.stop.side_effect = lambda audio: (in_stop.set(), release.wait(5), "words")[2]
+    # The finalizer goes on to copy the text from its own thread; left real, it
+    # lands on whatever clipboard a later test has patched in.
+    vio._copy_result_async = MagicMock()
     _finalizing(vio, tmp_path, session)
     assert in_stop.wait(5)
     gen = vio._generation
     vio.cancel()
     assert vio._state == _State.FINALIZING and vio._generation == gen
     release.set()
-    for _ in range(200):
-        if vio._state == _State.IDLE:
+    for _ in range(500):
+        if vio._state == _State.IDLE and not vio._outputs:
             break
         time.sleep(0.01)
     vio.engine.commit.assert_called_once()

{python_voiceio-1.8.1 → python_voiceio-1.8.3}/tests/test_ibus_textlimit.py

@@ -92,3 +92,30 @@ def test_preedit_tail_survives_multibyte_text():
     shown = tail_utf8(text)
     assert _nbytes(shown) <= MAX_TEXT_BYTES
     assert shown == TRUNCATION_MARK + "é" * ((MAX_TEXT_BYTES - _nbytes(TRUNCATION_MARK)) // 2)
+
+
+def test_commit_secs_zero_for_one_chunk():
+    from voiceio.ibus.textlimit import commit_secs
+    assert commit_secs("short note") == 0.0
+
+
+def test_commit_secs_covers_every_chunk_gap(long_note):
+    from voiceio.ibus.textlimit import COMMIT_GAP_MS, commit_secs
+    n = len(split_utf8(long_note))
+    assert n > 1
+    assert commit_secs(long_note) == n * COMMIT_GAP_MS / 1000
+
+
+def test_ibus_typer_holds_a_chunked_commit_until_delivered(long_note, monkeypatch):
+    """Returning early let the input source be restored while tail chunks
+    were still pending, so a long note lost its end."""
+    from voiceio.typers import ibus as ibus_typer
+    t = ibus_typer.IBusTyper.__new__(ibus_typer.IBusTyper)
+    sent, slept = [], []
+    monkeypatch.setattr(t, "_send", sent.append, raising=False)
+    monkeypatch.setattr(t, "_copy_to_clipboard", lambda _t: None, raising=False)
+    monkeypatch.setattr(t, "_engine_has_focus", lambda: True, raising=False)
+    monkeypatch.setattr(ibus_typer.time, "sleep", slept.append)
+    t.commit_text(long_note)
+    assert sent == [f"commit:{long_note}"]
+    assert slept and slept[-1] >= 0.3

python_voiceio-1.8.3/tests/test_learn_language.py

@@ -0,0 +1,39 @@
+"""Fine-tuning and labeling follow the dictation language."""
+from __future__ import annotations
+
+import pytest
+
+from voiceio.cli import learn as learn_cli
+from voiceio.config import Config
+
+
+def _with_language(monkeypatch, lang):
+    cfg = Config()
+    cfg.model.language = lang
+    monkeypatch.setattr(learn_cli, "_cfg", lambda: cfg)
+
+
+def test_training_uses_the_dictation_language(monkeypatch):
+    """A Spanish voice trained under the "en" token learned the wrong prompt."""
+    _with_language(monkeypatch, "es")
+    assert learn_cli._train_language() == "es"
+
+
+def test_training_refuses_auto_with_a_hint(monkeypatch):
+    _with_language(monkeypatch, "auto")
+    with pytest.raises(SystemExit, match="model.language"):
+        learn_cli._train_language()
+
+
+def test_teacher_detects_when_language_is_auto(monkeypatch):
+    from voiceio.learn import teacher
+    monkeypatch.setattr("voiceio.core.decoders.faster_whisper.worker.load_model", lambda *a, **k: object())
+    assert teacher.WhisperTeacher("tiny", language="auto")._language is None
+    assert teacher.WhisperTeacher("tiny", language="es")._language == "es"
+
+
+def test_word_frequency_survives_auto_and_unknown_languages():
+    """language = "auto" raised LookupError in vocabulary biasing."""
+    from voiceio.wordfreq import is_common, is_known
+    assert is_common("house", "auto") and is_known("casa", "es")
+    assert not is_common("house", "xx")

{python_voiceio-1.8.1 → python_voiceio-1.8.3}/tests/test_postprocess.py

@@ -102,7 +102,20 @@ class TestCapitalization:
         assert cleanup("hello world") == "Hello world"
 
     def test_after_period(self):
-        assert cleanup("hello. world") == "Hello. World"
+        assert cleanup("hello? world") == "Hello? World"
+        assert cleanup("Hello. World") == "Hello. World"
+
+    def test_lowercase_after_a_segment_stop_is_a_comma(self):
+        # Whisper ends each segment with "." and continues the sentence in
+        # lowercase; capitalizing it made a false stop permanent.
+        assert cleanup("a bit bigger. and remove the latch.") == "A bit bigger, and remove the latch."
+        assert cleanup("hello. world") == "Hello, world"
+
+    def test_abbreviations_and_dotted_names_keep_their_dot(self):
+        assert cleanup("pears etc. and more") == "Pears etc. and more"
+        assert cleanup("use e.g. this one") == "Use e.g. this one"
+        assert cleanup("go to hugo0.com. and then") == "Go to hugo0.com. and then"
+        assert cleanup("I paused... and then") == "I paused... and then"
 
     def test_after_question_mark(self):
         assert cleanup("really? yes") == "Really? Yes"
@@ -184,7 +197,7 @@ class TestEdgeCases:
         assert result == "..."
 
     def test_unicode_accents(self):
-        assert cleanup("café. résumé") == "Café. Résumé"
+        assert cleanup("café? résumé") == "Café? Résumé"
 
 
 class TestWordMatchCompatibility:

python_voiceio-1.8.3/voiceio/__init__.py

@@ -0,0 +1 @@
+__version__ = "1.8.3"

{python_voiceio-1.8.1 → python_voiceio-1.8.3}/voiceio/cli/learn.py

@@ -417,6 +417,16 @@ def cmd_train(args: argparse.Namespace) -> None:
               f"  (compare with the current model, then `voiceio learn promote {run.name}`)")
 
 
+def _train_language() -> str:
+    """The language a fine-tune is trained under: the one dictation decodes
+    with. "auto" has no single answer, so it has to be set first."""
+    lang = _cfg().model.language or "en"
+    if lang == "auto":
+        raise SystemExit("Fine-tuning needs one language: set it with "
+                         "`voiceio config set model.language es` (your dictation language).")
+    return lang
+
+
 def _train(ds: Path, *, base: str = "small", epochs: int = 4, synthetic_share: float = 0.3,
            full: bool = False, remote: bool = False) -> Path | None:
     """Export `ds` and fine-tune on it; returns the run dir (None for --remote)."""
@@ -433,8 +443,11 @@ def _train(ds: Path, *, base: str = "small", epochs: int = 4, synthetic_share: f
         return None
     train.require_train_deps()
     hotwords = (folder / train.HOTWORDS_FILE).read_text(encoding="utf-8")
+    # The language token the model is trained under must be the one it decodes
+    # with: training a Spanish voice under "en" taught it the wrong prompt.
     hf = train.train(folder, run, train.TrainConfig(
-        base=base, epochs=epochs, decoder_only=False if full else None, hotwords=hotwords))
+        base=base, epochs=epochs, decoder_only=False if full else None, hotwords=hotwords,
+        language=_train_language()))
     train.convert(hf, run / "ct2")
     return run
 

{python_voiceio-1.8.1 → python_voiceio-1.8.3}/voiceio/ibus/engine.py

@@ -30,7 +30,7 @@ from voiceio.ibus import (
     acquire_singleton_lock,
 )
 from voiceio.ibus.pending import PendingBuffer
-from voiceio.ibus.textlimit import split_utf8, tail_utf8
+from voiceio.ibus.textlimit import COMMIT_GAP_MS, split_utf8, tail_utf8
 
 log = logging.getLogger(__name__)
 
@@ -44,9 +44,6 @@ COMPONENT_NAME = "org.voiceio.ibus"
 # largest datagram a sender can produce (SO_SNDBUF, ~208K by default) so that
 # cannot happen.
 RECV_BYTES = 262144
-# Between commit chunks: long enough for the compositor to flush the previous
-# one to the app, short enough that a 10KB note lands in a blink.
-COMMIT_GAP_MS = 40
 
 
 class VoiceIOEngine(IBus.Engine):

{python_voiceio-1.8.1 → python_voiceio-1.8.3}/voiceio/ibus/textlimit.py

@@ -14,6 +14,19 @@ from __future__ import annotations
 
 MAX_TEXT_BYTES = 2048
 TRUNCATION_MARK = "… "
+# Between commit chunks. A chunk the app hasn't acknowledged yet is replaced,
+# not extended, by the next one, and a busy Chromium/Electron window took
+# longer than the 40 ms this used to be: long notes landed as their last chunk.
+COMMIT_GAP_MS = 150
+
+
+def commit_secs(text: str) -> float:
+    """How long the engine needs to deliver ``text``: zero for one chunk, else
+    one gap per extra chunk plus one for the last to be acknowledged. The
+    caller must not hand the input source back before this has passed, or
+    the tail chunks arrive after the app lost focus."""
+    n = len(split_utf8(text))
+    return 0.0 if n <= 1 else n * COMMIT_GAP_MS / 1000
 
 
 def _is_continuation(byte: int) -> bool:

{python_voiceio-1.8.1 → python_voiceio-1.8.3}/voiceio/learn/teacher.py

@@ -45,7 +45,7 @@ class WhisperTeacher:
         # A background batch job (see cli.learn._yield_cpu): use every core idle.
         self._model = load_model(model, device=device, compute_type=compute_type,
                                  cpu_threads=max(4, (os.cpu_count() or 8) // 2))
-        self._language = language
+        self._language = None if language in ("", "auto") else language   # None: detect
         terms = [t for t in vocabulary if t]
         # Prompt, not hotwords: hotwords are capped at 223 tokens; a prompt
         # of the most important terms still biases spelling. The tokenizer

{python_voiceio-1.8.1 → python_voiceio-1.8.3}/voiceio/postprocess.py

@@ -23,6 +23,21 @@ _NO_CASE_LANGUAGES = frozenset({"zh", "ja", "ko", "ar", "he", "th", "hi", "bn",
 _FILLER_RE = re.compile(
     r"[^\S\n]*,?[^\S\n]*\b(?:[Uu]h[-\s]?huh|[Mm]hm|[Uu]h+m*|[Uu]m+|[Hh]m{2,})\b[^\S\n]*,?[^\S\n]*",
 )
+# A word (2+ letters, not part of a dotted name or an ellipsis) ending in "."
+# and followed by a lowercase word: a segment boundary, not a sentence end.
+_SEGMENT_STOP_RE = re.compile(r"(?<![.\w])([A-Za-zÀ-ɏ]{2,})\.(\s+)(?=[a-zà-ÿ])")
+_ABBREVIATIONS = frozenset({
+    "etc", "vs", "approx", "cf", "al", "mr", "mrs", "ms", "dr", "st", "jr", "sr",
+    "inc", "ltd", "co", "fig", "eq", "vol", "no", "pp", "ca", "min", "max", "sec",
+})
+
+
+def _not_an_ending(word: str) -> bool:
+    """The word before a "." is an abbreviation or a dotted name, not a
+    sentence's last word."""
+    return "." in word or word.lower() in _ABBREVIATIONS
+
+
 # A re-decode artifact is a whole duplicated sentence; require this many words
 # so emphatic short repeats ("No. No.", "Stop. Stop.") are preserved.
 _MIN_DEDUP_WORDS = 4
@@ -95,14 +110,28 @@ def cleanup(text: str, language: str = "en") -> str:
     text = re.sub(r"\s+([.,;:?!])", r"\1", text)
 
     if language not in _NO_CASE_LANGUAGES:
+        # Whisper closes every segment with "." even mid-sentence, and the next
+        # segment then carries on in lowercase ("a bit bigger. and remove the
+        # latch"): on real dictation that was 1 in 4 notes. A lowercase
+        # continuation means the sentence did not end, so it gets a comma —
+        # before the capitalization below would make the false stop permanent.
+        text = _SEGMENT_STOP_RE.sub(
+            lambda m: m.group(0) if m.group(1).lower() in _ABBREVIATIONS
+            else m.group(1) + "," + m.group(2),
+            text,
+        )
+
+
         # Capitalize first character
         text = text[0].upper() + text[1:]
 
         # Capitalize after sentence-ending punctuation (an ellipsis is a
-        # pause, not an ending)
+        # pause, not an ending, and so is the dot of "etc.", "e.g." or a name
+        # like "hugo0.com")
         text = re.sub(
-            r"(?<!\.\.)([.?!]\s+)([a-z\u00E0-\u00FF])",
-            lambda m: m.group(1) + m.group(2).upper(),
+            r"(\S*)(?<!\.\.)([.?!]\s+)([a-z\u00E0-\u00FF])",
+            lambda m: m.group(0) if m.group(2)[0] == "." and _not_an_ending(m.group(1))
+            else m.group(1) + m.group(2) + m.group(3).upper(),
             text,
         )
 

{python_voiceio-1.8.1 → python_voiceio-1.8.3}/voiceio/typers/ibus.py

@@ -10,6 +10,7 @@ import time
 
 from voiceio.backends import ProbeResult
 from voiceio.ibus import SOCKET_PATH, install
+from voiceio.ibus.textlimit import commit_secs
 from voiceio.typers import paste
 from voiceio.typers.ydotool import ydotoold_running
 
@@ -123,10 +124,7 @@ class IBusTyper:
     def type_text(self, text: str) -> None:
         if not text:
             return
-        self._send(f"commit:{text}")
-        self._copy_to_clipboard(text)
-        if not self._engine_has_focus():
-            self._simulate_paste()
+        self._commit(text)
 
     def delete_chars(self, n: int) -> None:
         pass  # Not needed: preedit handles corrections atomically
@@ -140,10 +138,17 @@ class IBusTyper:
             self._send("clear")
             return
         log.debug("Committing via IBus (%d chars)", len(text))
+        self._commit(text)
+
+    def _commit(self, text: str) -> None:
         self._send(f"commit:{text}")
         self._copy_to_clipboard(text)
         if not self._engine_has_focus():
             self._simulate_paste()
+            return
+        # A long note goes out in timed chunks; returning early would let the
+        # caller restore the input source while the tail is still pending.
+        time.sleep(commit_secs(text))
 
     def clear_preedit(self) -> None:
         self._send("clear")

{python_voiceio-1.8.1 → python_voiceio-1.8.3}/voiceio/wordfreq.py

@@ -41,14 +41,26 @@ _CONTRACTIONS = frozenset({
 })
 
 
+def _zipf(word: str, language: str) -> float:
+    """Zipf frequency, 0.0 for a language wordfreq has no list for. "auto"
+    (detect per dictation) has no single list either: score it as English,
+    which is where the jargon in a vocabulary list usually comes from."""
+    if language in ("", "auto"):
+        language = "en"
+    try:
+        return zipf_frequency(word.lower(), language)
+    except LookupError:
+        return 0.0
+
+
 def is_common(word: str, language: str = "en", threshold: float = _ZIPF_COMMON_THRESHOLD) -> bool:
     """Check if a word is common (high frequency). Used for suggesting corrections."""
-    return zipf_frequency(word.lower(), language) >= threshold
+    return _zipf(word, language) >= threshold
 
 
 def rank(word: str, language: str = "en") -> int | None:
     """Get approximate frequency rank. None if not found."""
-    z = zipf_frequency(word.lower(), language)
+    z = _zipf(word, language)
     if z == 0.0:
         return None
     # Convert zipf to approximate rank (higher zipf = lower rank number)
@@ -61,7 +73,7 @@ def is_known(word: str, language: str = "en") -> bool:
     wl = word.lower()
     if wl in _CONTRACTIONS:
         return True
-    return zipf_frequency(wl, language) >= _ZIPF_KNOWN_THRESHOLD
+    return _zipf(wl, language) >= _ZIPF_KNOWN_THRESHOLD
 
 
 def extract_words(text: str) -> list[str]:

python_voiceio-1.8.1/voiceio/__init__.py

@@ -1 +0,0 @@
-__version__ = "1.8.1"