python-voiceio 0.9.5__tar.gz → 0.9.6__tar.gz

{python_voiceio-0.9.5/python_voiceio.egg-info → python_voiceio-0.9.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-voiceio
-Version: 0.9.5
+Version: 0.9.6
 Summary: Voice dictation for Linux. Speak → text, locally, instantly.
 Author: Hugo Montenegro
 License-Expression: MIT

{python_voiceio-0.9.5 → python_voiceio-0.9.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "python-voiceio"
-version = "0.9.5"
+version = "0.9.6"
 description = "Voice dictation for Linux. Speak → text, locally, instantly."
 readme = "README.md"
 license = "MIT"

{python_voiceio-0.9.5 → python_voiceio-0.9.6/python_voiceio.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: python-voiceio
-Version: 0.9.5
+Version: 0.9.6
 Summary: Voice dictation for Linux. Speak → text, locally, instantly.
 Author: Hugo Montenegro
 License-Expression: MIT

{python_voiceio-0.9.5 → python_voiceio-0.9.6}/tests/test_postcorrect.py

@@ -7,11 +7,13 @@ from voiceio.config import Config, PostCorrectConfig
 from voiceio.postcorrect import PostCorrector
 
 
-def _cfg(*, enabled=True, api_key="test-key", min_words=4, model="") -> Config:
+def _cfg(*, enabled=True, api_key="test-key", min_words=4, model="",
+         remove_disfluencies=False) -> Config:
     cfg = Config()
     cfg.postcorrect = PostCorrectConfig(
         enabled=enabled, min_words=min_words, model=model,
     )
+    cfg.output.remove_disfluencies = remove_disfluencies
     cfg.autocorrect.api_key = api_key
     return cfg
 
@@ -79,6 +81,56 @@ def test_word_count_guard_rejects_length_change():
         assert pc.correct(original) == original
 
 
+# ── disfluency mode: delete-only, never change meaning ───────────────────
+
+
+def test_disfluency_mode_allows_deletion():
+    """Removing filler shrinks the text well past the 20% fix-mode ceiling —
+    the disfluency guard must allow it (deletions only)."""
+    pc = PostCorrector(_cfg(remove_disfluencies=True))
+    original = "so um we should like ship the thing today"
+    cleaned = "so we should ship the thing today"  # dropped 'um' and 'like'
+    with patch("voiceio.llm_api.chat", return_value=cleaned):
+        assert pc.correct(original) == cleaned
+
+
+def test_disfluency_mode_rejects_insertion():
+    """Adding any word could alter meaning — reject, keep the original."""
+    pc = PostCorrector(_cfg(remove_disfluencies=True))
+    original = "we should ship the thing today"
+    added = "we should ship the thing today and also test everything first"
+    with patch("voiceio.llm_api.chat", return_value=added):
+        assert pc.correct(original) == original
+
+
+def test_disfluency_mode_rejects_overdeletion():
+    """Deleting more than the cap means real content was stripped — reject."""
+    pc = PostCorrector(_cfg(remove_disfluencies=True))
+    original = "please run the full test suite before you deploy tonight okay"
+    gutted = "please run test deploy"  # >40% of words gone
+    with patch("voiceio.llm_api.chat", return_value=gutted):
+        assert pc.correct(original) == original
+
+
+def test_disfluency_mode_rejects_rewording():
+    """Many substitutions = a rewrite, not filler removal — reject."""
+    pc = PostCorrector(_cfg(remove_disfluencies=True))
+    original = "the server crashed at midnight during the backup"
+    reworded = "the machine went down at noon while copying files"
+    with patch("voiceio.llm_api.chat", return_value=reworded):
+        assert pc.correct(original) == original
+
+
+def test_fix_mode_still_rejects_big_deletion():
+    """With disfluency mode OFF, the original conservative guards stand: a big
+    deletion is a length change and must be rejected."""
+    pc = PostCorrector(_cfg(remove_disfluencies=False))
+    original = "so um we should like ship the thing today"
+    cleaned = "so we should ship the thing today"
+    with patch("voiceio.llm_api.chat", return_value=cleaned):
+        assert pc.correct(original) == original
+
+
 def test_llm_error_returns_original():
     pc = PostCorrector(_cfg())
     with patch("voiceio.llm_api.chat", side_effect=RuntimeError("boom")):

{python_voiceio-0.9.5 → python_voiceio-0.9.6}/tests/test_postprocess.py

@@ -1,10 +1,56 @@
 """Tests for text post-processing (punctuation, capitalization)."""
 from __future__ import annotations
 
-from voiceio.postprocess import apply_pipeline, cleanup
+from voiceio.postprocess import apply_pipeline, cleanup, strip_disfluencies
 from voiceio.streaming import _word_match_len
 
 
+class TestStripDisfluencies:
+    """Delete-only regex layer: filler sounds + duplicate re-decode sentences,
+    never anything that could change meaning."""
+
+    def test_removes_filler_sounds(self):
+        out = strip_disfluencies("so um we should uh ship it")
+        assert "um" not in out.split()
+        assert "uh" not in out.split()
+        assert "we should" in out and "ship it" in out
+
+    def test_removes_comma_wrapped_filler(self):
+        assert "uh" not in strip_disfluencies("we tested, uh, everything").split()
+
+    def test_preserves_valid_word_repetition(self):
+        # "had had" is valid English; regex must NOT touch word repeats — that
+        # judgment belongs to the LLM layer.
+        assert strip_disfluencies("I had had enough") == "I had had enough"
+
+    def test_dedups_duplicate_sentence(self):
+        # The Whisper re-decode artifact: an exact sentence repeated.
+        out = strip_disfluencies("Do the research. Do the research.")
+        assert out == "Do the research."
+
+    def test_keeps_meaningful_words(self):
+        # 'like' as a real verb/preposition and content must survive.
+        text = "I like the design and it works like a charm"
+        assert strip_disfluencies(text) == text
+
+    def test_empty(self):
+        assert strip_disfluencies("") == ""
+
+
+class TestPipelineDisfluencies:
+    def test_flag_off_keeps_fillers(self):
+        text, _ = apply_pipeline("um hello there friend", do_cleanup=True, final=True)
+        assert "um" in text.lower().split()
+
+    def test_flag_on_strips_fillers(self):
+        text, _ = apply_pipeline(
+            "um hello there friend",
+            do_cleanup=True, remove_disfluencies=True, final=True,
+        )
+        assert "um" not in text.lower().split()
+        assert "hello there friend" in text.lower()
+
+
 class TestCapitalization:
     def test_first_char(self):
         assert cleanup("hello world") == "Hello world"

python_voiceio-0.9.6/voiceio/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.9.6"

{python_voiceio-0.9.5 → python_voiceio-0.9.6}/voiceio/app.py

@@ -188,6 +188,7 @@ class VoiceIO:
         self._command_processor = CommandProcessor(enabled=cfg.commands.enabled, editing=cfg.commands.editing)
         self._cleanup = cfg.output.punctuation_cleanup
         self._number_conversion = cfg.output.number_conversion
+        self._remove_disfluencies = cfg.output.remove_disfluencies
         self._voice_input_prefix = cfg.output.voice_input_prefix
         self._streaming = cfg.output.streaming
 
@@ -458,6 +459,7 @@ class VoiceIO:
                 self.transcriber, self._typer, self.recorder,
                 generation=self._generation,
                 cleanup=self._cleanup,
+                remove_disfluencies=self._remove_disfluencies,
                 number_conversion=self._number_conversion,
                 language=self.cfg.model.language,
                 commands=self._command_processor,
@@ -669,6 +671,7 @@ class VoiceIO:
                 text, abort = apply_pipeline(
                     text,
                     do_cleanup=self._cleanup,
+                    remove_disfluencies=self._remove_disfluencies,
                     number_conversion=self._number_conversion,
                     language=self.cfg.model.language,
                     commands=self._command_processor,

{python_voiceio-0.9.5 → python_voiceio-0.9.6}/voiceio/config.py

@@ -88,6 +88,12 @@ class OutputConfig:
     cancel_window_secs: float = 0.5
     punctuation_cleanup: bool = True
     number_conversion: bool = True
+    # Strip spoken disfluencies so dictation reads as writing. Delete-only and
+    # meaning-preserving: a free regex layer removes filler sounds (um/uh/er)
+    # and duplicate re-decode sentences on every pass; when [postcorrect] is
+    # also enabled, its LLM pass additionally removes false starts and filler
+    # "like", guarded so it can only delete — never add, rephrase, or reorder.
+    remove_disfluencies: bool = False
     voice_input_prefix: str = ""           # e.g. "[voice]" — empty disables
     # Incremental finalization: once the un-finalized audio tail grows past
     # this many seconds, it is cut at the nearest interior speech pause,

{python_voiceio-0.9.5 → python_voiceio-0.9.6}/voiceio/postcorrect.py

@@ -26,6 +26,14 @@ log = logging.getLogger(__name__)
 _MAX_EDIT_RATIO = 0.3    # word-level SequenceMatcher edit ratio ceiling
 _MAX_WORDCOUNT_DELTA = 0.2  # allowed relative change in word count
 
+# Disfluency-mode guards. The whole promise is "never change meaning", enforced
+# structurally on the word-level diff: no INSERTIONS (can't add or rephrase),
+# few REPLACEMENTS (ASR fixes only, not reword), bounded DELETIONS (can't nuke
+# real content). Deletions are what disfluency removal legitimately does.
+_MAX_INSERTED_WORDS = 0     # adding any word = altering meaning → reject
+_MAX_REPLACE_FRAC = 0.15    # ASR word-fixes only, never wholesale rewording
+_MAX_DELETE_FRAC = 0.4      # backstop against deleting real content
+
 _SYSTEM_PROMPT = (
     "You fix automatic speech recognition errors in dictated text. "
     "The user dictates about software engineering and their projects. "
@@ -35,6 +43,23 @@ _SYSTEM_PROMPT = (
     "Return only the corrected text with no commentary."
 )
 
+# Disfluency mode: also strip spoken filler, delete-only. The strict rules
+# mirror the guards — the model is told exactly what the diff check enforces.
+_SYSTEM_PROMPT_CLEAN = (
+    "You convert dictated speech into clean written text. The user dictates "
+    "about software engineering and their projects. Do exactly two things:\n"
+    "1. Fix words the recognizer misheard (wrong proper nouns, homophones, "
+    "garbled technical terms).\n"
+    "2. Remove speech disfluencies: filler sounds (um, uh, er); filler uses of "
+    "'like', 'you know', 'I mean'; false starts and self-corrections (keep the "
+    "corrected version); and stray word repetitions.\n"
+    "STRICT RULES: Only DELETE disfluencies and FIX misheard words. NEVER add "
+    "words. NEVER rephrase, reword, reorder, or summarize. NEVER drop real "
+    "content, meaningful hedges, or negations. If unsure whether something is a "
+    "disfluency, KEEP it. Preserve the speaker's own wording and punctuation. "
+    "Return only the cleaned text, nothing else."
+)
+
 _MAX_RECENT = 3
 
 
@@ -82,6 +107,7 @@ class PostCorrector:
     def __init__(self, cfg: Config):
         self._cfg = cfg
         self._pc = cfg.postcorrect
+        self._remove_disfluencies = cfg.output.remove_disfluencies
         # API key / base_url resolution is shared with [autocorrect].
         self._ac = cfg.autocorrect
         self._available: bool | None = None
@@ -210,10 +236,14 @@ class PostCorrector:
         t0 = time.monotonic()
         outcome: dict = {}
 
+        system_prompt = (
+            _SYSTEM_PROMPT_CLEAN if self._remove_disfluencies else _SYSTEM_PROMPT
+        )
+
         def _call() -> None:
             try:
                 outcome["response"] = chat(
-                    self._client_cfg(), _SYSTEM_PROMPT, user_msg, max_tokens=1024,
+                    self._client_cfg(), system_prompt, user_msg, max_tokens=1024,
                 )
             except Exception as e:
                 outcome["error"] = e
@@ -251,26 +281,47 @@ class PostCorrector:
             self._record(text, corrected, "unchanged")
             return text
 
-        # Guard: word-count must not change materially.
-        orig_wc, new_wc = len(text.split()), len(corrected.split())
-        if orig_wc and abs(new_wc - orig_wc) / orig_wc > _MAX_WORDCOUNT_DELTA:
-            log.debug(
-                "PostCorrector reject: word count %d→%d (>%.0f%%) — keeping original",
-                orig_wc, new_wc, _MAX_WORDCOUNT_DELTA * 100,
-            )
-            self._record(text, corrected, "rejected_wordcount")
-            return text
-
-        # Guard: only a small fraction of words may change.
-        ratio = _word_edit_ratio(text, corrected)
-        if ratio > _MAX_EDIT_RATIO:
-            log.debug(
-                "PostCorrector reject: edit ratio %.2f > %.2f — keeping original",
-                ratio, _MAX_EDIT_RATIO,
-            )
-            self._record(text, corrected, "rejected_editratio")
+        accept, reject_reason = self._guard(text, corrected)
+        if not accept:
+            log.debug("PostCorrector reject (%s) — keeping original", reject_reason)
+            self._record(text, corrected, f"rejected_{reject_reason}")
             return text
 
         log.info("PostCorrector fixed: %s", ", ".join(_changed_words(text, corrected)))
         self._record(text, corrected, "applied")
         return corrected
+
+    def _guard(self, text: str, corrected: str) -> tuple[bool, str]:
+        """Decide whether the LLM's edit is within bounds.
+
+        Returns (accept, reject_reason). In disfluency mode the promise is
+        "never change meaning", enforced on the word-level diff: zero
+        insertions (nothing added/rephrased), few replacements (ASR fixes, not
+        rewording), bounded deletions (can't strip real content). Otherwise the
+        original conservative fix-only guards apply.
+        """
+        aw, bw = text.split(), corrected.split()
+        if not self._remove_disfluencies:
+            n = len(aw)
+            if n and abs(len(bw) - n) / n > _MAX_WORDCOUNT_DELTA:
+                return False, "wordcount"
+            if _word_edit_ratio(text, corrected) > _MAX_EDIT_RATIO:
+                return False, "editratio"
+            return True, ""
+
+        inserted = replaced = deleted = 0
+        for tag, i1, i2, j1, j2 in difflib.SequenceMatcher(a=aw, b=bw).get_opcodes():
+            if tag == "insert":
+                inserted += j2 - j1
+            elif tag == "replace":
+                replaced += max(i2 - i1, j2 - j1)
+            elif tag == "delete":
+                deleted += i2 - i1
+        n = len(aw)
+        if inserted > _MAX_INSERTED_WORDS:
+            return False, "inserted"          # added content — meaning changed
+        if replaced > max(3, int(_MAX_REPLACE_FRAC * n)):
+            return False, "reworded"           # too many substitutions = rewrite
+        if n and deleted > _MAX_DELETE_FRAC * n:
+            return False, "overdeleted"        # nuked real content
+        return True, ""

{python_voiceio-0.9.5 → python_voiceio-0.9.6}/voiceio/postprocess.py

@@ -13,6 +13,43 @@ if TYPE_CHECKING:
 # Languages that don't use letter casing
 _NO_CASE_LANGUAGES = frozenset({"zh", "ja", "ko", "ar", "he", "th", "hi", "bn", "ka", "my"})
 
+# Filler SOUNDS only — tokens with no lexical meaning, so deleting them can
+# never change meaning. Word repetitions ("had had" is valid English) and
+# filler uses of "like"/"you know" need judgment and are left to the LLM layer.
+# Consumes an adjacent comma on either side so "be, uh, found" → "be found".
+_FILLER_RE = re.compile(
+    r"\s*,?\s*\b(?:u+m+|u+h+m*|e+r+m*|erm+|a+h+|h+m+|mm+|mhm|uh[-\s]?huh)\b\s*,?\s*",
+    re.IGNORECASE,
+)
+
+
+def strip_disfluencies(text: str) -> str:
+    """Delete-only, meaning-safe disfluency cleanup (regex layer).
+
+    Removes filler sounds (um, uh, er, …) and exact duplicate adjacent
+    sentences (a Whisper re-decode artifact). Only ever deletes — never
+    rephrases, reorders, or touches lexical words — so meaning is preserved by
+    construction. The judgment cases (false starts, filler "like", word
+    repetitions) are handled by the guarded LLM layer in postcorrect.
+    """
+    if not text:
+        return text
+    text = _FILLER_RE.sub(" ", text)
+    text = _dedup_adjacent_sentences(text)
+    text = re.sub(r"\s{2,}", " ", text)
+    return text.strip()
+
+
+def _dedup_adjacent_sentences(text: str) -> str:
+    """Drop a sentence identical to the one immediately before it."""
+    parts = re.split(r"(?<=[.?!])\s+", text)
+    out: list[str] = []
+    for p in parts:
+        if out and p.strip().lower() == out[-1].strip().lower():
+            continue
+        out.append(p)
+    return " ".join(out)
+
 
 def cleanup(text: str, language: str = "en") -> str:
     """Clean up Whisper output. Idempotent — safe to run multiple times."""
@@ -50,6 +87,7 @@ def apply_pipeline(
     text: str,
     *,
     do_cleanup: bool = False,
+    remove_disfluencies: bool = False,
     number_conversion: bool = False,
     language: str = "en",
     commands: CommandProcessor | None = None,
@@ -64,6 +102,11 @@ def apply_pipeline(
     Returns (processed_text, abort). If abort is True, the caller should
     discard the result (e.g. undo/flag command was triggered).
     """
+    # Disfluency removal runs BEFORE cleanup so cleanup re-fixes the spacing,
+    # commas, and capitalization the deletions leave behind. Delete-only.
+    if remove_disfluencies:
+        text = strip_disfluencies(text)
+
     if do_cleanup:
         text = cleanup(text, language)
 

{python_voiceio-0.9.5 → python_voiceio-0.9.6}/voiceio/streaming.py

@@ -135,6 +135,7 @@ class StreamingSession:
         recorder: AudioRecorder,
         generation: int = 0,
         cleanup: bool = False,
+        remove_disfluencies: bool = False,
         number_conversion: bool = False,
         language: str = "en",
         commands: CommandProcessor | None = None,
@@ -158,6 +159,7 @@ class StreamingSession:
         self._sample_rate = recorder.sample_rate
         self._generation = generation
         self._cleanup = cleanup
+        self._remove_disfluencies = remove_disfluencies
         self._number_conversion = number_conversion
         self._language = language
         self._commands = commands
@@ -450,6 +452,7 @@ class StreamingSession:
             text, abort = apply_pipeline(
                 raw,
                 do_cleanup=self._cleanup,
+                remove_disfluencies=self._remove_disfluencies,
                 number_conversion=self._number_conversion,
                 language=self._language,
                 commands=self._commands,

python_voiceio-0.9.5/voiceio/__init__.py

@@ -1 +0,0 @@
-__version__ = "0.9.5"