polytext 0.2.8b5__tar.gz → 0.2.8b6__tar.gz

{polytext-0.2.8b5 → polytext-0.2.8b6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: polytext
-Version: 0.2.8b5
+Version: 0.2.8b6
 Summary: Python utilities to simplify document files management
 Home-page: https://github.com/docsity/polytext
 Author: Matteo Senardi
@@ -31,6 +31,7 @@ Requires-Dist: boto3>=1.42.64
 Requires-Dist: botocore>=1.42.64
 Requires-Dist: ffmpeg-python==0.2.0
 Requires-Dist: pydub==0.25.1
+Requires-Dist: fast-langdetect==0.2.5
 Requires-Dist: youtube-transcript-api==1.0.3
 Requires-Dist: yt-dlp==2025.5.22
 Requires-Dist: charset-normalizer<4.0.0,>=3.4.2

{polytext-0.2.8b5 → polytext-0.2.8b6}/polytext/converter/audio_to_text.py

@@ -15,6 +15,7 @@ from google.genai import errors as genai_errors
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from google.api_core import exceptions as google_exceptions
 from pydub import AudioSegment
+from fast_langdetect import detect
 
 from ..exceptions import EmptyDocument
 from ..prompts.transcription import (
@@ -73,6 +74,199 @@ AUDIO_PROMPT_VARIANT_DEFAULT = "default"
 AUDIO_PROMPT_VARIANT_NON_LITERAL_FALLBACK = "non_literal_fallback"
 AUDIO_RETRIABLE_OUTPUT_ERROR_CODES = (996, 997, 998, 999)
 NO_HUMAN_SPEECH_MARKER = "no human speech detected"
+AUDIO_LANGUAGE_VALIDATION_MIN_WORDS = int(os.getenv("AUDIO_LANGUAGE_VALIDATION_MIN_WORDS", "50"))
+AUDIO_LANGUAGE_VALIDATION_MIN_CONFIDENCE = float(
+    os.getenv("AUDIO_LANGUAGE_VALIDATION_MIN_CONFIDENCE", "0.80")
+)
+AUDIO_LANGUAGE_VALIDATION_MIN_SUPPORT = float(
+    os.getenv("AUDIO_LANGUAGE_VALIDATION_MIN_SUPPORT", "0.67")
+)
+AUDIO_LANGUAGE_VALIDATION_MIN_FOREIGN_WINDOWS = int(
+    os.getenv("AUDIO_LANGUAGE_VALIDATION_MIN_FOREIGN_WINDOWS", "2")
+)
+
+
+def _normalize_language_detection(result) -> tuple[str, float]:
+    if isinstance(result, list):
+        result = result[0] if result else {}
+    if not isinstance(result, dict):
+        return "", 0.0
+    language = str(result.get("lang", "")).lower().split("-")[0]
+    return language, float(result.get("score", 0.0) or 0.0)
+
+
+def detect_dominant_language(
+        text: str,
+        max_windows: int = 5,
+        window_chars: int = 160,
+) -> dict:
+    """Detect the dominant language over multiple short, evenly spaced text windows."""
+    normalized_text = re.sub(r"\s+", " ", text or "").strip()
+    if not normalized_text:
+        return {
+            "lang": "",
+            "score": 0.0,
+            "support": 0.0,
+            "windows": 0,
+            "language_windows": {},
+        }
+
+    if len(normalized_text) <= window_chars:
+        samples = [normalized_text]
+    else:
+        last_start = len(normalized_text) - window_chars
+        sample_count = min(max_windows, max(2, len(normalized_text) // window_chars))
+        starts = {
+            round(index * last_start / (sample_count - 1))
+            for index in range(sample_count)
+        }
+        samples = [normalized_text[start:start + window_chars] for start in sorted(starts)]
+
+    language_scores = {}
+    language_counts = {}
+    valid_windows = 0
+    for sample in samples:
+        try:
+            detected = detect(sample, low_memory=True)
+        except TypeError:
+            # fast-langdetect >= 1.0 uses model/k instead of low_memory.
+            detected = detect(sample, model="lite", k=1)
+        except Exception:
+            logger.exception("Language detection failed for an audio transcript window")
+            continue
+
+        language, score = _normalize_language_detection(detected)
+        if not language:
+            continue
+        valid_windows += 1
+        language_scores[language] = language_scores.get(language, 0.0) + score
+        language_counts[language] = language_counts.get(language, 0) + 1
+
+    if not valid_windows:
+        return {
+            "lang": "",
+            "score": 0.0,
+            "support": 0.0,
+            "windows": 0,
+            "language_windows": {},
+        }
+
+    dominant_language = max(language_scores, key=language_scores.get)
+    count = language_counts[dominant_language]
+    return {
+        "lang": dominant_language,
+        "score": language_scores[dominant_language] / count,
+        "support": count / valid_windows,
+        "windows": valid_windows,
+        "language_windows": {
+            language: {
+                "windows": language_counts[language],
+                "score": language_scores[language] / language_counts[language],
+            }
+            for language in language_counts
+        },
+    }
+
+
+def find_suspicious_fallback_language_indices(
+        transcripts: list[str],
+        chunk_results: list[dict | None],
+) -> list[int]:
+    """Find internal fallback chunks whose language conflicts with two agreeing neighbours."""
+    suspicious_indices = []
+    logger.info("Starting fallback language validation for %s transcript chunks", len(transcripts))
+
+    def has_fallback(result: dict | None) -> bool:
+        if not result:
+            return False
+        if result.get("fallback_from_model") or result.get("fallback_to_model"):
+            return True
+        return any(has_fallback(item) for item in result.get("split_results", []))
+
+    for index in range(1, len(transcripts) - 1):
+        result = chunk_results[index] or {}
+        if not has_fallback(result):
+            continue
+
+        text = transcripts[index]
+        word_count = len(re.findall(r"\b\w+\b", text, flags=re.UNICODE))
+        if word_count < AUDIO_LANGUAGE_VALIDATION_MIN_WORDS:
+            logger.info(
+                "Skipping language validation for fallback chunk %s: only %s words (minimum %s)",
+                index + 1,
+                word_count,
+                AUDIO_LANGUAGE_VALIDATION_MIN_WORDS,
+            )
+            continue
+
+        candidate_language = detect_dominant_language(text)
+        previous_language = detect_dominant_language(transcripts[index - 1])
+        next_language = detect_dominant_language(transcripts[index + 1])
+        logger.info(
+            "Language validation for fallback chunk %s: previous=%s, candidate=%s, next=%s",
+            index + 1,
+            previous_language,
+            candidate_language,
+            next_language,
+        )
+        neighbour_profiles = (previous_language, next_language)
+        if any(
+            profile["score"] < AUDIO_LANGUAGE_VALIDATION_MIN_CONFIDENCE
+            or profile["support"] < AUDIO_LANGUAGE_VALIDATION_MIN_SUPPORT
+            for profile in neighbour_profiles
+        ):
+            logger.info(
+                "Skipping language comparison for fallback chunk %s: neighbour confidence or support below threshold",
+                index + 1,
+            )
+            continue
+
+        if previous_language["lang"] != next_language["lang"]:
+            logger.info(
+                "Skipping language comparison for fallback chunk %s: neighbouring languages disagree",
+                index + 1,
+            )
+            continue
+
+        expected_language = previous_language["lang"]
+        foreign_windows = {
+            language: profile
+            for language, profile in candidate_language.get("language_windows", {}).items()
+            if (
+                language != expected_language
+                and profile["windows"] >= AUDIO_LANGUAGE_VALIDATION_MIN_FOREIGN_WINDOWS
+                and profile["score"] >= AUDIO_LANGUAGE_VALIDATION_MIN_CONFIDENCE
+            )
+        }
+        if foreign_windows:
+            logger.warning(
+                "Mixed-language anomaly detected for fallback chunk %s: expected %s, foreign windows=%s",
+                index + 1,
+                expected_language,
+                foreign_windows,
+            )
+            suspicious_indices.append(index)
+            continue
+
+        if (
+            candidate_language["score"] >= AUDIO_LANGUAGE_VALIDATION_MIN_CONFIDENCE
+            and candidate_language["support"] >= AUDIO_LANGUAGE_VALIDATION_MIN_SUPPORT
+            and candidate_language["lang"] != expected_language
+        ):
+            logger.warning(
+                "Language anomaly detected for fallback chunk %s: %s between two %s chunks",
+                index + 1,
+                candidate_language["lang"],
+                expected_language,
+            )
+            suspicious_indices.append(index)
+        elif candidate_language["support"] < AUDIO_LANGUAGE_VALIDATION_MIN_SUPPORT:
+            logger.info(
+                "No actionable language anomaly for fallback chunk %s: mixed windows remain below thresholds",
+                index + 1,
+            )
+
+    return suspicious_indices
 
 
 def normalize_no_human_speech_marker(text: str) -> tuple[str, bool]:
@@ -782,6 +976,41 @@ class AudioToTextConverter:
         transcript_dict = self.transcribe_audio(chunk["file_path"])
         return index, transcript_dict
 
+    def recover_suspicious_fallback_languages(
+            self,
+            chunks: list[dict],
+            transcript_chunks: list[str],
+            chunk_results: list[dict | None],
+    ) -> None:
+        """Re-transcribe suspicious fallback chunks in smaller parts, in place."""
+        suspicious_indices = find_suspicious_fallback_language_indices(
+            transcript_chunks,
+            chunk_results,
+        )
+        for index in suspicious_indices:
+            original_result = chunk_results[index] or {}
+            logger.warning(
+                "Fallback transcript language conflicts with both neighbouring chunks; "
+                "re-transcribing chunk %s with adaptive split",
+                index + 1,
+            )
+            try:
+                recovered_result = self.transcribe_audio_halves(chunks[index]["file_path"])
+            except Exception:
+                logger.exception(
+                    "Language-based recovery failed for chunk %s; preserving the original transcript",
+                    index + 1,
+                )
+                continue
+
+            recovered_result["language_validation_triggered"] = True
+            recovered_result["language_validation_original_model"] = original_result.get(
+                "completion_model"
+            )
+            transcript_chunks[index] = recovered_result["transcript"]
+            chunks[index]["transcript"] = recovered_result["transcript"]
+            chunk_results[index] = recovered_result
+
     def format_audio_output_text(self, text: str) -> str:
         return add_line_break_after_each_sentence(text)
 
@@ -869,15 +1098,19 @@ class AudioToTextConverter:
             }
 
             # Process completed transcriptions in order of completion
-            completion_tokens = 0
-            prompt_tokens = 0
             for future in as_completed(future_to_chunk):
                 index, transcript_dict = future.result()
                 chunks[index]["transcript"] = transcript_dict["transcript"]
                 transcript_chunks[index] = transcript_dict["transcript"]
                 chunk_results[index] = transcript_dict
-                completion_tokens += transcript_dict["completion_tokens"]
-                prompt_tokens += transcript_dict["prompt_tokens"]
+
+        self.recover_suspicious_fallback_languages(
+            chunks,
+            transcript_chunks,
+            chunk_results,
+        )
+        completion_tokens = sum(result["completion_tokens"] for result in chunk_results)
+        prompt_tokens = sum(result["prompt_tokens"] for result in chunk_results)
 
         text_merger = TextMerger(llm_api_key=self.llm_api_key)
         # Merge all transcripts

{polytext-0.2.8b5 → polytext-0.2.8b6}/polytext.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: polytext
-Version: 0.2.8b5
+Version: 0.2.8b6
 Summary: Python utilities to simplify document files management
 Home-page: https://github.com/docsity/polytext
 Author: Matteo Senardi
@@ -31,6 +31,7 @@ Requires-Dist: boto3>=1.42.64
 Requires-Dist: botocore>=1.42.64
 Requires-Dist: ffmpeg-python==0.2.0
 Requires-Dist: pydub==0.25.1
+Requires-Dist: fast-langdetect==0.2.5
 Requires-Dist: youtube-transcript-api==1.0.3
 Requires-Dist: yt-dlp==2025.5.22
 Requires-Dist: charset-normalizer<4.0.0,>=3.4.2

{polytext-0.2.8b5 → polytext-0.2.8b6}/polytext.egg-info/requires.txt

@@ -13,6 +13,7 @@ boto3>=1.42.64
 botocore>=1.42.64
 ffmpeg-python==0.2.0
 pydub==0.25.1
+fast-langdetect==0.2.5
 youtube-transcript-api==1.0.3
 yt-dlp==2025.5.22
 charset-normalizer<4.0.0,>=3.4.2

{polytext-0.2.8b5 → polytext-0.2.8b6}/setup.py

@@ -51,7 +51,7 @@ def get_requirements(*requirements_file):
 
 setup(
     name='polytext',
-    version='0.2.8b5',
+    version='0.2.8b6',
     url='https://github.com/docsity/polytext',
     # download_url='https://github.com/pualien/py-polytext/archive/0.1.23.tar.gz',
     license='MIT',

{polytext-0.2.8b5 → polytext-0.2.8b6}/tests/test_audio_transcription_model_migration.py

@@ -1,6 +1,7 @@
 import unittest
 import tempfile
 import os
+import wave
 from types import SimpleNamespace
 from unittest.mock import MagicMock, patch
 from pydub import AudioSegment
@@ -13,6 +14,8 @@ from polytext.converter.audio_to_text import (
     AUDIO_TO_MARKDOWN_PROMPT_IS_RAW,
     AUDIO_TO_PLAIN_TEXT_PROMPT,
     AudioToTextConverter,
+    detect_dominant_language,
+    find_suspicious_fallback_language_indices,
     normalize_no_human_speech_marker,
     transcribe_full_audio,
 )
@@ -84,6 +87,148 @@ class _FakeClient:
         self.models = _FakeModels(responses=responses)
 
 
+class TestFallbackLanguageValidation(unittest.TestCase):
+    @patch("polytext.converter.audio_to_text.detect")
+    def test_detect_dominant_language_aggregates_multiple_text_windows(self, mock_detect):
+        mock_detect.side_effect = [
+            {"lang": "it", "score": 0.96},
+            {"lang": "it", "score": 0.93},
+            {"lang": "en", "score": 0.60},
+        ]
+
+        text = " ".join(f"parola{i}" for i in range(120))
+
+        result = detect_dominant_language(text, max_windows=3, window_chars=120)
+
+        self.assertEqual(result["lang"], "it")
+        self.assertGreater(result["support"], 0.60)
+        self.assertEqual(result["windows"], 3)
+
+    @patch("polytext.converter.audio_to_text.detect_dominant_language")
+    def test_flags_internal_fallback_when_both_neighbours_agree(self, mock_detect):
+        mock_detect.side_effect = [
+            {"lang": "en", "score": 0.98, "support": 1.0, "windows": 3},
+            {"lang": "it", "score": 0.97, "support": 1.0, "windows": 3},
+            {"lang": "it", "score": 0.96, "support": 1.0, "windows": 3},
+        ]
+        transcripts = [
+            "testo italiano " * 60,
+            "unexpected English transcript " * 60,
+            "altro testo italiano " * 60,
+        ]
+        results = [
+            {
+                "transcript": transcripts[0],
+                "fallback_from_model": "gemini-3.1-flash-lite",
+            },
+            {
+                "transcript": transcripts[1],
+                "adaptive_split": True,
+                "split_results": [
+                    {
+                        "transcript": transcripts[1],
+                        "fallback_from_model": "gemini-3.1-flash-lite",
+                    }
+                ],
+            },
+            {
+                "transcript": transcripts[2],
+                "fallback_to_model": "gemini-3.5-flash-lite",
+            },
+        ]
+
+        suspicious = find_suspicious_fallback_language_indices(transcripts, results)
+
+        self.assertEqual(suspicious, [1])
+
+    @patch("polytext.converter.audio_to_text.detect_dominant_language")
+    def test_preserves_possible_multilingual_transition_when_neighbours_disagree(self, mock_detect):
+        mock_detect.side_effect = [
+            {"lang": "en", "score": 0.98, "support": 1.0, "windows": 3},
+            {"lang": "it", "score": 0.97, "support": 1.0, "windows": 3},
+            {"lang": "fr", "score": 0.96, "support": 1.0, "windows": 3},
+        ]
+        transcripts = ["italiano " * 60, "english " * 60, "francais " * 60]
+        results = [
+            {"transcript": transcripts[0]},
+            {"transcript": transcripts[1], "fallback_to_model": "gemini-3.5-flash-lite"},
+            {"transcript": transcripts[2]},
+        ]
+
+        suspicious = find_suspicious_fallback_language_indices(transcripts, results)
+
+        self.assertEqual(suspicious, [])
+
+    @patch("polytext.converter.audio_to_text.detect_dominant_language")
+    def test_flags_fallback_with_two_confident_foreign_windows(self, mock_detect):
+        mock_detect.side_effect = [
+            {
+                "lang": "it",
+                "score": 0.99,
+                "support": 0.6,
+                "windows": 5,
+                "language_windows": {
+                    "it": {"windows": 3, "score": 0.99},
+                    "en": {"windows": 2, "score": 0.95},
+                },
+            },
+            {"lang": "it", "score": 0.98, "support": 1.0, "windows": 5},
+            {"lang": "it", "score": 0.97, "support": 1.0, "windows": 5},
+        ]
+        transcripts = ["italiano " * 60, "testo misto " * 60, "italiano " * 60]
+        results = [
+            {"transcript": transcripts[0]},
+            {"transcript": transcripts[1], "fallback_to_model": "gemini-3.5-flash-lite"},
+            {"transcript": transcripts[2]},
+        ]
+
+        suspicious = find_suspicious_fallback_language_indices(transcripts, results)
+
+        self.assertEqual(suspicious, [1])
+
+    @patch(
+        "polytext.converter.audio_to_text.find_suspicious_fallback_language_indices",
+        return_value=[1],
+    )
+    def test_suspicious_fallback_is_retranscribed_with_adaptive_split(self, _mock_find):
+        converter = AudioToTextConverter()
+        chunks = [
+            {"file_path": "/tmp/chunk-0.wav"},
+            {"file_path": "/tmp/chunk-1.wav"},
+            {"file_path": "/tmp/chunk-2.wav"},
+        ]
+        transcripts = ["prima", "wrong English fallback", "dopo"]
+        results = [
+            {"transcript": "prima", "completion_tokens": 1, "prompt_tokens": 1},
+            {
+                "transcript": "wrong English fallback",
+                "completion_tokens": 2,
+                "prompt_tokens": 2,
+                "completion_model": "gemini-3.5-flash-lite",
+                "fallback_from_model": "gemini-3.1-flash-lite",
+            },
+            {"transcript": "dopo", "completion_tokens": 1, "prompt_tokens": 1},
+        ]
+        recovered = {
+            "transcript": "testo italiano recuperato",
+            "completion_tokens": 4,
+            "prompt_tokens": 3,
+            "completion_model": "gemini-3.1-flash-lite",
+        }
+
+        with patch.object(converter, "transcribe_audio_halves", return_value=recovered) as mock_split:
+            converter.recover_suspicious_fallback_languages(chunks, transcripts, results)
+
+        mock_split.assert_called_once_with("/tmp/chunk-1.wav")
+        self.assertEqual(transcripts[1], "testo italiano recuperato")
+        self.assertEqual(results[1]["transcript"], "testo italiano recuperato")
+        self.assertTrue(results[1]["language_validation_triggered"])
+        self.assertEqual(
+            results[1]["language_validation_original_model"],
+            "gemini-3.5-flash-lite",
+        )
+
+
 class _FlakyServerErrorModels:
     def __init__(self):
         self.generate_content_calls = 0
@@ -455,6 +600,20 @@ class TestAudioTranscriptionModelMigration(unittest.TestCase):
         audio_part = fake_client.models.generate_content_contents[2]
         self.assertEqual(audio_part.inline_data.mime_type, "audio/aac")
 
+    @patch("polytext.converter.audio_to_text.genai.Client")
+    def test_inline_wav_uses_gemini_supported_mime_type(self, mock_client_cls):
+        fake_client = _FakeClient()
+        mock_client_cls.return_value = fake_client
+
+        converter = AudioToTextConverter()
+        with tempfile.NamedTemporaryFile(suffix=".wav") as temp_audio:
+            temp_audio.write(b"fake-audio")
+            temp_audio.flush()
+            converter.transcribe_audio(temp_audio.name)
+
+        audio_part = fake_client.models.generate_content_contents[2]
+        self.assertEqual(audio_part.inline_data.mime_type, "audio/wav")
+
     @patch("retry.api.time.sleep")
     def test_does_not_retry_genai_client_error_for_audio_transcription(self, _mock_sleep):
         fake_client = _ClientErrorClient()
@@ -613,10 +772,18 @@ class TestAudioTranscriptionModelMigration(unittest.TestCase):
         with tempfile.NamedTemporaryFile(suffix=".mp3") as source_audio:
             AudioSegment.silent(duration=10_000).export(source_audio.name, format="mp3").close()
             converter = AudioToTextConverter()
+            def transcribe_split(split_path, temperature=0.0):
+                self.assertTrue(split_path.endswith(".wav"))
+                with wave.open(split_path, "rb") as split_audio:
+                    self.assertEqual(split_audio.getframerate(), 16000)
+                    self.assertEqual(split_audio.getnchannels(), 1)
+                    self.assertEqual(split_audio.getsampwidth(), 2)
+                return split_responses.pop(0)
+
             with patch.object(
                 AudioToTextConverter,
                 "transcribe_audio",
-                side_effect=split_responses,
+                side_effect=transcribe_split,
             ) as mock_transcribe:
                 result = converter.transcribe_audio_halves(source_audio.name)