polytext 0.2.8b5__tar.gz → 0.2.8b7__tar.gz

{polytext-0.2.8b5 → polytext-0.2.8b7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: polytext
-Version: 0.2.8b5
+Version: 0.2.8b7
 Summary: Python utilities to simplify document files management
 Home-page: https://github.com/docsity/polytext
 Author: Matteo Senardi
@@ -31,6 +31,7 @@ Requires-Dist: boto3>=1.42.64
 Requires-Dist: botocore>=1.42.64
 Requires-Dist: ffmpeg-python==0.2.0
 Requires-Dist: pydub==0.25.1
+Requires-Dist: fast-langdetect==0.2.5
 Requires-Dist: youtube-transcript-api==1.0.3
 Requires-Dist: yt-dlp==2025.5.22
 Requires-Dist: charset-normalizer<4.0.0,>=3.4.2

{polytext-0.2.8b5 → polytext-0.2.8b7}/polytext/converter/audio_to_text.py

@@ -15,6 +15,7 @@ from google.genai import errors as genai_errors
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from google.api_core import exceptions as google_exceptions
 from pydub import AudioSegment
+from fast_langdetect import detect
 
 from ..exceptions import EmptyDocument
 from ..prompts.transcription import (
@@ -62,6 +63,8 @@ AUDIO_DEFAULT_MAX_OUTPUT_TOKENS = 4096
 AUDIO_MAX_ADAPTIVE_SPLIT_DEPTH = 1
 AUDIO_ADAPTIVE_SPLIT_OVERLAP_MS = 2000
 AUDIO_LONG_DURATION_THRESHOLD_MS = 80 * 60 * 1000
+AUDIO_LOW_TEXT_THRESHOLD_WORDS_PER_MINUTE = 20
+AUDIO_LOW_TEXT_MIN_DURATION_MS = 60_000
 AUDIO_TAIL_REPETITION_LINES = int(os.getenv("AUDIO_TAIL_REPETITION_LINES", "200"))
 AUDIO_TAIL_REPETITION_THRESHOLD = float(os.getenv("AUDIO_TAIL_REPETITION_THRESHOLD", "0.35"))
 AUDIO_FALLBACK_SOURCE_PATTERN = os.getenv("AUDIO_FALLBACK_SOURCE_PATTERN", "flash-lite")
@@ -73,6 +76,199 @@ AUDIO_PROMPT_VARIANT_DEFAULT = "default"
 AUDIO_PROMPT_VARIANT_NON_LITERAL_FALLBACK = "non_literal_fallback"
 AUDIO_RETRIABLE_OUTPUT_ERROR_CODES = (996, 997, 998, 999)
 NO_HUMAN_SPEECH_MARKER = "no human speech detected"
+AUDIO_LANGUAGE_VALIDATION_MIN_WORDS = int(os.getenv("AUDIO_LANGUAGE_VALIDATION_MIN_WORDS", "50"))
+AUDIO_LANGUAGE_VALIDATION_MIN_CONFIDENCE = float(
+    os.getenv("AUDIO_LANGUAGE_VALIDATION_MIN_CONFIDENCE", "0.80")
+)
+AUDIO_LANGUAGE_VALIDATION_MIN_SUPPORT = float(
+    os.getenv("AUDIO_LANGUAGE_VALIDATION_MIN_SUPPORT", "0.67")
+)
+AUDIO_LANGUAGE_VALIDATION_MIN_FOREIGN_WINDOWS = int(
+    os.getenv("AUDIO_LANGUAGE_VALIDATION_MIN_FOREIGN_WINDOWS", "2")
+)
+
+
+def _normalize_language_detection(result) -> tuple[str, float]:
+    if isinstance(result, list):
+        result = result[0] if result else {}
+    if not isinstance(result, dict):
+        return "", 0.0
+    language = str(result.get("lang", "")).lower().split("-")[0]
+    return language, float(result.get("score", 0.0) or 0.0)
+
+
+def detect_dominant_language(
+        text: str,
+        max_windows: int = 5,
+        window_chars: int = 160,
+) -> dict:
+    """Detect the dominant language over multiple short, evenly spaced text windows."""
+    normalized_text = re.sub(r"\s+", " ", text or "").strip()
+    if not normalized_text:
+        return {
+            "lang": "",
+            "score": 0.0,
+            "support": 0.0,
+            "windows": 0,
+            "language_windows": {},
+        }
+
+    if len(normalized_text) <= window_chars:
+        samples = [normalized_text]
+    else:
+        last_start = len(normalized_text) - window_chars
+        sample_count = min(max_windows, max(2, len(normalized_text) // window_chars))
+        starts = {
+            round(index * last_start / (sample_count - 1))
+            for index in range(sample_count)
+        }
+        samples = [normalized_text[start:start + window_chars] for start in sorted(starts)]
+
+    language_scores = {}
+    language_counts = {}
+    valid_windows = 0
+    for sample in samples:
+        try:
+            detected = detect(sample, low_memory=True)
+        except TypeError:
+            # fast-langdetect >= 1.0 uses model/k instead of low_memory.
+            detected = detect(sample, model="lite", k=1)
+        except Exception:
+            logger.exception("Language detection failed for an audio transcript window")
+            continue
+
+        language, score = _normalize_language_detection(detected)
+        if not language:
+            continue
+        valid_windows += 1
+        language_scores[language] = language_scores.get(language, 0.0) + score
+        language_counts[language] = language_counts.get(language, 0) + 1
+
+    if not valid_windows:
+        return {
+            "lang": "",
+            "score": 0.0,
+            "support": 0.0,
+            "windows": 0,
+            "language_windows": {},
+        }
+
+    dominant_language = max(language_scores, key=language_scores.get)
+    count = language_counts[dominant_language]
+    return {
+        "lang": dominant_language,
+        "score": language_scores[dominant_language] / count,
+        "support": count / valid_windows,
+        "windows": valid_windows,
+        "language_windows": {
+            language: {
+                "windows": language_counts[language],
+                "score": language_scores[language] / language_counts[language],
+            }
+            for language in language_counts
+        },
+    }
+
+
+def find_suspicious_fallback_language_indices(
+        transcripts: list[str],
+        chunk_results: list[dict | None],
+) -> list[int]:
+    """Find internal fallback chunks whose language conflicts with two agreeing neighbours."""
+    suspicious_indices = []
+    logger.info("Starting fallback language validation for %s transcript chunks", len(transcripts))
+
+    def has_fallback(result: dict | None) -> bool:
+        if not result:
+            return False
+        if result.get("fallback_from_model") or result.get("fallback_to_model"):
+            return True
+        return any(has_fallback(item) for item in result.get("split_results", []))
+
+    for index in range(1, len(transcripts) - 1):
+        result = chunk_results[index] or {}
+        if not has_fallback(result):
+            continue
+
+        text = transcripts[index]
+        word_count = len(re.findall(r"\b\w+\b", text, flags=re.UNICODE))
+        if word_count < AUDIO_LANGUAGE_VALIDATION_MIN_WORDS:
+            logger.info(
+                "Skipping language validation for fallback chunk %s: only %s words (minimum %s)",
+                index + 1,
+                word_count,
+                AUDIO_LANGUAGE_VALIDATION_MIN_WORDS,
+            )
+            continue
+
+        candidate_language = detect_dominant_language(text)
+        previous_language = detect_dominant_language(transcripts[index - 1])
+        next_language = detect_dominant_language(transcripts[index + 1])
+        logger.info(
+            "Language validation for fallback chunk %s: previous=%s, candidate=%s, next=%s",
+            index + 1,
+            previous_language,
+            candidate_language,
+            next_language,
+        )
+        neighbour_profiles = (previous_language, next_language)
+        if any(
+            profile["score"] < AUDIO_LANGUAGE_VALIDATION_MIN_CONFIDENCE
+            or profile["support"] < AUDIO_LANGUAGE_VALIDATION_MIN_SUPPORT
+            for profile in neighbour_profiles
+        ):
+            logger.info(
+                "Skipping language comparison for fallback chunk %s: neighbour confidence or support below threshold",
+                index + 1,
+            )
+            continue
+
+        if previous_language["lang"] != next_language["lang"]:
+            logger.info(
+                "Skipping language comparison for fallback chunk %s: neighbouring languages disagree",
+                index + 1,
+            )
+            continue
+
+        expected_language = previous_language["lang"]
+        foreign_windows = {
+            language: profile
+            for language, profile in candidate_language.get("language_windows", {}).items()
+            if (
+                language != expected_language
+                and profile["windows"] >= AUDIO_LANGUAGE_VALIDATION_MIN_FOREIGN_WINDOWS
+                and profile["score"] >= AUDIO_LANGUAGE_VALIDATION_MIN_CONFIDENCE
+            )
+        }
+        if foreign_windows:
+            logger.warning(
+                "Mixed-language anomaly detected for fallback chunk %s: expected %s, foreign windows=%s",
+                index + 1,
+                expected_language,
+                foreign_windows,
+            )
+            suspicious_indices.append(index)
+            continue
+
+        if (
+            candidate_language["score"] >= AUDIO_LANGUAGE_VALIDATION_MIN_CONFIDENCE
+            and candidate_language["support"] >= AUDIO_LANGUAGE_VALIDATION_MIN_SUPPORT
+            and candidate_language["lang"] != expected_language
+        ):
+            logger.warning(
+                "Language anomaly detected for fallback chunk %s: %s between two %s chunks",
+                index + 1,
+                candidate_language["lang"],
+                expected_language,
+            )
+            suspicious_indices.append(index)
+        elif candidate_language["support"] < AUDIO_LANGUAGE_VALIDATION_MIN_SUPPORT:
+            logger.info(
+                "No actionable language anomaly for fallback chunk %s: mixed windows remain below thresholds",
+                index + 1,
+            )
+
+    return suspicious_indices
 
 
 def normalize_no_human_speech_marker(text: str) -> tuple[str, bool]:
@@ -90,6 +286,37 @@ def normalize_no_human_speech_marker(text: str) -> tuple[str, bool]:
     return cleaned_text, False
 
 
+def summarize_audio_chunk_quality(chunks: list[dict], transcripts: list[str]) -> dict:
+    """Report empty and unusually short transcripts without discarding any text.
+
+    Indices are one-based. Short audio chunks are excluded from the low-text
+    count because a brief spoken phrase can be a complete transcription.
+    """
+    empty_indices = []
+    low_text_indices = []
+    for index, (chunk, transcript) in enumerate(zip(chunks, transcripts), start=1):
+        word_count = len(transcript.split())
+        if word_count == 0:
+            empty_indices.append(index)
+        else:
+            duration_ms = chunk.get("duration_ms", 0)
+            if (
+                duration_ms >= AUDIO_LOW_TEXT_MIN_DURATION_MS
+                and word_count * 60_000 < AUDIO_LOW_TEXT_THRESHOLD_WORDS_PER_MINUTE * duration_ms
+            ):
+                low_text_indices.append(index)
+
+    return {
+        "total_chunks": len(chunks),
+        "empty_chunks": len(empty_indices),
+        "low_text_chunks": len(low_text_indices),
+        "empty_chunk_indices": empty_indices,
+        "low_text_chunk_indices": low_text_indices,
+        "low_text_threshold_words_per_minute": AUDIO_LOW_TEXT_THRESHOLD_WORDS_PER_MINUTE,
+        "low_text_min_duration_ms": AUDIO_LOW_TEXT_MIN_DURATION_MS,
+    }
+
+
 def add_line_break_after_each_sentence(text: str) -> str:
     if not text:
         return text
@@ -782,6 +1009,41 @@ class AudioToTextConverter:
         transcript_dict = self.transcribe_audio(chunk["file_path"])
         return index, transcript_dict
 
+    def recover_suspicious_fallback_languages(
+            self,
+            chunks: list[dict],
+            transcript_chunks: list[str],
+            chunk_results: list[dict | None],
+    ) -> None:
+        """Re-transcribe suspicious fallback chunks in smaller parts, in place."""
+        suspicious_indices = find_suspicious_fallback_language_indices(
+            transcript_chunks,
+            chunk_results,
+        )
+        for index in suspicious_indices:
+            original_result = chunk_results[index] or {}
+            logger.warning(
+                "Fallback transcript language conflicts with both neighbouring chunks; "
+                "re-transcribing chunk %s with adaptive split",
+                index + 1,
+            )
+            try:
+                recovered_result = self.transcribe_audio_halves(chunks[index]["file_path"])
+            except Exception:
+                logger.exception(
+                    "Language-based recovery failed for chunk %s; preserving the original transcript",
+                    index + 1,
+                )
+                continue
+
+            recovered_result["language_validation_triggered"] = True
+            recovered_result["language_validation_original_model"] = original_result.get(
+                "completion_model"
+            )
+            transcript_chunks[index] = recovered_result["transcript"]
+            chunks[index]["transcript"] = recovered_result["transcript"]
+            chunk_results[index] = recovered_result
+
     def format_audio_output_text(self, text: str) -> str:
         return add_line_break_after_each_sentence(text)
 
@@ -801,6 +1063,8 @@ class AudioToTextConverter:
                 - prompt_tokens (int): Total number of prompt tokens used
                 - completion_model (str): Name of the transcription model used
                 - completion_model_provider (str): Provider of the transcription model
+                - audio_chunk_quality (dict): Counts of empty and low-text chunks,
+                  with one-based indices and the applied words-per-minute threshold
 
         Raises:
             ValueError: If the audio file format is not recognized
@@ -869,15 +1133,19 @@ class AudioToTextConverter:
             }
 
             # Process completed transcriptions in order of completion
-            completion_tokens = 0
-            prompt_tokens = 0
             for future in as_completed(future_to_chunk):
                 index, transcript_dict = future.result()
                 chunks[index]["transcript"] = transcript_dict["transcript"]
                 transcript_chunks[index] = transcript_dict["transcript"]
                 chunk_results[index] = transcript_dict
-                completion_tokens += transcript_dict["completion_tokens"]
-                prompt_tokens += transcript_dict["prompt_tokens"]
+
+        self.recover_suspicious_fallback_languages(
+            chunks,
+            transcript_chunks,
+            chunk_results,
+        )
+        completion_tokens = sum(result["completion_tokens"] for result in chunk_results)
+        prompt_tokens = sum(result["prompt_tokens"] for result in chunk_results)
 
         text_merger = TextMerger(llm_api_key=self.llm_api_key)
         # Merge all transcripts
@@ -888,7 +1156,8 @@ class AudioToTextConverter:
             "completion_tokens": completion_tokens + full_text_merged_dict["completion_tokens"],
             "prompt_tokens": prompt_tokens + full_text_merged_dict["prompt_tokens"],
             "completion_model": self.transcription_model,
-            "completion_model_provider": self.transcription_model_provider
+            "completion_model_provider": self.transcription_model_provider,
+            "audio_chunk_quality": summarize_audio_chunk_quality(chunks, transcript_chunks),
         }
         if len(chunk_results) == 1 and chunk_results[0]:
             for key in (

{polytext-0.2.8b5 → polytext-0.2.8b7}/polytext/loader/base.py

@@ -42,6 +42,7 @@ logger = logging.getLogger(__name__)
 
 MIN_DOC_TEXT_LENGTH_ACCEPTED = int(os.getenv("MIN_DOC_TEXT_LENGTH_ACCEPTED", "400"))
 OCR_INCLUDE_IMAGE_DESCRIPTIONS_ENV = "OCR_INCLUDE_IMAGE_DESCRIPTIONS"
+FALLBACK_OCR_ENV = "FALLBACK_OCR"
 LLM_OUTPUT_ERROR_CODES = {
     995: "INVALID_ARGUMENT",
     996: "RECITATION",
@@ -119,7 +120,7 @@ class BaseLoader:
                 - target_size (int, optional): Target file size in bytes. Defaults to 1MB
                 - source (str): Source of the document. Must be either "cloud" or "local"
                 - fallback_ocr (bool, optional): If True, Optical Character Recognition (OCR) will be used as a fallback
-                  if direct text extraction fails, particularly for image-based content. Defaults to True.
+                  if direct text extraction fails. When omitted, defaults from FALLBACK_OCR and otherwise to False.
                 - save_transcript_chunks (bool, optional): Whether to save chunk transcripts in final output. Defaults to False.
                 - bitrate_quality (int, optional): Variable bitrate quality from 0-9 (9 being lowest). Defaults to 9
 
@@ -143,7 +144,10 @@ class BaseLoader:
         self.kwargs = kwargs
         self.target_size = kwargs.get("target_size", 1)
         self.source = kwargs.get("source", "cloud")
-        self.fallback_ocr = kwargs.get("fallback_ocr", False)
+        self.fallback_ocr = kwargs.get(
+            "fallback_ocr",
+            _read_bool_env(FALLBACK_OCR_ENV),
+        )
         self.save_transcript_chunks = kwargs.get("save_transcript_chunks", False)
         self.bitrate_quality = kwargs.get("bitrate_quality", 9)
         self.aws_auth_mode = kwargs.get("aws_auth_mode")
@@ -548,6 +552,7 @@ class BaseLoader:
                 - completion_model (str): Model name used, from the first processed source.
                 - completion_model_provider (str): Provider of the model, from the first processed source.
                 - text_chunks (list, optional): List of text chunks, if chunking was applied.
+                - audio_chunk_quality (dict, optional): Counts of empty and low-text audio chunks.
                 - type (str): Type of the processed source (e.g., ocr, video, audio, text).
                 - input (str): The input path or URL of the first processed source.
                 - output_list (list): List of individual extraction results (one per input).
@@ -608,7 +613,7 @@ class BaseLoader:
             "output_list": [result_dict],
         }
 
-        for metadata_key in ("ocr_failed_pages", "ocr_failed_pages_detail"):
+        for metadata_key in ("ocr_failed_pages", "ocr_failed_pages_detail", "audio_chunk_quality"):
             if metadata_key in result_dict:
                 final_result[metadata_key] = result_dict[metadata_key]
 

{polytext-0.2.8b5 → polytext-0.2.8b7}/polytext.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: polytext
-Version: 0.2.8b5
+Version: 0.2.8b7
 Summary: Python utilities to simplify document files management
 Home-page: https://github.com/docsity/polytext
 Author: Matteo Senardi
@@ -31,6 +31,7 @@ Requires-Dist: boto3>=1.42.64
 Requires-Dist: botocore>=1.42.64
 Requires-Dist: ffmpeg-python==0.2.0
 Requires-Dist: pydub==0.25.1
+Requires-Dist: fast-langdetect==0.2.5
 Requires-Dist: youtube-transcript-api==1.0.3
 Requires-Dist: yt-dlp==2025.5.22
 Requires-Dist: charset-normalizer<4.0.0,>=3.4.2

{polytext-0.2.8b5 → polytext-0.2.8b7}/polytext.egg-info/SOURCES.txt

@@ -56,6 +56,7 @@ polytext/prompts/text_to_md.py
 polytext/prompts/transcription.py
 polytext/utils/__init__.py
 polytext/utils/utils.py
+tests/test_audio_chunk_quality.py
 tests/test_audio_chunker.py
 tests/test_audio_comparison_helpers.py
 tests/test_audio_transcription_model_migration.py

{polytext-0.2.8b5 → polytext-0.2.8b7}/polytext.egg-info/requires.txt

@@ -13,6 +13,7 @@ boto3>=1.42.64
 botocore>=1.42.64
 ffmpeg-python==0.2.0
 pydub==0.25.1
+fast-langdetect==0.2.5
 youtube-transcript-api==1.0.3
 yt-dlp==2025.5.22
 charset-normalizer<4.0.0,>=3.4.2

{polytext-0.2.8b5 → polytext-0.2.8b7}/setup.py

@@ -51,7 +51,7 @@ def get_requirements(*requirements_file):
 
 setup(
     name='polytext',
-    version='0.2.8b5',
+    version='0.2.8b7',
     url='https://github.com/docsity/polytext',
     # download_url='https://github.com/pualien/py-polytext/archive/0.1.23.tar.gz',
     license='MIT',

polytext-0.2.8b7/tests/test_audio_chunk_quality.py

@@ -0,0 +1,76 @@
+import tempfile
+import unittest
+from unittest.mock import MagicMock, patch
+
+from polytext.converter import audio_to_text
+from polytext.loader import BaseLoader
+
+
+class AudioChunkQualityTests(unittest.TestCase):
+    def test_counts_empty_and_low_text_chunks_separately(self):
+        chunks = [
+            {"duration_ms": 120_000},
+            {"duration_ms": 120_000},
+            {"duration_ms": 120_000},
+            {"duration_ms": 20_000},
+            {"duration_ms": 1_200_000},
+        ]
+        transcripts = ["parola " * 40, "  \n", "parola " * 12, "ciao", "parola " * 100]
+
+        quality = audio_to_text.summarize_audio_chunk_quality(chunks, transcripts)
+
+        self.assertEqual(quality, {
+            "total_chunks": 5,
+            "empty_chunks": 1,
+            "low_text_chunks": 2,
+            "empty_chunk_indices": [2],
+            "low_text_chunk_indices": [3, 5],
+            "low_text_threshold_words_per_minute": 20,
+            "low_text_min_duration_ms": 60_000,
+        })
+
+    @patch("polytext.converter.audio_to_text.TextMerger")
+    @patch("polytext.converter.audio_to_text.AudioChunker")
+    @patch.object(audio_to_text.AudioToTextConverter, "process_chunk")
+    def test_quality_is_returned_without_saving_chunk_transcripts(
+        self, mock_process_chunk, mock_chunker_cls, mock_merger_cls
+    ):
+        chunker = mock_chunker_cls.return_value
+        chunker.duration_ms = 240_000
+        chunker.extract_chunks.return_value = [
+            {"file_path": "/tmp/chunk1.mp3", "duration_ms": 120_000},
+            {"file_path": "/tmp/chunk2.mp3", "duration_ms": 120_000},
+        ]
+        mock_process_chunk.side_effect = lambda chunk, index: (
+            index,
+            {"transcript": "parola " * 40 if index == 0 else "", "completion_tokens": 4, "prompt_tokens": 5},
+        )
+        mock_merger_cls.return_value.merge_chunks_with_llm_sequential.return_value = {
+            "full_text_merged": "testo finale", "completion_tokens": 0, "prompt_tokens": 0
+        }
+
+        with tempfile.NamedTemporaryFile(suffix=".mp3") as source_audio:
+            converter = audio_to_text.AudioToTextConverter()
+            result = converter.transcribe_full_audio(source_audio.name)
+
+        self.assertEqual(result["text"], "testo finale")
+        self.assertEqual(result["audio_chunk_quality"]["total_chunks"], 2)
+        self.assertEqual(result["audio_chunk_quality"]["empty_chunks"], 1)
+        self.assertNotIn("text_chunks", result)
+
+    def test_quality_reaches_top_level_loader_response(self):
+        quality = {"total_chunks": 2, "empty_chunks": 1, "low_text_chunks": 0}
+        file_loader = MagicMock()
+        file_loader.load.return_value = {
+            "text": "trascrizione", "completion_tokens": 3, "prompt_tokens": 4,
+            "audio_chunk_quality": quality,
+        }
+
+        response = BaseLoader().run_loader_class(file_loader, ["audio.mp3"])
+
+        self.assertEqual(response["audio_chunk_quality"], quality)
+        self.assertEqual(response["output_list"][0]["audio_chunk_quality"], quality)
+
+
+if __name__ == "__main__":
+    unittest.main()

{polytext-0.2.8b5 → polytext-0.2.8b7}/tests/test_audio_transcription_model_migration.py

@@ -1,6 +1,7 @@
 import unittest
 import tempfile
 import os
+import wave
 from types import SimpleNamespace
 from unittest.mock import MagicMock, patch
 from pydub import AudioSegment
@@ -13,6 +14,8 @@ from polytext.converter.audio_to_text import (
     AUDIO_TO_MARKDOWN_PROMPT_IS_RAW,
     AUDIO_TO_PLAIN_TEXT_PROMPT,
     AudioToTextConverter,
+    detect_dominant_language,
+    find_suspicious_fallback_language_indices,
     normalize_no_human_speech_marker,
     transcribe_full_audio,
 )
@@ -84,6 +87,148 @@ class _FakeClient:
         self.models = _FakeModels(responses=responses)
 
 
+class TestFallbackLanguageValidation(unittest.TestCase):
+    @patch("polytext.converter.audio_to_text.detect")
+    def test_detect_dominant_language_aggregates_multiple_text_windows(self, mock_detect):
+        mock_detect.side_effect = [
+            {"lang": "it", "score": 0.96},
+            {"lang": "it", "score": 0.93},
+            {"lang": "en", "score": 0.60},
+        ]
+
+        text = " ".join(f"parola{i}" for i in range(120))
+
+        result = detect_dominant_language(text, max_windows=3, window_chars=120)
+
+        self.assertEqual(result["lang"], "it")
+        self.assertGreater(result["support"], 0.60)
+        self.assertEqual(result["windows"], 3)
+
+    @patch("polytext.converter.audio_to_text.detect_dominant_language")
+    def test_flags_internal_fallback_when_both_neighbours_agree(self, mock_detect):
+        mock_detect.side_effect = [
+            {"lang": "en", "score": 0.98, "support": 1.0, "windows": 3},
+            {"lang": "it", "score": 0.97, "support": 1.0, "windows": 3},
+            {"lang": "it", "score": 0.96, "support": 1.0, "windows": 3},
+        ]
+        transcripts = [
+            "testo italiano " * 60,
+            "unexpected English transcript " * 60,
+            "altro testo italiano " * 60,
+        ]
+        results = [
+            {
+                "transcript": transcripts[0],
+                "fallback_from_model": "gemini-3.1-flash-lite",
+            },
+            {
+                "transcript": transcripts[1],
+                "adaptive_split": True,
+                "split_results": [
+                    {
+                        "transcript": transcripts[1],
+                        "fallback_from_model": "gemini-3.1-flash-lite",
+                    }
+                ],
+            },
+            {
+                "transcript": transcripts[2],
+                "fallback_to_model": "gemini-3.5-flash-lite",
+            },
+        ]
+
+        suspicious = find_suspicious_fallback_language_indices(transcripts, results)
+
+        self.assertEqual(suspicious, [1])
+
+    @patch("polytext.converter.audio_to_text.detect_dominant_language")
+    def test_preserves_possible_multilingual_transition_when_neighbours_disagree(self, mock_detect):
+        mock_detect.side_effect = [
+            {"lang": "en", "score": 0.98, "support": 1.0, "windows": 3},
+            {"lang": "it", "score": 0.97, "support": 1.0, "windows": 3},
+            {"lang": "fr", "score": 0.96, "support": 1.0, "windows": 3},
+        ]
+        transcripts = ["italiano " * 60, "english " * 60, "francais " * 60]
+        results = [
+            {"transcript": transcripts[0]},
+            {"transcript": transcripts[1], "fallback_to_model": "gemini-3.5-flash-lite"},
+            {"transcript": transcripts[2]},
+        ]
+
+        suspicious = find_suspicious_fallback_language_indices(transcripts, results)
+
+        self.assertEqual(suspicious, [])
+
+    @patch("polytext.converter.audio_to_text.detect_dominant_language")
+    def test_flags_fallback_with_two_confident_foreign_windows(self, mock_detect):
+        mock_detect.side_effect = [
+            {
+                "lang": "it",
+                "score": 0.99,
+                "support": 0.6,
+                "windows": 5,
+                "language_windows": {
+                    "it": {"windows": 3, "score": 0.99},
+                    "en": {"windows": 2, "score": 0.95},
+                },
+            },
+            {"lang": "it", "score": 0.98, "support": 1.0, "windows": 5},
+            {"lang": "it", "score": 0.97, "support": 1.0, "windows": 5},
+        ]
+        transcripts = ["italiano " * 60, "testo misto " * 60, "italiano " * 60]
+        results = [
+            {"transcript": transcripts[0]},
+            {"transcript": transcripts[1], "fallback_to_model": "gemini-3.5-flash-lite"},
+            {"transcript": transcripts[2]},
+        ]
+
+        suspicious = find_suspicious_fallback_language_indices(transcripts, results)
+
+        self.assertEqual(suspicious, [1])
+
+    @patch(
+        "polytext.converter.audio_to_text.find_suspicious_fallback_language_indices",
+        return_value=[1],
+    )
+    def test_suspicious_fallback_is_retranscribed_with_adaptive_split(self, _mock_find):
+        converter = AudioToTextConverter()
+        chunks = [
+            {"file_path": "/tmp/chunk-0.wav"},
+            {"file_path": "/tmp/chunk-1.wav"},
+            {"file_path": "/tmp/chunk-2.wav"},
+        ]
+        transcripts = ["prima", "wrong English fallback", "dopo"]
+        results = [
+            {"transcript": "prima", "completion_tokens": 1, "prompt_tokens": 1},
+            {
+                "transcript": "wrong English fallback",
+                "completion_tokens": 2,
+                "prompt_tokens": 2,
+                "completion_model": "gemini-3.5-flash-lite",
+                "fallback_from_model": "gemini-3.1-flash-lite",
+            },
+            {"transcript": "dopo", "completion_tokens": 1, "prompt_tokens": 1},
+        ]
+        recovered = {
+            "transcript": "testo italiano recuperato",
+            "completion_tokens": 4,
+            "prompt_tokens": 3,
+            "completion_model": "gemini-3.1-flash-lite",
+        }
+
+        with patch.object(converter, "transcribe_audio_halves", return_value=recovered) as mock_split:
+            converter.recover_suspicious_fallback_languages(chunks, transcripts, results)
+
+        mock_split.assert_called_once_with("/tmp/chunk-1.wav")
+        self.assertEqual(transcripts[1], "testo italiano recuperato")
+        self.assertEqual(results[1]["transcript"], "testo italiano recuperato")
+        self.assertTrue(results[1]["language_validation_triggered"])
+        self.assertEqual(
+            results[1]["language_validation_original_model"],
+            "gemini-3.5-flash-lite",
+        )
+
+
 class _FlakyServerErrorModels:
     def __init__(self):
         self.generate_content_calls = 0
@@ -455,6 +600,20 @@ class TestAudioTranscriptionModelMigration(unittest.TestCase):
         audio_part = fake_client.models.generate_content_contents[2]
         self.assertEqual(audio_part.inline_data.mime_type, "audio/aac")
 
+    @patch("polytext.converter.audio_to_text.genai.Client")
+    def test_inline_wav_uses_gemini_supported_mime_type(self, mock_client_cls):
+        fake_client = _FakeClient()
+        mock_client_cls.return_value = fake_client
+
+        converter = AudioToTextConverter()
+        with tempfile.NamedTemporaryFile(suffix=".wav") as temp_audio:
+            temp_audio.write(b"fake-audio")
+            temp_audio.flush()
+            converter.transcribe_audio(temp_audio.name)
+
+        audio_part = fake_client.models.generate_content_contents[2]
+        self.assertEqual(audio_part.inline_data.mime_type, "audio/wav")
+
     @patch("retry.api.time.sleep")
     def test_does_not_retry_genai_client_error_for_audio_transcription(self, _mock_sleep):
         fake_client = _ClientErrorClient()
@@ -613,10 +772,18 @@ class TestAudioTranscriptionModelMigration(unittest.TestCase):
         with tempfile.NamedTemporaryFile(suffix=".mp3") as source_audio:
             AudioSegment.silent(duration=10_000).export(source_audio.name, format="mp3").close()
             converter = AudioToTextConverter()
+            def transcribe_split(split_path, temperature=0.0):
+                self.assertTrue(split_path.endswith(".wav"))
+                with wave.open(split_path, "rb") as split_audio:
+                    self.assertEqual(split_audio.getframerate(), 16000)
+                    self.assertEqual(split_audio.getnchannels(), 1)
+                    self.assertEqual(split_audio.getsampwidth(), 2)
+                return split_responses.pop(0)
+
             with patch.object(
                 AudioToTextConverter,
                 "transcribe_audio",
-                side_effect=split_responses,
+                side_effect=transcribe_split,
             ) as mock_transcribe:
                 result = converter.transcribe_audio_halves(source_audio.name)
 

{polytext-0.2.8b5 → polytext-0.2.8b7}/tests/test_base_loader_error_mapping.py

@@ -1,3 +1,4 @@
+import os
 import unittest
 from types import SimpleNamespace
 from unittest.mock import Mock, patch
@@ -60,6 +61,24 @@ class _BeautifulTextLoader(BaseLoader):
 
 
 class TestBaseLoaderErrorMapping(unittest.TestCase):
+    def test_document_ocr_fallback_is_enabled_by_environment(self):
+        with patch.dict(os.environ, {"FALLBACK_OCR": "true"}, clear=True):
+            loader = BaseLoader()
+
+        self.assertTrue(loader.fallback_ocr)
+
+    def test_explicit_document_ocr_fallback_setting_overrides_environment(self):
+        with patch.dict(os.environ, {"FALLBACK_OCR": "true"}, clear=True):
+            loader = BaseLoader(fallback_ocr=False)
+
+        self.assertFalse(loader.fallback_ocr)
+
+    def test_document_ocr_fallback_remains_disabled_without_configuration(self):
+        with patch.dict(os.environ, {}, clear=True):
+            loader = BaseLoader()
+
+        self.assertFalse(loader.fallback_ocr)
+
     def test_llm_output_empty_document_codes_are_raised_as_loader_errors(self):
         cases = [
             (995, "INVALID_ARGUMENT"),