polytext 0.2.8b4__tar.gz → 0.2.8b6__tar.gz

{polytext-0.2.8b4 → polytext-0.2.8b6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: polytext
-Version: 0.2.8b4
+Version: 0.2.8b6
 Summary: Python utilities to simplify document files management
 Home-page: https://github.com/docsity/polytext
 Author: Matteo Senardi
@@ -31,6 +31,7 @@ Requires-Dist: boto3>=1.42.64
 Requires-Dist: botocore>=1.42.64
 Requires-Dist: ffmpeg-python==0.2.0
 Requires-Dist: pydub==0.25.1
+Requires-Dist: fast-langdetect==0.2.5
 Requires-Dist: youtube-transcript-api==1.0.3
 Requires-Dist: yt-dlp==2025.5.22
 Requires-Dist: charset-normalizer<4.0.0,>=3.4.2

{polytext-0.2.8b4 → polytext-0.2.8b6}/polytext/converter/audio_to_text.py

@@ -15,6 +15,7 @@ from google.genai import errors as genai_errors
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from google.api_core import exceptions as google_exceptions
 from pydub import AudioSegment
+from fast_langdetect import detect
 
 from ..exceptions import EmptyDocument
 from ..prompts.transcription import (
@@ -73,6 +74,199 @@ AUDIO_PROMPT_VARIANT_DEFAULT = "default"
 AUDIO_PROMPT_VARIANT_NON_LITERAL_FALLBACK = "non_literal_fallback"
 AUDIO_RETRIABLE_OUTPUT_ERROR_CODES = (996, 997, 998, 999)
 NO_HUMAN_SPEECH_MARKER = "no human speech detected"
+AUDIO_LANGUAGE_VALIDATION_MIN_WORDS = int(os.getenv("AUDIO_LANGUAGE_VALIDATION_MIN_WORDS", "50"))
+AUDIO_LANGUAGE_VALIDATION_MIN_CONFIDENCE = float(
+    os.getenv("AUDIO_LANGUAGE_VALIDATION_MIN_CONFIDENCE", "0.80")
+)
+AUDIO_LANGUAGE_VALIDATION_MIN_SUPPORT = float(
+    os.getenv("AUDIO_LANGUAGE_VALIDATION_MIN_SUPPORT", "0.67")
+)
+AUDIO_LANGUAGE_VALIDATION_MIN_FOREIGN_WINDOWS = int(
+    os.getenv("AUDIO_LANGUAGE_VALIDATION_MIN_FOREIGN_WINDOWS", "2")
+)
+
+
+def _normalize_language_detection(result) -> tuple[str, float]:
+    if isinstance(result, list):
+        result = result[0] if result else {}
+    if not isinstance(result, dict):
+        return "", 0.0
+    language = str(result.get("lang", "")).lower().split("-")[0]
+    return language, float(result.get("score", 0.0) or 0.0)
+
+
+def detect_dominant_language(
+        text: str,
+        max_windows: int = 5,
+        window_chars: int = 160,
+) -> dict:
+    """Detect the dominant language over multiple short, evenly spaced text windows."""
+    normalized_text = re.sub(r"\s+", " ", text or "").strip()
+    if not normalized_text:
+        return {
+            "lang": "",
+            "score": 0.0,
+            "support": 0.0,
+            "windows": 0,
+            "language_windows": {},
+        }
+
+    if len(normalized_text) <= window_chars:
+        samples = [normalized_text]
+    else:
+        last_start = len(normalized_text) - window_chars
+        sample_count = min(max_windows, max(2, len(normalized_text) // window_chars))
+        starts = {
+            round(index * last_start / (sample_count - 1))
+            for index in range(sample_count)
+        }
+        samples = [normalized_text[start:start + window_chars] for start in sorted(starts)]
+
+    language_scores = {}
+    language_counts = {}
+    valid_windows = 0
+    for sample in samples:
+        try:
+            detected = detect(sample, low_memory=True)
+        except TypeError:
+            # fast-langdetect >= 1.0 uses model/k instead of low_memory.
+            detected = detect(sample, model="lite", k=1)
+        except Exception:
+            logger.exception("Language detection failed for an audio transcript window")
+            continue
+
+        language, score = _normalize_language_detection(detected)
+        if not language:
+            continue
+        valid_windows += 1
+        language_scores[language] = language_scores.get(language, 0.0) + score
+        language_counts[language] = language_counts.get(language, 0) + 1
+
+    if not valid_windows:
+        return {
+            "lang": "",
+            "score": 0.0,
+            "support": 0.0,
+            "windows": 0,
+            "language_windows": {},
+        }
+
+    dominant_language = max(language_scores, key=language_scores.get)
+    count = language_counts[dominant_language]
+    return {
+        "lang": dominant_language,
+        "score": language_scores[dominant_language] / count,
+        "support": count / valid_windows,
+        "windows": valid_windows,
+        "language_windows": {
+            language: {
+                "windows": language_counts[language],
+                "score": language_scores[language] / language_counts[language],
+            }
+            for language in language_counts
+        },
+    }
+
+
+def find_suspicious_fallback_language_indices(
+        transcripts: list[str],
+        chunk_results: list[dict | None],
+) -> list[int]:
+    """Find internal fallback chunks whose language conflicts with two agreeing neighbours."""
+    suspicious_indices = []
+    logger.info("Starting fallback language validation for %s transcript chunks", len(transcripts))
+
+    def has_fallback(result: dict | None) -> bool:
+        if not result:
+            return False
+        if result.get("fallback_from_model") or result.get("fallback_to_model"):
+            return True
+        return any(has_fallback(item) for item in result.get("split_results", []))
+
+    for index in range(1, len(transcripts) - 1):
+        result = chunk_results[index] or {}
+        if not has_fallback(result):
+            continue
+
+        text = transcripts[index]
+        word_count = len(re.findall(r"\b\w+\b", text, flags=re.UNICODE))
+        if word_count < AUDIO_LANGUAGE_VALIDATION_MIN_WORDS:
+            logger.info(
+                "Skipping language validation for fallback chunk %s: only %s words (minimum %s)",
+                index + 1,
+                word_count,
+                AUDIO_LANGUAGE_VALIDATION_MIN_WORDS,
+            )
+            continue
+
+        candidate_language = detect_dominant_language(text)
+        previous_language = detect_dominant_language(transcripts[index - 1])
+        next_language = detect_dominant_language(transcripts[index + 1])
+        logger.info(
+            "Language validation for fallback chunk %s: previous=%s, candidate=%s, next=%s",
+            index + 1,
+            previous_language,
+            candidate_language,
+            next_language,
+        )
+        neighbour_profiles = (previous_language, next_language)
+        if any(
+            profile["score"] < AUDIO_LANGUAGE_VALIDATION_MIN_CONFIDENCE
+            or profile["support"] < AUDIO_LANGUAGE_VALIDATION_MIN_SUPPORT
+            for profile in neighbour_profiles
+        ):
+            logger.info(
+                "Skipping language comparison for fallback chunk %s: neighbour confidence or support below threshold",
+                index + 1,
+            )
+            continue
+
+        if previous_language["lang"] != next_language["lang"]:
+            logger.info(
+                "Skipping language comparison for fallback chunk %s: neighbouring languages disagree",
+                index + 1,
+            )
+            continue
+
+        expected_language = previous_language["lang"]
+        foreign_windows = {
+            language: profile
+            for language, profile in candidate_language.get("language_windows", {}).items()
+            if (
+                language != expected_language
+                and profile["windows"] >= AUDIO_LANGUAGE_VALIDATION_MIN_FOREIGN_WINDOWS
+                and profile["score"] >= AUDIO_LANGUAGE_VALIDATION_MIN_CONFIDENCE
+            )
+        }
+        if foreign_windows:
+            logger.warning(
+                "Mixed-language anomaly detected for fallback chunk %s: expected %s, foreign windows=%s",
+                index + 1,
+                expected_language,
+                foreign_windows,
+            )
+            suspicious_indices.append(index)
+            continue
+
+        if (
+            candidate_language["score"] >= AUDIO_LANGUAGE_VALIDATION_MIN_CONFIDENCE
+            and candidate_language["support"] >= AUDIO_LANGUAGE_VALIDATION_MIN_SUPPORT
+            and candidate_language["lang"] != expected_language
+        ):
+            logger.warning(
+                "Language anomaly detected for fallback chunk %s: %s between two %s chunks",
+                index + 1,
+                candidate_language["lang"],
+                expected_language,
+            )
+            suspicious_indices.append(index)
+        elif candidate_language["support"] < AUDIO_LANGUAGE_VALIDATION_MIN_SUPPORT:
+            logger.info(
+                "No actionable language anomaly for fallback chunk %s: mixed windows remain below thresholds",
+                index + 1,
+            )
+
+    return suspicious_indices
 
 
 def normalize_no_human_speech_marker(text: str) -> tuple[str, bool]:
@@ -782,6 +976,41 @@ class AudioToTextConverter:
         transcript_dict = self.transcribe_audio(chunk["file_path"])
         return index, transcript_dict
 
+    def recover_suspicious_fallback_languages(
+            self,
+            chunks: list[dict],
+            transcript_chunks: list[str],
+            chunk_results: list[dict | None],
+    ) -> None:
+        """Re-transcribe suspicious fallback chunks in smaller parts, in place."""
+        suspicious_indices = find_suspicious_fallback_language_indices(
+            transcript_chunks,
+            chunk_results,
+        )
+        for index in suspicious_indices:
+            original_result = chunk_results[index] or {}
+            logger.warning(
+                "Fallback transcript language conflicts with both neighbouring chunks; "
+                "re-transcribing chunk %s with adaptive split",
+                index + 1,
+            )
+            try:
+                recovered_result = self.transcribe_audio_halves(chunks[index]["file_path"])
+            except Exception:
+                logger.exception(
+                    "Language-based recovery failed for chunk %s; preserving the original transcript",
+                    index + 1,
+                )
+                continue
+
+            recovered_result["language_validation_triggered"] = True
+            recovered_result["language_validation_original_model"] = original_result.get(
+                "completion_model"
+            )
+            transcript_chunks[index] = recovered_result["transcript"]
+            chunks[index]["transcript"] = recovered_result["transcript"]
+            chunk_results[index] = recovered_result
+
     def format_audio_output_text(self, text: str) -> str:
         return add_line_break_after_each_sentence(text)
 
@@ -869,15 +1098,19 @@ class AudioToTextConverter:
             }
 
             # Process completed transcriptions in order of completion
-            completion_tokens = 0
-            prompt_tokens = 0
             for future in as_completed(future_to_chunk):
                 index, transcript_dict = future.result()
                 chunks[index]["transcript"] = transcript_dict["transcript"]
                 transcript_chunks[index] = transcript_dict["transcript"]
                 chunk_results[index] = transcript_dict
-                completion_tokens += transcript_dict["completion_tokens"]
-                prompt_tokens += transcript_dict["prompt_tokens"]
+
+        self.recover_suspicious_fallback_languages(
+            chunks,
+            transcript_chunks,
+            chunk_results,
+        )
+        completion_tokens = sum(result["completion_tokens"] for result in chunk_results)
+        prompt_tokens = sum(result["prompt_tokens"] for result in chunk_results)
 
         text_merger = TextMerger(llm_api_key=self.llm_api_key)
         # Merge all transcripts

{polytext-0.2.8b4 → polytext-0.2.8b6}/polytext/converter/document_ocr_to_text.py

@@ -115,6 +115,7 @@ def get_document_ocr(
     max_output_tokens: int | None = None,
     include_image_descriptions: bool = False,
     allow_partial_ocr_failures: bool = False,
+    ocr_render_dpi: int | None = None,
 ):
     """
     Convenience function to extract text from an image file using OCR, optionally formatted as Markdown.
@@ -141,6 +142,8 @@ def get_document_ocr(
         allow_partial_ocr_failures (bool, optional): If True, pages that still
             fail OCR after all retries are recorded inline instead of aborting
             the whole document extraction. Defaults to False.
+        ocr_render_dpi (int | None, optional): Render PDF pages at this DPI before
+            OCR. When omitted, preserves PyMuPDF's default rendering behavior.
 
     Returns:
         dict: Dictionary containing the OCR results and metadata.
@@ -155,6 +158,7 @@ def get_document_ocr(
         max_output_tokens=max_output_tokens,
         include_image_descriptions=include_image_descriptions,
         allow_partial_ocr_failures=allow_partial_ocr_failures,
+        ocr_render_dpi=ocr_render_dpi,
     )
     return converter.get_document_ocr(document_for_ocr)
 
@@ -164,7 +168,8 @@ class DocumentOCRToTextConverter:
                  page_range=None, timeout_minutes: int = None, fallback_stage: int = 0,
                  max_output_tokens: int | None = None, include_image_descriptions: bool = False,
                  prompt_variant: str = OCR_PROMPT_VARIANT_DEFAULT,
-                 allow_partial_ocr_failures: bool = False):
+                 allow_partial_ocr_failures: bool = False,
+                 ocr_render_dpi: int | None = None):
         """
         Initialize the DocumentOCRToTextConverter class with specified OCR model and formatting options.
 
@@ -192,6 +197,8 @@ class DocumentOCRToTextConverter:
             allow_partial_ocr_failures (bool, optional): If True, pages that still
                 fail OCR after all retries are recorded inline instead of aborting
                 the whole document extraction. Defaults to False.
+            ocr_render_dpi (int | None, optional): Render PDF pages at this DPI
+                before OCR. Defaults to PyMuPDF's native rendering behavior.
 
         Raises:
             OSError: If temp directory creation fails
@@ -207,6 +214,7 @@ class DocumentOCRToTextConverter:
         self.include_image_descriptions = include_image_descriptions
         self.prompt_variant = prompt_variant
         self.allow_partial_ocr_failures = allow_partial_ocr_failures
+        self.ocr_render_dpi = ocr_render_dpi
         requested_output_tokens = OCR_MAX_OUTPUT_TOKENS if max_output_tokens is None else max_output_tokens
         self.max_output_tokens = max(requested_output_tokens, OCR_MIN_OUTPUT_TOKENS)
         self.fallback_stage = fallback_stage
@@ -297,6 +305,7 @@ class DocumentOCRToTextConverter:
             include_image_descriptions=self.include_image_descriptions,
             prompt_variant=resolved_prompt_variant,
             allow_partial_ocr_failures=self.allow_partial_ocr_failures,
+            ocr_render_dpi=self.ocr_render_dpi,
         )
         result = fallback_converter.get_ocr(
             file_for_ocr=file_for_ocr,
@@ -553,7 +562,10 @@ class DocumentOCRToTextConverter:
 
             try:
                 # Convert page to image
-                pix = page.get_pixmap()
+                if self.ocr_render_dpi is None:
+                    pix = page.get_pixmap()
+                else:
+                    pix = page.get_pixmap(dpi=self.ocr_render_dpi, alpha=False)
                 pix.save(temp_image_path)
 
                 # Perform OCR on the page

{polytext-0.2.8b4 → polytext-0.2.8b6}/polytext/loader/base.py

@@ -92,6 +92,7 @@ class BaseLoader:
                  ocr_model: str = "gpt-5-mini", timeout_minutes: int | None = None,
                  include_image_descriptions: bool | None = None,
                  force_ocr: bool = False,
+                 ocr_render_dpi: int | None = None,
                  **kwargs):
         """
         Initialize the BaseLoader with cloud storage and LLM configurations.
@@ -112,6 +113,8 @@ class BaseLoader:
                 If None, defaults from OCR_INCLUDE_IMAGE_DESCRIPTIONS. Defaults to None.
             force_ocr (bool, optional): If True, supported document files are routed to
                 OCRLoader instead of the standard DocumentLoader. Defaults to False.
+            ocr_render_dpi (int | None, optional): Render document pages at this DPI
+                before OCR. Defaults to the existing renderer behavior.
              **kwargs: Additional keyword arguments to pass to the underlying loader or extraction logic.
                 - target_size (int, optional): Target file size in bytes. Defaults to 1MB
                 - source (str): Source of the document. Must be either "cloud" or "local"
@@ -136,6 +139,7 @@ class BaseLoader:
             else include_image_descriptions
         )
         self.force_ocr = force_ocr
+        self.ocr_render_dpi = ocr_render_dpi
         self.kwargs = kwargs
         self.target_size = kwargs.get("target_size", 1)
         self.source = kwargs.get("source", "cloud")
@@ -422,7 +426,7 @@ class BaseLoader:
             file_extension = file_extension.lower()
 
         if is_document_fallback:
-            return DocumentOCRLoader(llm_api_key=llm_api_key, markdown_output=self.markdown_output, temp_dir=self.temp_dir, timeout_minutes=self.timeout_minutes, ocr_provider=self.provider, ocr_model=self.ocr_model, include_image_descriptions=self.include_image_descriptions, **kwargs)
+            return DocumentOCRLoader(llm_api_key=llm_api_key, markdown_output=self.markdown_output, temp_dir=self.temp_dir, timeout_minutes=self.timeout_minutes, ocr_provider=self.provider, ocr_model=self.ocr_model, include_image_descriptions=self.include_image_descriptions, ocr_render_dpi=self.ocr_render_dpi, **kwargs)
 
         if file_extension in [".xml", ".xbrl"]:
             return XmlXbrlLoader(temp_dir=self.temp_dir, markdown_output=self.markdown_output, **kwargs)
@@ -458,6 +462,7 @@ class BaseLoader:
                         ocr_provider=self.provider,
                         ocr_model=self.ocr_model,
                         include_image_descriptions=self.include_image_descriptions,
+                        ocr_render_dpi=self.ocr_render_dpi,
                         **document_kwargs,
                     )
 

{polytext-0.2.8b4 → polytext-0.2.8b6}/polytext/loader/document_ocr.py

@@ -47,6 +47,7 @@ class DocumentOCRLoader:
                  ocr_model: str | None = None,
                  include_image_descriptions: bool = False,
                  allow_partial_ocr_failures: bool = False,
+                 ocr_render_dpi: int | None = None,
                  **kwargs
                  ):
         """
@@ -86,6 +87,8 @@ class DocumentOCRLoader:
             allow_partial_ocr_failures (bool, optional): If True, pages that still
                 fail OCR after all retries are recorded inline instead of aborting
                 the whole document extraction. Defaults to False.
+            ocr_render_dpi (int | None, optional): Render PDF pages at this DPI
+                before OCR. Defaults to the existing PyMuPDF rendering behavior.
             **kwargs:
                 max_output_tokens (int, optional): Maximum Gemini output tokens for
                     Google document OCR generation.
@@ -110,6 +113,7 @@ class DocumentOCRLoader:
         self.ocr_model = ocr_model
         self.include_image_descriptions = include_image_descriptions
         self.allow_partial_ocr_failures = allow_partial_ocr_failures
+        self.ocr_render_dpi = ocr_render_dpi
         self.max_output_tokens = kwargs.get("max_output_tokens")
 
         # Set up custom temp directory
@@ -277,6 +281,7 @@ class DocumentOCRLoader:
                 max_output_tokens=self.max_output_tokens,
                 include_image_descriptions=self.include_image_descriptions,
                 allow_partial_ocr_failures=self.allow_partial_ocr_failures,
+                ocr_render_dpi=self.ocr_render_dpi,
             )
 
         result_dict["type"] = self.type

{polytext-0.2.8b4 → polytext-0.2.8b6}/polytext.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: polytext
-Version: 0.2.8b4
+Version: 0.2.8b6
 Summary: Python utilities to simplify document files management
 Home-page: https://github.com/docsity/polytext
 Author: Matteo Senardi
@@ -31,6 +31,7 @@ Requires-Dist: boto3>=1.42.64
 Requires-Dist: botocore>=1.42.64
 Requires-Dist: ffmpeg-python==0.2.0
 Requires-Dist: pydub==0.25.1
+Requires-Dist: fast-langdetect==0.2.5
 Requires-Dist: youtube-transcript-api==1.0.3
 Requires-Dist: yt-dlp==2025.5.22
 Requires-Dist: charset-normalizer<4.0.0,>=3.4.2

{polytext-0.2.8b4 → polytext-0.2.8b6}/polytext.egg-info/requires.txt

@@ -13,6 +13,7 @@ boto3>=1.42.64
 botocore>=1.42.64
 ffmpeg-python==0.2.0
 pydub==0.25.1
+fast-langdetect==0.2.5
 youtube-transcript-api==1.0.3
 yt-dlp==2025.5.22
 charset-normalizer<4.0.0,>=3.4.2

{polytext-0.2.8b4 → polytext-0.2.8b6}/setup.py

@@ -51,7 +51,7 @@ def get_requirements(*requirements_file):
 
 setup(
     name='polytext',
-    version='0.2.8b4',
+    version='0.2.8b6',
     url='https://github.com/docsity/polytext',
     # download_url='https://github.com/pualien/py-polytext/archive/0.1.23.tar.gz',
     license='MIT',

{polytext-0.2.8b4 → polytext-0.2.8b6}/tests/test_audio_transcription_model_migration.py

@@ -1,6 +1,7 @@
 import unittest
 import tempfile
 import os
+import wave
 from types import SimpleNamespace
 from unittest.mock import MagicMock, patch
 from pydub import AudioSegment
@@ -13,6 +14,8 @@ from polytext.converter.audio_to_text import (
     AUDIO_TO_MARKDOWN_PROMPT_IS_RAW,
     AUDIO_TO_PLAIN_TEXT_PROMPT,
     AudioToTextConverter,
+    detect_dominant_language,
+    find_suspicious_fallback_language_indices,
     normalize_no_human_speech_marker,
     transcribe_full_audio,
 )
@@ -84,6 +87,148 @@ class _FakeClient:
         self.models = _FakeModels(responses=responses)
 
 
+class TestFallbackLanguageValidation(unittest.TestCase):
+    @patch("polytext.converter.audio_to_text.detect")
+    def test_detect_dominant_language_aggregates_multiple_text_windows(self, mock_detect):
+        mock_detect.side_effect = [
+            {"lang": "it", "score": 0.96},
+            {"lang": "it", "score": 0.93},
+            {"lang": "en", "score": 0.60},
+        ]
+
+        text = " ".join(f"parola{i}" for i in range(120))
+
+        result = detect_dominant_language(text, max_windows=3, window_chars=120)
+
+        self.assertEqual(result["lang"], "it")
+        self.assertGreater(result["support"], 0.60)
+        self.assertEqual(result["windows"], 3)
+
+    @patch("polytext.converter.audio_to_text.detect_dominant_language")
+    def test_flags_internal_fallback_when_both_neighbours_agree(self, mock_detect):
+        mock_detect.side_effect = [
+            {"lang": "en", "score": 0.98, "support": 1.0, "windows": 3},
+            {"lang": "it", "score": 0.97, "support": 1.0, "windows": 3},
+            {"lang": "it", "score": 0.96, "support": 1.0, "windows": 3},
+        ]
+        transcripts = [
+            "testo italiano " * 60,
+            "unexpected English transcript " * 60,
+            "altro testo italiano " * 60,
+        ]
+        results = [
+            {
+                "transcript": transcripts[0],
+                "fallback_from_model": "gemini-3.1-flash-lite",
+            },
+            {
+                "transcript": transcripts[1],
+                "adaptive_split": True,
+                "split_results": [
+                    {
+                        "transcript": transcripts[1],
+                        "fallback_from_model": "gemini-3.1-flash-lite",
+                    }
+                ],
+            },
+            {
+                "transcript": transcripts[2],
+                "fallback_to_model": "gemini-3.5-flash-lite",
+            },
+        ]
+
+        suspicious = find_suspicious_fallback_language_indices(transcripts, results)
+
+        self.assertEqual(suspicious, [1])
+
+    @patch("polytext.converter.audio_to_text.detect_dominant_language")
+    def test_preserves_possible_multilingual_transition_when_neighbours_disagree(self, mock_detect):
+        mock_detect.side_effect = [
+            {"lang": "en", "score": 0.98, "support": 1.0, "windows": 3},
+            {"lang": "it", "score": 0.97, "support": 1.0, "windows": 3},
+            {"lang": "fr", "score": 0.96, "support": 1.0, "windows": 3},
+        ]
+        transcripts = ["italiano " * 60, "english " * 60, "francais " * 60]
+        results = [
+            {"transcript": transcripts[0]},
+            {"transcript": transcripts[1], "fallback_to_model": "gemini-3.5-flash-lite"},
+            {"transcript": transcripts[2]},
+        ]
+
+        suspicious = find_suspicious_fallback_language_indices(transcripts, results)
+
+        self.assertEqual(suspicious, [])
+
+    @patch("polytext.converter.audio_to_text.detect_dominant_language")
+    def test_flags_fallback_with_two_confident_foreign_windows(self, mock_detect):
+        mock_detect.side_effect = [
+            {
+                "lang": "it",
+                "score": 0.99,
+                "support": 0.6,
+                "windows": 5,
+                "language_windows": {
+                    "it": {"windows": 3, "score": 0.99},
+                    "en": {"windows": 2, "score": 0.95},
+                },
+            },
+            {"lang": "it", "score": 0.98, "support": 1.0, "windows": 5},
+            {"lang": "it", "score": 0.97, "support": 1.0, "windows": 5},
+        ]
+        transcripts = ["italiano " * 60, "testo misto " * 60, "italiano " * 60]
+        results = [
+            {"transcript": transcripts[0]},
+            {"transcript": transcripts[1], "fallback_to_model": "gemini-3.5-flash-lite"},
+            {"transcript": transcripts[2]},
+        ]
+
+        suspicious = find_suspicious_fallback_language_indices(transcripts, results)
+
+        self.assertEqual(suspicious, [1])
+
+    @patch(
+        "polytext.converter.audio_to_text.find_suspicious_fallback_language_indices",
+        return_value=[1],
+    )
+    def test_suspicious_fallback_is_retranscribed_with_adaptive_split(self, _mock_find):
+        converter = AudioToTextConverter()
+        chunks = [
+            {"file_path": "/tmp/chunk-0.wav"},
+            {"file_path": "/tmp/chunk-1.wav"},
+            {"file_path": "/tmp/chunk-2.wav"},
+        ]
+        transcripts = ["prima", "wrong English fallback", "dopo"]
+        results = [
+            {"transcript": "prima", "completion_tokens": 1, "prompt_tokens": 1},
+            {
+                "transcript": "wrong English fallback",
+                "completion_tokens": 2,
+                "prompt_tokens": 2,
+                "completion_model": "gemini-3.5-flash-lite",
+                "fallback_from_model": "gemini-3.1-flash-lite",
+            },
+            {"transcript": "dopo", "completion_tokens": 1, "prompt_tokens": 1},
+        ]
+        recovered = {
+            "transcript": "testo italiano recuperato",
+            "completion_tokens": 4,
+            "prompt_tokens": 3,
+            "completion_model": "gemini-3.1-flash-lite",
+        }
+
+        with patch.object(converter, "transcribe_audio_halves", return_value=recovered) as mock_split:
+            converter.recover_suspicious_fallback_languages(chunks, transcripts, results)
+
+        mock_split.assert_called_once_with("/tmp/chunk-1.wav")
+        self.assertEqual(transcripts[1], "testo italiano recuperato")
+        self.assertEqual(results[1]["transcript"], "testo italiano recuperato")
+        self.assertTrue(results[1]["language_validation_triggered"])
+        self.assertEqual(
+            results[1]["language_validation_original_model"],
+            "gemini-3.5-flash-lite",
+        )
+
+
 class _FlakyServerErrorModels:
     def __init__(self):
         self.generate_content_calls = 0
@@ -455,6 +600,20 @@ class TestAudioTranscriptionModelMigration(unittest.TestCase):
         audio_part = fake_client.models.generate_content_contents[2]
         self.assertEqual(audio_part.inline_data.mime_type, "audio/aac")
 
+    @patch("polytext.converter.audio_to_text.genai.Client")
+    def test_inline_wav_uses_gemini_supported_mime_type(self, mock_client_cls):
+        fake_client = _FakeClient()
+        mock_client_cls.return_value = fake_client
+
+        converter = AudioToTextConverter()
+        with tempfile.NamedTemporaryFile(suffix=".wav") as temp_audio:
+            temp_audio.write(b"fake-audio")
+            temp_audio.flush()
+            converter.transcribe_audio(temp_audio.name)
+
+        audio_part = fake_client.models.generate_content_contents[2]
+        self.assertEqual(audio_part.inline_data.mime_type, "audio/wav")
+
     @patch("retry.api.time.sleep")
     def test_does_not_retry_genai_client_error_for_audio_transcription(self, _mock_sleep):
         fake_client = _ClientErrorClient()
@@ -613,10 +772,18 @@ class TestAudioTranscriptionModelMigration(unittest.TestCase):
         with tempfile.NamedTemporaryFile(suffix=".mp3") as source_audio:
             AudioSegment.silent(duration=10_000).export(source_audio.name, format="mp3").close()
             converter = AudioToTextConverter()
+            def transcribe_split(split_path, temperature=0.0):
+                self.assertTrue(split_path.endswith(".wav"))
+                with wave.open(split_path, "rb") as split_audio:
+                    self.assertEqual(split_audio.getframerate(), 16000)
+                    self.assertEqual(split_audio.getnchannels(), 1)
+                    self.assertEqual(split_audio.getsampwidth(), 2)
+                return split_responses.pop(0)
+
             with patch.object(
                 AudioToTextConverter,
                 "transcribe_audio",
-                side_effect=split_responses,
+                side_effect=transcribe_split,
             ) as mock_transcribe:
                 result = converter.transcribe_audio_halves(source_audio.name)
 

{polytext-0.2.8b4 → polytext-0.2.8b6}/tests/test_ocr_fallbacks.py

@@ -88,8 +88,10 @@ class _FakePixmap:
 class _FakePage:
     def __init__(self, payload: bytes = b"fake-page-image"):
         self.payload = payload
+        self.pixmap_calls = []
 
-    def get_pixmap(self):
+    def get_pixmap(self, **kwargs):
+        self.pixmap_calls.append(kwargs)
         return _FakePixmap(payload=self.payload)
 
 
@@ -112,6 +114,44 @@ def _immediate_as_completed(futures):
 
 
 class TestOcrFallbacks(unittest.TestCase):
+    @patch("polytext.converter.document_ocr_to_text.genai.Client")
+    @patch("concurrent.futures.as_completed", side_effect=_immediate_as_completed)
+    @patch("concurrent.futures.ThreadPoolExecutor", _ImmediateExecutor)
+    @patch("fitz.open")
+    def test_document_ocr_renders_pages_at_requested_dpi(
+        self,
+        mock_fitz_open,
+        _mock_as_completed,
+        mock_client_cls,
+    ):
+        page = _FakePage()
+        mock_fitz_open.return_value = _FakePdf([page])
+        mock_client_cls.return_value = _FakeClient()
+
+        converter = DocumentOCRToTextConverter(ocr_render_dpi=200)
+        converter.get_document_ocr("dummy.pdf")
+
+        self.assertEqual(page.pixmap_calls, [{"dpi": 200, "alpha": False}])
+
+    @patch("polytext.converter.document_ocr_to_text.genai.Client")
+    @patch("concurrent.futures.as_completed", side_effect=_immediate_as_completed)
+    @patch("concurrent.futures.ThreadPoolExecutor", _ImmediateExecutor)
+    @patch("fitz.open")
+    def test_document_ocr_preserves_default_pixmap_rendering(
+        self,
+        mock_fitz_open,
+        _mock_as_completed,
+        mock_client_cls,
+    ):
+        page = _FakePage()
+        mock_fitz_open.return_value = _FakePdf([page])
+        mock_client_cls.return_value = _FakeClient()
+
+        converter = DocumentOCRToTextConverter()
+        converter.get_document_ocr("dummy.pdf")
+
+        self.assertEqual(page.pixmap_calls, [{}])
+
     def test_default_ocr_max_output_tokens_is_8192(self):
         converter = OCRToTextConverter()
         self.assertEqual(converter.max_output_tokens, 8192)

{polytext-0.2.8b4 → polytext-0.2.8b6}/tests/test_ocr_image_descriptions.py

@@ -138,6 +138,23 @@ class TestOCRImageDescriptions(unittest.TestCase):
         self.assertIsInstance(document_ocr_loader, DocumentOCRLoader)
         self.assertFalse(document_ocr_loader.allow_partial_ocr_failures)
 
+    def test_forced_document_ocr_passes_custom_render_dpi(self):
+        loader = BaseLoader(
+            source="local",
+            force_ocr=True,
+            ocr_render_dpi=200,
+        )
+
+        document_ocr_loader = loader.init_loader_class(
+            input="/tmp/example.pdf",
+            storage_client={},
+            llm_api_key=None,
+            source="local",
+        )
+
+        self.assertIsInstance(document_ocr_loader, DocumentOCRLoader)
+        self.assertEqual(document_ocr_loader.ocr_render_dpi, 200)
+
     def test_google_ocr_converter_builds_augmented_markdown_prompt(self):
         converter = OCRToTextConverter(include_image_descriptions=True)