polytext 0.2.8b6__tar.gz → 0.2.8b7__tar.gz

{polytext-0.2.8b6 → polytext-0.2.8b7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: polytext
-Version: 0.2.8b6
+Version: 0.2.8b7
 Summary: Python utilities to simplify document files management
 Home-page: https://github.com/docsity/polytext
 Author: Matteo Senardi

{polytext-0.2.8b6 → polytext-0.2.8b7}/polytext/converter/audio_to_text.py

@@ -63,6 +63,8 @@ AUDIO_DEFAULT_MAX_OUTPUT_TOKENS = 4096
 AUDIO_MAX_ADAPTIVE_SPLIT_DEPTH = 1
 AUDIO_ADAPTIVE_SPLIT_OVERLAP_MS = 2000
 AUDIO_LONG_DURATION_THRESHOLD_MS = 80 * 60 * 1000
+AUDIO_LOW_TEXT_THRESHOLD_WORDS_PER_MINUTE = 20
+AUDIO_LOW_TEXT_MIN_DURATION_MS = 60_000
 AUDIO_TAIL_REPETITION_LINES = int(os.getenv("AUDIO_TAIL_REPETITION_LINES", "200"))
 AUDIO_TAIL_REPETITION_THRESHOLD = float(os.getenv("AUDIO_TAIL_REPETITION_THRESHOLD", "0.35"))
 AUDIO_FALLBACK_SOURCE_PATTERN = os.getenv("AUDIO_FALLBACK_SOURCE_PATTERN", "flash-lite")
@@ -284,6 +286,37 @@ def normalize_no_human_speech_marker(text: str) -> tuple[str, bool]:
     return cleaned_text, False
 
 
+def summarize_audio_chunk_quality(chunks: list[dict], transcripts: list[str]) -> dict:
+    """Report empty and unusually short transcripts without discarding any text.
+
+    Indices are one-based. Short audio chunks are excluded from the low-text
+    count because a brief spoken phrase can be a complete transcription.
+    """
+    empty_indices = []
+    low_text_indices = []
+    for index, (chunk, transcript) in enumerate(zip(chunks, transcripts), start=1):
+        word_count = len(transcript.split())
+        if word_count == 0:
+            empty_indices.append(index)
+        else:
+            duration_ms = chunk.get("duration_ms", 0)
+            if (
+                duration_ms >= AUDIO_LOW_TEXT_MIN_DURATION_MS
+                and word_count * 60_000 < AUDIO_LOW_TEXT_THRESHOLD_WORDS_PER_MINUTE * duration_ms
+            ):
+                low_text_indices.append(index)
+
+    return {
+        "total_chunks": len(chunks),
+        "empty_chunks": len(empty_indices),
+        "low_text_chunks": len(low_text_indices),
+        "empty_chunk_indices": empty_indices,
+        "low_text_chunk_indices": low_text_indices,
+        "low_text_threshold_words_per_minute": AUDIO_LOW_TEXT_THRESHOLD_WORDS_PER_MINUTE,
+        "low_text_min_duration_ms": AUDIO_LOW_TEXT_MIN_DURATION_MS,
+    }
+
+
 def add_line_break_after_each_sentence(text: str) -> str:
     if not text:
         return text
@@ -1030,6 +1063,8 @@ class AudioToTextConverter:
                 - prompt_tokens (int): Total number of prompt tokens used
                 - completion_model (str): Name of the transcription model used
                 - completion_model_provider (str): Provider of the transcription model
+                - audio_chunk_quality (dict): Counts of empty and low-text chunks,
+                  with one-based indices and the applied words-per-minute threshold
 
         Raises:
             ValueError: If the audio file format is not recognized
@@ -1121,7 +1156,8 @@ class AudioToTextConverter:
             "completion_tokens": completion_tokens + full_text_merged_dict["completion_tokens"],
             "prompt_tokens": prompt_tokens + full_text_merged_dict["prompt_tokens"],
             "completion_model": self.transcription_model,
-            "completion_model_provider": self.transcription_model_provider
+            "completion_model_provider": self.transcription_model_provider,
+            "audio_chunk_quality": summarize_audio_chunk_quality(chunks, transcript_chunks),
         }
         if len(chunk_results) == 1 and chunk_results[0]:
             for key in (

{polytext-0.2.8b6 → polytext-0.2.8b7}/polytext/loader/base.py

@@ -42,6 +42,7 @@ logger = logging.getLogger(__name__)
 
 MIN_DOC_TEXT_LENGTH_ACCEPTED = int(os.getenv("MIN_DOC_TEXT_LENGTH_ACCEPTED", "400"))
 OCR_INCLUDE_IMAGE_DESCRIPTIONS_ENV = "OCR_INCLUDE_IMAGE_DESCRIPTIONS"
+FALLBACK_OCR_ENV = "FALLBACK_OCR"
 LLM_OUTPUT_ERROR_CODES = {
     995: "INVALID_ARGUMENT",
     996: "RECITATION",
@@ -119,7 +120,7 @@ class BaseLoader:
                 - target_size (int, optional): Target file size in bytes. Defaults to 1MB
                 - source (str): Source of the document. Must be either "cloud" or "local"
                 - fallback_ocr (bool, optional): If True, Optical Character Recognition (OCR) will be used as a fallback
-                  if direct text extraction fails, particularly for image-based content. Defaults to True.
+                  if direct text extraction fails. When omitted, defaults from FALLBACK_OCR and otherwise to False.
                 - save_transcript_chunks (bool, optional): Whether to save chunk transcripts in final output. Defaults to False.
                 - bitrate_quality (int, optional): Variable bitrate quality from 0-9 (9 being lowest). Defaults to 9
 
@@ -143,7 +144,10 @@ class BaseLoader:
         self.kwargs = kwargs
         self.target_size = kwargs.get("target_size", 1)
         self.source = kwargs.get("source", "cloud")
-        self.fallback_ocr = kwargs.get("fallback_ocr", False)
+        self.fallback_ocr = kwargs.get(
+            "fallback_ocr",
+            _read_bool_env(FALLBACK_OCR_ENV),
+        )
         self.save_transcript_chunks = kwargs.get("save_transcript_chunks", False)
         self.bitrate_quality = kwargs.get("bitrate_quality", 9)
         self.aws_auth_mode = kwargs.get("aws_auth_mode")
@@ -548,6 +552,7 @@ class BaseLoader:
                 - completion_model (str): Model name used, from the first processed source.
                 - completion_model_provider (str): Provider of the model, from the first processed source.
                 - text_chunks (list, optional): List of text chunks, if chunking was applied.
+                - audio_chunk_quality (dict, optional): Counts of empty and low-text audio chunks.
                 - type (str): Type of the processed source (e.g., ocr, video, audio, text).
                 - input (str): The input path or URL of the first processed source.
                 - output_list (list): List of individual extraction results (one per input).
@@ -608,7 +613,7 @@ class BaseLoader:
             "output_list": [result_dict],
         }
 
-        for metadata_key in ("ocr_failed_pages", "ocr_failed_pages_detail"):
+        for metadata_key in ("ocr_failed_pages", "ocr_failed_pages_detail", "audio_chunk_quality"):
             if metadata_key in result_dict:
                 final_result[metadata_key] = result_dict[metadata_key]
 

{polytext-0.2.8b6 → polytext-0.2.8b7}/polytext.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: polytext
-Version: 0.2.8b6
+Version: 0.2.8b7
 Summary: Python utilities to simplify document files management
 Home-page: https://github.com/docsity/polytext
 Author: Matteo Senardi

{polytext-0.2.8b6 → polytext-0.2.8b7}/polytext.egg-info/SOURCES.txt

@@ -56,6 +56,7 @@ polytext/prompts/text_to_md.py
 polytext/prompts/transcription.py
 polytext/utils/__init__.py
 polytext/utils/utils.py
+tests/test_audio_chunk_quality.py
 tests/test_audio_chunker.py
 tests/test_audio_comparison_helpers.py
 tests/test_audio_transcription_model_migration.py

{polytext-0.2.8b6 → polytext-0.2.8b7}/setup.py

@@ -51,7 +51,7 @@ def get_requirements(*requirements_file):
 
 setup(
     name='polytext',
-    version='0.2.8b6',
+    version='0.2.8b7',
     url='https://github.com/docsity/polytext',
     # download_url='https://github.com/pualien/py-polytext/archive/0.1.23.tar.gz',
     license='MIT',

polytext-0.2.8b7/tests/test_audio_chunk_quality.py

@@ -0,0 +1,76 @@
+import tempfile
+import unittest
+from unittest.mock import MagicMock, patch
+
+from polytext.converter import audio_to_text
+from polytext.loader import BaseLoader
+
+
+class AudioChunkQualityTests(unittest.TestCase):
+    def test_counts_empty_and_low_text_chunks_separately(self):
+        chunks = [
+            {"duration_ms": 120_000},
+            {"duration_ms": 120_000},
+            {"duration_ms": 120_000},
+            {"duration_ms": 20_000},
+            {"duration_ms": 1_200_000},
+        ]
+        transcripts = ["parola " * 40, "  \n", "parola " * 12, "ciao", "parola " * 100]
+
+        quality = audio_to_text.summarize_audio_chunk_quality(chunks, transcripts)
+
+        self.assertEqual(quality, {
+            "total_chunks": 5,
+            "empty_chunks": 1,
+            "low_text_chunks": 2,
+            "empty_chunk_indices": [2],
+            "low_text_chunk_indices": [3, 5],
+            "low_text_threshold_words_per_minute": 20,
+            "low_text_min_duration_ms": 60_000,
+        })
+
+    @patch("polytext.converter.audio_to_text.TextMerger")
+    @patch("polytext.converter.audio_to_text.AudioChunker")
+    @patch.object(audio_to_text.AudioToTextConverter, "process_chunk")
+    def test_quality_is_returned_without_saving_chunk_transcripts(
+        self, mock_process_chunk, mock_chunker_cls, mock_merger_cls
+    ):
+        chunker = mock_chunker_cls.return_value
+        chunker.duration_ms = 240_000
+        chunker.extract_chunks.return_value = [
+            {"file_path": "/tmp/chunk1.mp3", "duration_ms": 120_000},
+            {"file_path": "/tmp/chunk2.mp3", "duration_ms": 120_000},
+        ]
+        mock_process_chunk.side_effect = lambda chunk, index: (
+            index,
+            {"transcript": "parola " * 40 if index == 0 else "", "completion_tokens": 4, "prompt_tokens": 5},
+        )
+        mock_merger_cls.return_value.merge_chunks_with_llm_sequential.return_value = {
+            "full_text_merged": "testo finale", "completion_tokens": 0, "prompt_tokens": 0
+        }
+
+        with tempfile.NamedTemporaryFile(suffix=".mp3") as source_audio:
+            converter = audio_to_text.AudioToTextConverter()
+            result = converter.transcribe_full_audio(source_audio.name)
+
+        self.assertEqual(result["text"], "testo finale")
+        self.assertEqual(result["audio_chunk_quality"]["total_chunks"], 2)
+        self.assertEqual(result["audio_chunk_quality"]["empty_chunks"], 1)
+        self.assertNotIn("text_chunks", result)
+
+    def test_quality_reaches_top_level_loader_response(self):
+        quality = {"total_chunks": 2, "empty_chunks": 1, "low_text_chunks": 0}
+        file_loader = MagicMock()
+        file_loader.load.return_value = {
+            "text": "trascrizione", "completion_tokens": 3, "prompt_tokens": 4,
+            "audio_chunk_quality": quality,
+        }
+
+        response = BaseLoader().run_loader_class(file_loader, ["audio.mp3"])
+
+        self.assertEqual(response["audio_chunk_quality"], quality)
+        self.assertEqual(response["output_list"][0]["audio_chunk_quality"], quality)
+
+
+if __name__ == "__main__":
+    unittest.main()

{polytext-0.2.8b6 → polytext-0.2.8b7}/tests/test_base_loader_error_mapping.py

@@ -1,3 +1,4 @@
+import os
 import unittest
 from types import SimpleNamespace
 from unittest.mock import Mock, patch
@@ -60,6 +61,24 @@ class _BeautifulTextLoader(BaseLoader):
 
 
 class TestBaseLoaderErrorMapping(unittest.TestCase):
+    def test_document_ocr_fallback_is_enabled_by_environment(self):
+        with patch.dict(os.environ, {"FALLBACK_OCR": "true"}, clear=True):
+            loader = BaseLoader()
+
+        self.assertTrue(loader.fallback_ocr)
+
+    def test_explicit_document_ocr_fallback_setting_overrides_environment(self):
+        with patch.dict(os.environ, {"FALLBACK_OCR": "true"}, clear=True):
+            loader = BaseLoader(fallback_ocr=False)
+
+        self.assertFalse(loader.fallback_ocr)
+
+    def test_document_ocr_fallback_remains_disabled_without_configuration(self):
+        with patch.dict(os.environ, {}, clear=True):
+            loader = BaseLoader()
+
+        self.assertFalse(loader.fallback_ocr)
+
     def test_llm_output_empty_document_codes_are_raised_as_loader_errors(self):
         cases = [
             (995, "INVALID_ARGUMENT"),