polytext 0.2.2b2__tar.gz → 0.2.3__tar.gz

{polytext-0.2.2b2 → polytext-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: polytext
-Version: 0.2.2b2
+Version: 0.2.3
 Summary: Python utilities to simplify document files management
 Home-page: https://github.com/docsity/polytext
 Author: Matteo Senardi

{polytext-0.2.2b2 → polytext-0.2.3}/polytext/converter/audio_to_text.py

@@ -90,16 +90,22 @@ def compress_and_convert_audio(input_path: str, bitrate_quality: int = 9) -> str
     os.close(fd)
 
     logger.info(f"Compressing audio to bitrate quality: {bitrate_quality}")
-    ffmpeg.input(input_path).output(
-        temp_audio_path,
-        q=bitrate_quality,  # Variable bitrate quality (0-9, 9 being lowest)
-        acodec='libmp3lame',
-        ac=1,  # Convert to mono
-        ar=16000,  # Lower sample rate
-        vn=None,
-        threads=0,  # Use maximum available threads
-        loglevel='error',  # Reduce logging overhead
-    ).run(quiet=True, overwrite_output=True)
+    try:
+        ffmpeg.input(input_path).output(
+            temp_audio_path,
+            q=bitrate_quality,  # Variable bitrate quality (0-9, 9 being lowest)
+            acodec='libmp3lame',
+            ac=1,  # Convert to mono
+            ar=16000,  # Lower sample rate
+            vn=None,
+            threads=0,  # Use maximum available threads
+            loglevel='error',  # Reduce logging overhead
+        ).run(quiet=True, overwrite_output=True)
+    except Exception:
+        logger.exception("FFmpeg error during audio processing for %s", input_path)
+        if os.path.exists(temp_audio_path):
+            os.unlink(temp_audio_path)
+        raise
 
     logger.info(f"Successfully converted and compressed audio: {temp_audio_path}")
     return temp_audio_path
@@ -313,7 +319,11 @@ class AudioToTextConverter:
 
         mime_type, _ = mimetypes.guess_type(audio_file)
         if mime_type is None:
-            raise ValueError("Audio format not recognized")
+            try:
+                raise ValueError("Audio format not recognized")
+            except ValueError:
+                logger.exception("Unsupported audio format for %s", audio_file)
+                raise
 
         return client.models.generate_content(
             model=self.transcription_model,

{polytext-0.2.2b2 → polytext-0.2.3}/polytext/converter/document_ocr_to_text.py

@@ -96,6 +96,7 @@ def compress_and_convert_image(input_path: str, target_size=1):
         return temp_image_path
 
     except Exception as e:
+        logger.exception("FFmpeg error during image processing for %s", input_path)
         raise RuntimeError(f"FFmpeg error during image processing: {e}") from e
 
 def get_document_ocr(
@@ -389,7 +390,11 @@ class DocumentOCRToTextConverter:
                 # Determine mimetype
                 mime_type, _ = mimetypes.guess_type(temp_file_for_ocr)
                 if mime_type is None:
-                    raise ValueError("Image format not recognized")
+                    try:
+                        raise ValueError("Image format not recognized")
+                    except ValueError:
+                        logger.exception("Unsupported image format for %s", temp_file_for_ocr)
+                        raise
 
                 response = client.models.generate_content(
                     model=self.ocr_model,

{polytext-0.2.2b2 → polytext-0.2.3}/polytext/converter/document_ocr_to_text_azure_oai.py

@@ -88,6 +88,7 @@ def compress_and_convert_image(input_path: str, target_size=1) -> str:
         return temp_image_path
 
     except Exception as e:
+        logger.exception("FFmpeg error during image processing for %s", input_path)
         raise RuntimeError(f"FFmpeg error during image processing: {e}") from e
 
 
@@ -233,7 +234,11 @@ class DocumentOCRToTextConverter:
 
             mime_type, _ = mimetypes.guess_type(temp_file_for_ocr)
             if mime_type is None:
-                raise ValueError("Image format not recognized")
+                try:
+                    raise ValueError("Image format not recognized")
+                except ValueError:
+                    logger.exception("Unsupported image format for %s", temp_file_for_ocr)
+                    raise
 
             with open(temp_file_for_ocr, "rb") as f:
                 image_b64 = base64.b64encode(f.read()).decode("utf-8")
@@ -308,7 +313,7 @@ class DocumentOCRToTextConverter:
             pdf = fitz.open(document_for_ocr)
             total_pages = len(pdf)
             if total_pages == 0:
-                raise EmptyDocument(message="The document has no pages.", code=997)
+                raise EmptyDocument(message="The document has no pages.", code=998)
 
             start_page, end_page = self.validate_page_range(total_pages)
 

{polytext-0.2.2b2 → polytext-0.2.3}/polytext/converter/ocr_to_text.py

@@ -96,6 +96,7 @@ def compress_and_convert_image(input_path: str, target_size=1):
         return temp_image_path
 
     except Exception as e:
+        logger.exception("FFmpeg error during image processing for %s", input_path)
         raise RuntimeError(f"FFmpeg error during image processing: {e}") from e
 
 def get_ocr(
@@ -383,7 +384,11 @@ class OCRToTextConverter:
                 # Determine mimetype
                 mime_type, _ = mimetypes.guess_type(temp_file_for_ocr)
                 if mime_type is None:
-                    raise ValueError("Image format not recognized")
+                    try:
+                        raise ValueError("Image format not recognized")
+                    except ValueError:
+                        logger.exception("Unsupported image format for %s", temp_file_for_ocr)
+                        raise
 
                 response = client.models.generate_content(
                     model=self.ocr_model,

{polytext-0.2.2b2 → polytext-0.2.3}/polytext/converter/ocr_to_text_azure_oai.py

@@ -87,6 +87,7 @@ def compress_and_convert_image(input_path: str, target_size=1) -> str:
         return temp_image_path
 
     except Exception as e:
+        logger.exception("FFmpeg error during image processing for %s", input_path)
         raise RuntimeError(f"FFmpeg error during image processing: {e}") from e
 
 
@@ -224,7 +225,11 @@ class OCRToTextConverter:
             # We'll use base64 data-URL.
             mime_type, _ = mimetypes.guess_type(temp_file_for_ocr)
             if mime_type is None:
-                raise ValueError("Image format not recognized")
+                try:
+                    raise ValueError("Image format not recognized")
+                except ValueError:
+                    logger.exception("Unsupported image format for %s", temp_file_for_ocr)
+                    raise
 
             with open(temp_file_for_ocr, "rb") as f:
                 image_b64 = base64.b64encode(f.read()).decode("utf-8")

{polytext-0.2.2b2 → polytext-0.2.3}/polytext/converter/video_to_audio.py

@@ -52,12 +52,12 @@ def convert_video_to_audio(video_file: str , bitrate_quality: int =9) -> str:
         return temp_audio_path
 
     except ffmpeg.Error as e:
-        logger.info(f"FFmpeg conversion failed: {e.stderr.decode()}")
+        logger.exception("FFmpeg conversion failed: %s", e.stderr.decode())
         if os.path.exists(temp_audio_path):
             os.unlink(temp_audio_path)
         raise
     except Exception as e:
-        logger.info(f"Failed to convert video to audio: {str(e)}")
+        logger.exception("Failed to convert video to audio: %s", str(e))
         if os.path.exists(temp_audio_path):
             os.unlink(temp_audio_path)
-        raise
+        raise

{polytext-0.2.2b2 → polytext-0.2.3}/polytext/loader/base.py

@@ -40,6 +40,12 @@ logger = logging.getLogger(__name__)
 
 MIN_DOC_TEXT_LENGTH_ACCEPTED = int(os.getenv("MIN_DOC_TEXT_LENGTH_ACCEPTED", "400"))
 OCR_INCLUDE_IMAGE_DESCRIPTIONS_ENV = "OCR_INCLUDE_IMAGE_DESCRIPTIONS"
+LLM_OUTPUT_ERROR_CODES = {
+    995: "INVALID_ARGUMENT",
+    996: "RECITATION",
+    997: "REPETITIVE_OUTPUT",
+    999: "MAX_TOKENS",
+}
 
 
 def _read_bool_env(name: str, default: bool = False) -> bool:
@@ -148,6 +154,18 @@ class BaseLoader:
             response = self.run_loader_class(loader_class=loader_class, input_list=input_list)
         except EmptyDocument as e:
             logger.info(f"Empty document encountered: {e.message}")
+            if e.code in LLM_OUTPUT_ERROR_CODES:
+                logger.exception(
+                    "Raising LoaderError: status=422 code=%s original_empty_document_code=%s message=%s",
+                    LLM_OUTPUT_ERROR_CODES[e.code],
+                    e.code,
+                    e.message,
+                )
+                raise LoaderError(
+                    message=e.message,
+                    status=422,
+                    code=LLM_OUTPUT_ERROR_CODES[e.code],
+                ) from e
             if self.fallback_ocr:
                 loader_class = self.init_loader_class(input=first_file_url, storage_client=storage_client,
                                                       llm_api_key=self.llm_api_key, is_document_fallback=True, **kwargs)
@@ -317,6 +335,7 @@ class BaseLoader:
                 return YoutubeTranscriptLoaderWithLlm(llm_api_key=llm_api_key, markdown_output=self.markdown_output, temp_dir=self.temp_dir, timeout_minutes=self.timeout_minutes, **kwargs)
             else:
                 return HtmlLoader(markdown_output=self.markdown_output)
+        # Handle markdown files based on extension or MIME type
         if file_extension in [".md", ".markdown"] or (
                 mime_type and mime_type.startswith("text/markdown")
         ):
@@ -344,7 +363,11 @@ class BaseLoader:
                     **kwargs,
                 )
             else:
-                raise ValueError(f"Unsupported MIME type: {mime_type}")
+                try:
+                    raise ValueError(f"Unsupported MIME type: {mime_type}")
+                except ValueError:
+                    logger.exception("Unsupported media type while initializing loader: %s", mime_type)
+                    raise
 
         elif self.validate_user_text(text=input):
             return PlainTextLoader(

{polytext-0.2.2b2 → polytext-0.2.3}/polytext/loader/youtube_llm.py

@@ -384,7 +384,7 @@ class YoutubeTranscriptLoaderWithLlm:
                     fallback_model=self.final_fallback_model,
                     fallback_temperature=self.final_fallback_temperature,
                 )
-            raise Exception(f"Invalid argument: {e.message}; details={getattr(e, 'details', None)}")
+            raise e_tmp from e
 
         except errors.ServerError as e:
             logger.info("ServerError occurred with status %s and message: %s", e.status, e.message)

{polytext-0.2.2b2 → polytext-0.2.3}/polytext.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: polytext
-Version: 0.2.2b2
+Version: 0.2.3
 Summary: Python utilities to simplify document files management
 Home-page: https://github.com/docsity/polytext
 Author: Matteo Senardi

{polytext-0.2.2b2 → polytext-0.2.3}/polytext.egg-info/SOURCES.txt

@@ -56,6 +56,7 @@ polytext/utils/utils.py
 tests/test_audio_chunker.py
 tests/test_audio_comparison_helpers.py
 tests/test_audio_transcription_model_migration.py
+tests/test_base_loader_error_mapping.py
 tests/test_compare_audio_models.py
 tests/test_compare_document_ocr_to_text_models.py
 tests/test_compare_ocr_to_text_models.py
@@ -63,6 +64,7 @@ tests/test_compare_youtube_models.py
 tests/test_dowload_audio_from_youtube.py
 tests/test_dowload_audio_from_youtube_helpers.py
 tests/test_extracted_text_whitespace.py
+tests/test_gemini_quality_guards.py
 tests/test_get_audio_transcript_from_gcs.py
 tests/test_get_customized_pdf_from_markdown.py
 tests/test_get_document_ocr.py
@@ -79,7 +81,9 @@ tests/test_notebook_loader.py
 tests/test_ocr_fallbacks.py
 tests/test_ocr_image_descriptions.py
 tests/test_pain_text.py
+tests/test_python_version_metadata.py
 tests/test_split_audio_with_llm.py
 tests/test_xml_xbrl_loader.py
 tests/test_youtube_gemini_minimal_check.py
+tests/test_youtube_llm_fallbacks.py
 tests/test_youtube_transcript.py

{polytext-0.2.2b2 → polytext-0.2.3}/setup.py

@@ -51,7 +51,7 @@ def get_requirements(*requirements_file):
 
 setup(
     name='polytext',
-    version='0.2.2b2',
+    version='0.2.3',
     url='https://github.com/docsity/polytext',
     # download_url='https://github.com/pualien/py-polytext/archive/0.1.23.tar.gz',
     license='MIT',

polytext-0.2.3/tests/test_base_loader_error_mapping.py

@@ -0,0 +1,79 @@
+import unittest
+from unittest.mock import patch
+
+from polytext.exceptions import EmptyDocument, LoaderError
+from polytext.loader.base import BaseLoader
+
+
+class _FailingLoader:
+    def __init__(self, error):
+        self.error = error
+
+    def load(self, input_path):
+        raise self.error
+
+
+class _FakeBaseLoader(BaseLoader):
+    def __init__(self, error, **kwargs):
+        super().__init__(**kwargs)
+        self.error = error
+
+    def initiate_storage(self, input):
+        return {}
+
+    def init_loader_class(self, input, storage_client, llm_api_key, is_document_fallback=False, **kwargs):
+        return _FailingLoader(self.error)
+
+
+class TestBaseLoaderErrorMapping(unittest.TestCase):
+    def test_llm_output_empty_document_codes_are_raised_as_loader_errors(self):
+        cases = [
+            (995, "INVALID_ARGUMENT"),
+            (996, "RECITATION"),
+            (997, "REPETITIVE_OUTPUT"),
+            (999, "MAX_TOKENS"),
+        ]
+
+        for empty_document_code, expected_loader_code in cases:
+            with self.subTest(empty_document_code=empty_document_code):
+                loader = _FakeBaseLoader(
+                    EmptyDocument(
+                        message=f"diagnostic failure {empty_document_code}",
+                        code=empty_document_code,
+                    )
+                )
+
+                with patch("polytext.loader.base.logger.exception") as mock_exception:
+                    with self.assertRaises(LoaderError) as error_context:
+                        loader.get_text(["dummy.txt"])
+
+                error = error_context.exception
+                self.assertEqual(error.status, 422)
+                self.assertEqual(error.code, expected_loader_code)
+                self.assertEqual(error.message, f"diagnostic failure {empty_document_code}")
+                mock_exception.assert_called_once()
+                self.assertIn("Raising LoaderError", mock_exception.call_args.args[0])
+                self.assertEqual(mock_exception.call_args.args[1], expected_loader_code)
+                self.assertEqual(mock_exception.call_args.args[2], empty_document_code)
+                self.assertEqual(mock_exception.call_args.args[3], f"diagnostic failure {empty_document_code}")
+
+    def test_empty_or_too_short_documents_still_return_empty_response(self):
+        loader = _FakeBaseLoader(
+            EmptyDocument(
+                message="Document text with less than 400 characters",
+                code=998,
+            )
+        )
+
+        with patch("polytext.loader.base.logger.exception") as mock_exception:
+            response = loader.get_text(["empty.txt"])
+
+        self.assertEqual(response["text"], "")
+        self.assertEqual(response["completion_tokens"], 0)
+        self.assertEqual(response["prompt_tokens"], 0)
+        self.assertEqual(response["output_list"][0]["input"], "empty.txt")
+        mock_exception.assert_not_called()
+
+
+if __name__ == "__main__":
+    unittest.main()

polytext-0.2.3/tests/test_gemini_quality_guards.py

@@ -0,0 +1,31 @@
+import unittest
+
+from polytext.converter.gemini_quality_guards import tail_has_excessive_repetition
+
+
+class TestGeminiQualityGuards(unittest.TestCase):
+    def test_detects_consecutive_repeated_sentences_below_ratio_threshold(self):
+        text = (
+            "gli davamo nomi veri e falsi. "
+            "_Elio_ all'anagrafe e io gli gli dicevo _Roberto Gustativi_. "
+            "E questi scrivevano Roberto Gustativi. "
+            "E la soddisfazione perversa era andare a comprare il giornale. "
+            "È successo. "
+            "Sono stato. "
+            "Che è successo? "
+            "Siamo passati dal basement. "
+            "Siamo passati dal basement. "
+            "Siamo passati dal basement. "
+            "Siamo passati dal basement. "
+            "Il miglior finale di sempre. "
+            "Grazie, grazie, grazie, grazie, grazie, grazie, grazie. "
+            "E vi grazie."
+        )
+
+        self.assertTrue(
+            tail_has_excessive_repetition(text, tail_lines=200, threshold=0.35)
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

{polytext-0.2.2b2 → polytext-0.2.3}/tests/test_ocr_fallbacks.py

@@ -4,7 +4,11 @@ from types import SimpleNamespace
 from unittest.mock import patch
 
 from polytext.converter.document_ocr_to_text import DocumentOCRToTextConverter
+from polytext.converter.document_ocr_to_text_azure_oai import (
+    DocumentOCRToTextConverter as AzureDocumentOCRToTextConverter,
+)
 from polytext.converter.ocr_to_text import OCRToTextConverter
+from polytext.exceptions import EmptyDocument
 
 
 def _make_response(
@@ -208,11 +212,12 @@ class TestOcrFallbacks(unittest.TestCase):
         )
         mock_client_cls.return_value = fake_client
 
-        converter = OCRToTextConverter(ocr_model="gemini-3.1-flash-lite-preview")
-        with tempfile.NamedTemporaryFile(suffix=".png") as temp_image:
-            temp_image.write(b"fake-image")
-            temp_image.flush()
-            result = converter.get_ocr(temp_image.name)
+        with patch("polytext.converter.ocr_to_text.OCR_FINAL_FALLBACK_MODEL", "gemini-2.0-flash"):
+            converter = OCRToTextConverter(ocr_model="gemini-3.1-flash-lite-preview")
+            with tempfile.NamedTemporaryFile(suffix=".png") as temp_image:
+                temp_image.write(b"fake-image")
+                temp_image.flush()
+                result = converter.get_ocr(temp_image.name)
 
         self.assertEqual(result["text"], "final fallback text")
         self.assertEqual(
@@ -263,6 +268,24 @@ class TestOcrFallbacks(unittest.TestCase):
         )
         self.assertEqual(fake_client.models.generate_content_temperatures, [0.0, 0.0, 1.0])
 
+    @patch("fitz.open")
+    def test_azure_document_ocr_no_pages_is_empty_or_too_short(self, mock_fitz_open):
+        mock_fitz_open.return_value = _FakePdf([])
+
+        converter = AzureDocumentOCRToTextConverter(
+            azure_endpoint="https://example.openai.azure.com",
+            azure_api_version="2024-10-21",
+        )
+
+        with tempfile.NamedTemporaryFile(suffix=".pdf") as temp_pdf:
+            temp_pdf.write(b"%PDF-1.4\n")
+            temp_pdf.flush()
+            with self.assertRaises(EmptyDocument) as error_context:
+                converter.get_document_ocr(temp_pdf.name)
+
+        self.assertEqual(error_context.exception.code, 998)
+        self.assertEqual(error_context.exception.message, "The document has no pages.")
+
 
 if __name__ == "__main__":
     unittest.main()

polytext-0.2.3/tests/test_python_version_metadata.py

@@ -0,0 +1,45 @@
+import ast
+import tomllib
+import unittest
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def _setup_keyword(name):
+    tree = ast.parse((ROOT / "setup.py").read_text())
+    setup_call = next(
+        node
+        for node in ast.walk(tree)
+        if isinstance(node, ast.Call)
+        and getattr(node.func, "id", None) == "setup"
+    )
+    return next(
+        keyword.value
+        for keyword in setup_call.keywords
+        if keyword.arg == name
+    )
+
+
+class PythonVersionMetadataTest(unittest.TestCase):
+    def test_packaging_metadata_allows_python_311(self):
+        setup_python_requires = ast.literal_eval(_setup_keyword("python_requires"))
+        pyproject = tomllib.loads((ROOT / "pyproject.toml").read_text())
+
+        self.assertEqual(setup_python_requires, ">=3.11")
+        self.assertEqual(
+            pyproject["tool"]["poetry"]["dependencies"]["python"],
+            ">=3.11,<3.14",
+        )
+
+    def test_setup_classifiers_include_supported_python_versions(self):
+        classifiers = ast.literal_eval(_setup_keyword("classifiers"))
+
+        self.assertIn("Programming Language :: Python :: 3.11", classifiers)
+        self.assertIn("Programming Language :: Python :: 3.12", classifiers)
+        self.assertIn("Programming Language :: Python :: 3.13", classifiers)
+
+
+if __name__ == "__main__":
+    unittest.main()

polytext-0.2.3/tests/test_youtube_llm_fallbacks.py

@@ -0,0 +1,103 @@
+import unittest
+from types import SimpleNamespace
+from unittest.mock import patch
+
+from google.genai import errors as genai_errors
+from google.genai import types
+
+from polytext.loader.youtube_llm import YoutubeTranscriptLoaderWithLlm
+from polytext.exceptions import EmptyDocument
+
+
+def _make_response(text="full transcript"):
+    return SimpleNamespace(
+        text=text,
+        candidates=[SimpleNamespace(finish_reason="STOP")],
+        usage_metadata=SimpleNamespace(
+            candidates_token_count=3,
+            prompt_token_count=2,
+            total_token_count=5,
+        ),
+    )
+
+
+class _FakeModels:
+    def __init__(self, response):
+        self.response = response
+        self.generate_content_config = None
+        self.generate_content_model = None
+
+    def generate_content(self, model, contents, config):
+        self.generate_content_model = model
+        self.generate_content_config = config
+        if isinstance(self.response, Exception):
+            raise self.response
+        return self.response
+
+
+class _FakeClient:
+    def __init__(self, response):
+        self.models = _FakeModels(response)
+
+
+def _invalid_argument_error():
+    return genai_errors.ClientError(
+        400,
+        {
+            "error": {
+                "code": 400,
+                "message": "Request contains an invalid argument.",
+                "status": "INVALID_ARGUMENT",
+            }
+        },
+        None,
+    )
+
+
+def _long_transcript():
+    return " ".join(
+        f"This is transcript sentence number {index} with unique content."
+        for index in range(20)
+    )
+
+
+class TestYoutubeLlmFallbacks(unittest.TestCase):
+    @patch("polytext.loader.youtube_llm.genai.Client")
+    def test_invalid_argument_final_fallback_uses_original_temperature(self, mock_client_cls):
+        clients = [
+            _FakeClient(_invalid_argument_error()),
+            _FakeClient(_invalid_argument_error()),
+            _FakeClient(_make_response(_long_transcript())),
+        ]
+        mock_client_cls.side_effect = clients
+
+        loader = YoutubeTranscriptLoaderWithLlm()
+        result = loader.get_text_from_youtube("https://www.youtube.com/watch?v=example")
+
+        self.assertEqual(result["completion_model"], "models/gemini-2.5-flash")
+        self.assertEqual(clients[2].models.generate_content_config.temperature, 0.0)
+        self.assertEqual(
+            clients[2].models.generate_content_config.media_resolution,
+            types.MediaResolution.MEDIA_RESOLUTION_LOW,
+        )
+
+    @patch("polytext.loader.youtube_llm.genai.Client")
+    def test_invalid_argument_after_fallbacks_raises_empty_document_code_995(self, mock_client_cls):
+        clients = [
+            _FakeClient(_invalid_argument_error()),
+            _FakeClient(_invalid_argument_error()),
+            _FakeClient(_invalid_argument_error()),
+        ]
+        mock_client_cls.side_effect = clients
+
+        loader = YoutubeTranscriptLoaderWithLlm()
+
+        with self.assertRaises(EmptyDocument) as error_context:
+            loader.get_text_from_youtube("https://www.youtube.com/watch?v=example")
+
+        self.assertEqual(error_context.exception.code, 995)
+        self.assertIn("INVALID_ARGUMENT", error_context.exception.message)
+
+
+if __name__ == "__main__":
+    unittest.main()