polytext 0.2.3__tar.gz → 0.2.5__tar.gz

{polytext-0.2.3 → polytext-0.2.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: polytext
-Version: 0.2.3
+Version: 0.2.5
 Summary: Python utilities to simplify document files management
 Home-page: https://github.com/docsity/polytext
 Author: Matteo Senardi

{polytext-0.2.3 → polytext-0.2.5}/polytext/__init__.py

@@ -3,11 +3,36 @@ import os
 import logging
 import dotenv
 
+from .exceptions.base import EmptyDocument, ExceededMaxPages, ConversionError, LoaderError
+
 logger = logging.getLogger(__name__)
 
 # Load environment variables
 dotenv.load_dotenv()
 
+
+def _filter_expected_loader_errors(event, hint):
+    error = None
+    if hint:
+        exc_info = hint.get("exc_info")
+        if exc_info:
+            error = exc_info[1]
+        else:
+            error = hint.get("original_exception")
+
+    if isinstance(error, LoaderError) and error.code == "NO_TEXT_DETECTED":
+        return None
+
+    exception_values = (event or {}).get("exception", {}).get("values", [])
+    for exception_value in exception_values:
+        exception_type = exception_value.get("type") or ""
+        exception_message = exception_value.get("value")
+        if exception_type.endswith("LoaderError") and exception_message == "No text detected":
+            return None
+
+    return event
+
+
 # Initialize Sentry if DSN is configured
 sentry_dsn = os.getenv('SENTRY_DSN_POLYTEXT')
 if sentry_dsn:
@@ -18,6 +43,7 @@ if sentry_dsn:
             environment=os.getenv('ENV', 'prod'),
             traces_sample_rate=1.0,
             profiles_sample_rate=1.0,
+            before_send=_filter_expected_loader_errors,
         )
         logger.info("Sentry monitoring initialized")
     except ImportError:
@@ -26,7 +52,6 @@ if sentry_dsn:
 
 from .converter.pdf import convert_to_pdf, DocumentConverter
 from .loader.document import DocumentLoader
-from .exceptions.base import EmptyDocument, ExceededMaxPages, ConversionError
 from .generator.pdf import get_customized_pdf_from_markdown, PDFGenerator
 
 __all__ = [
@@ -38,4 +63,4 @@ __all__ = [
     'ConversionError',
     'get_customized_pdf_from_markdown',
     'PDFGenerator'
-]
+]

{polytext-0.2.3 → polytext-0.2.5}/polytext/converter/audio_to_text.py

@@ -46,7 +46,7 @@ AUDIO_TAIL_REPETITION_THRESHOLD = float(os.getenv("AUDIO_TAIL_REPETITION_THRESHO
 AUDIO_FALLBACK_SOURCE_PATTERN = os.getenv("AUDIO_FALLBACK_SOURCE_PATTERN", "flash-lite")
 AUDIO_FALLBACK_MODEL = os.getenv("AUDIO_FALLBACK_MODEL", "gemini-3-flash-preview")
 AUDIO_FALLBACK_TEMPERATURE = float(os.getenv("AUDIO_FALLBACK_TEMPERATURE", "1.0"))
-AUDIO_FINAL_FALLBACK_MODEL = os.getenv("AUDIO_FINAL_FALLBACK_MODEL", "gemini-2.0-flash")
+AUDIO_FINAL_FALLBACK_MODEL = os.getenv("AUDIO_FINAL_FALLBACK_MODEL", "gemini-3.5-flash")
 AUDIO_FILE_UPLOAD_THRESHOLD_BYTES = 20 * 1024 * 1024
 NO_HUMAN_SPEECH_MARKER = "no human speech detected"
 

{polytext-0.2.3 → polytext-0.2.5}/polytext/converter/ocr_to_text.py

@@ -33,7 +33,7 @@ OCR_TAIL_REPETITION_THRESHOLD = float(os.getenv("OCR_TAIL_REPETITION_THRESHOLD",
 OCR_FALLBACK_SOURCE_PATTERN = os.getenv("OCR_FALLBACK_SOURCE_PATTERN", "flash-lite-preview")
 OCR_FALLBACK_MODEL = os.getenv("OCR_FALLBACK_MODEL", "gemini-3-flash-preview")
 OCR_FALLBACK_TEMPERATURE = float(os.getenv("OCR_FALLBACK_TEMPERATURE", "1.0"))
-OCR_FINAL_FALLBACK_MODEL = os.getenv("OCR_FINAL_FALLBACK_MODEL", "gemini-2.0-flash")
+OCR_FINAL_FALLBACK_MODEL = os.getenv("OCR_FINAL_FALLBACK_MODEL", "gemini-3.5-flash")
 
 
 def compress_and_convert_image(input_path: str, target_size=1):

{polytext-0.2.3 → polytext-0.2.5}/polytext/converter/pdf.py

@@ -127,11 +127,24 @@ class DocumentConverter:
         ]
 
         try:
-            # Suppress Java runtime warnings by redirecting stderr
-            subprocess.check_call(command, stderr=subprocess.DEVNULL)
+            subprocess.run(
+                command,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                text=True,
+                check=True,
+            )
             logger.info(f"Conversion successful: '{output_file}'")
         except subprocess.CalledProcessError as e:
+            output_parts = []
+            if e.stdout:
+                output_parts.append(f"stdout: {e.stdout.strip()}")
+            if e.stderr:
+                output_parts.append(f"stderr: {e.stderr.strip()}")
+            details = "\n".join(output_parts)
             error_msg = f"Error during conversion: {e}"
+            if details:
+                error_msg = f"{error_msg}\n{details}"
             logger.info(error_msg)
             raise ConversionError(error_msg, e)
 
@@ -253,4 +266,4 @@ class DocumentConverter:
     #     except Exception as e:
     #         error_msg = f"Error during PDF conversion: {str(e)}"
     #         logger.error(error_msg)
-    #         raise ConversionError(error_msg)
+    #         raise ConversionError(error_msg)

{polytext-0.2.3 → polytext-0.2.5}/polytext/loader/base.py

@@ -25,7 +25,7 @@ from ..loader import (
     XmlXbrlLoader,
     NotebookLoader
 )
-from ..exceptions import EmptyDocument, LoaderTimeoutError, LoaderError
+from ..exceptions import ConversionError, EmptyDocument, LoaderTimeoutError, LoaderError
 from ..utils.utils import clean_extracted_text_whitespace, remove_markdown_strip
 
 # External imports
@@ -46,6 +46,10 @@ LLM_OUTPUT_ERROR_CODES = {
     997: "REPETITIVE_OUTPUT",
     999: "MAX_TOKENS",
 }
+EMPTY_DOCUMENT_LOADER_ERROR_CODES = {
+    **LLM_OUTPUT_ERROR_CODES,
+    998: "NO_TEXT_DETECTED",
+}
 
 
 def _read_bool_env(name: str, default: bool = False) -> bool:
@@ -55,6 +59,32 @@ def _read_bool_env(name: str, default: bool = False) -> bool:
     return value.strip().lower() in {"1", "true", "yes", "y", "on"}
 
 
+def _capture_exception_for_sentry(error: Exception) -> None:
+    try:
+        import sentry_sdk
+    except ImportError:
+        return
+
+    try:
+        sentry_sdk.capture_exception(error)
+    except Exception:
+        return
+
+
+def _raise_empty_document_loader_error(error: EmptyDocument) -> None:
+    loader_error_code = EMPTY_DOCUMENT_LOADER_ERROR_CODES.get(error.code, "NO_TEXT_DETECTED")
+    message = error.message
+    if loader_error_code == "NO_TEXT_DETECTED":
+        message = "No text detected"
+    else:
+        _capture_exception_for_sentry(error)
+    raise LoaderError(
+        message=message,
+        status=422,
+        code=loader_error_code,
+    ) from error
+
+
 class BaseLoader:
     def __init__(self, markdown_output=True, llm_api_key=None, provider: str = "google", temp_dir: str = "temp",
                  ocr_model: str = "gpt-5-mini", timeout_minutes: int | None = None,
@@ -153,28 +183,24 @@ class BaseLoader:
         try:
             response = self.run_loader_class(loader_class=loader_class, input_list=input_list)
         except EmptyDocument as e:
-            logger.info(f"Empty document encountered: {e.message}")
             if e.code in LLM_OUTPUT_ERROR_CODES:
-                logger.exception(
-                    "Raising LoaderError: status=422 code=%s original_empty_document_code=%s message=%s",
-                    LLM_OUTPUT_ERROR_CODES[e.code],
-                    e.code,
-                    e.message,
-                )
-                raise LoaderError(
-                    message=e.message,
-                    status=422,
-                    code=LLM_OUTPUT_ERROR_CODES[e.code],
-                ) from e
+                _raise_empty_document_loader_error(e)
             if self.fallback_ocr:
                 loader_class = self.init_loader_class(input=first_file_url, storage_client=storage_client,
                                                       llm_api_key=self.llm_api_key, is_document_fallback=True, **kwargs)
-                response = self.run_loader_class(loader_class=loader_class, input_list=input_list)
+                try:
+                    response = self.run_loader_class(loader_class=loader_class, input_list=input_list)
+                except EmptyDocument as fallback_error:
+                    _raise_empty_document_loader_error(fallback_error)
             else:
-                response = {"text": "", "completion_tokens": 0, "prompt_tokens": 0, "output_list": [
-                    {"text": "", "completion_tokens": 0, "prompt_tokens": 0, "completion_model": "not provided",
-                     "completion_model_provider": "not provided", "text_chunks": "not provided", "type": "document",
-                     "input": first_file_url}]}
+                _raise_empty_document_loader_error(e)
+        except ConversionError as e:
+            _capture_exception_for_sentry(e)
+            raise LoaderError(
+                message=e.message,
+                status=422,
+                code="CONVERSION_ERROR",
+            ) from e
         except LoaderTimeoutError:
             raise LoaderError(message="timeout gemini", status=504, code="TIMEOUT")
         except (httpx.ReadTimeout,

{polytext-0.2.3 → polytext-0.2.5}/polytext.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: polytext
-Version: 0.2.3
+Version: 0.2.5
 Summary: Python utilities to simplify document files management
 Home-page: https://github.com/docsity/polytext
 Author: Matteo Senardi

{polytext-0.2.3 → polytext-0.2.5}/polytext.egg-info/SOURCES.txt

@@ -81,6 +81,7 @@ tests/test_notebook_loader.py
 tests/test_ocr_fallbacks.py
 tests/test_ocr_image_descriptions.py
 tests/test_pain_text.py
+tests/test_pdf_conversion_error.py
 tests/test_python_version_metadata.py
 tests/test_split_audio_with_llm.py
 tests/test_xml_xbrl_loader.py

{polytext-0.2.3 → polytext-0.2.5}/setup.py

@@ -51,7 +51,7 @@ def get_requirements(*requirements_file):
 
 setup(
     name='polytext',
-    version='0.2.3',
+    version='0.2.5',
     url='https://github.com/docsity/polytext',
     # download_url='https://github.com/pualien/py-polytext/archive/0.1.23.tar.gz',
     license='MIT',

polytext-0.2.5/tests/test_base_loader_error_mapping.py

@@ -0,0 +1,148 @@
+import unittest
+from unittest.mock import Mock, patch
+
+from polytext.exceptions import ConversionError, EmptyDocument, LoaderError
+from polytext.loader.base import BaseLoader
+
+
+class _FailingLoader:
+    def __init__(self, error):
+        self.error = error
+
+    def load(self, input_path):
+        raise self.error
+
+
+class _FakeBaseLoader(BaseLoader):
+    def __init__(self, error, **kwargs):
+        super().__init__(**kwargs)
+        self.error = error
+
+    def initiate_storage(self, input):
+        return {}
+
+    def init_loader_class(self, input, storage_client, llm_api_key, is_document_fallback=False, **kwargs):
+        return _FailingLoader(self.error)
+
+
+class _FallbackFailingBaseLoader(BaseLoader):
+    def __init__(self, initial_error, fallback_error, **kwargs):
+        super().__init__(**kwargs)
+        self.initial_error = initial_error
+        self.fallback_error = fallback_error
+
+    def initiate_storage(self, input):
+        return {}
+
+    def init_loader_class(self, input, storage_client, llm_api_key, is_document_fallback=False, **kwargs):
+        if is_document_fallback:
+            return _FailingLoader(self.fallback_error)
+        return _FailingLoader(self.initial_error)
+
+
+class TestBaseLoaderErrorMapping(unittest.TestCase):
+    def test_llm_output_empty_document_codes_are_raised_as_loader_errors(self):
+        cases = [
+            (995, "INVALID_ARGUMENT"),
+            (996, "RECITATION"),
+            (997, "REPETITIVE_OUTPUT"),
+            (999, "MAX_TOKENS"),
+        ]
+
+        for empty_document_code, expected_loader_code in cases:
+            with self.subTest(empty_document_code=empty_document_code):
+                loader = _FakeBaseLoader(
+                    EmptyDocument(
+                        message=f"diagnostic failure {empty_document_code}",
+                        code=empty_document_code,
+                    )
+                )
+
+                sentry_sdk = Mock()
+                with patch("polytext.loader.base.logger.info") as mock_info:
+                    with patch("polytext.loader.base.logger.exception") as mock_exception:
+                        with patch.dict("sys.modules", {"sentry_sdk": sentry_sdk}):
+                            with self.assertRaises(LoaderError) as error_context:
+                                loader.get_text(["dummy.txt"])
+
+                error = error_context.exception
+                self.assertEqual(error.status, 422)
+                self.assertEqual(error.code, expected_loader_code)
+                self.assertEqual(error.message, f"diagnostic failure {empty_document_code}")
+                mock_info.assert_not_called()
+                mock_exception.assert_not_called()
+                sentry_sdk.capture_exception.assert_called_once()
+                self.assertIs(sentry_sdk.capture_exception.call_args.args[0], error.__cause__)
+
+    def test_empty_or_too_short_documents_are_raised_as_loader_errors(self):
+        loader = _FakeBaseLoader(
+            EmptyDocument(
+                message="Document text with less than 400 characters",
+                code=998,
+            )
+        )
+
+        sentry_sdk = Mock()
+        with patch("polytext.loader.base.logger.info") as mock_info:
+            with patch("polytext.loader.base.logger.exception") as mock_exception:
+                with patch.dict("sys.modules", {"sentry_sdk": sentry_sdk}):
+                    with self.assertRaises(LoaderError) as error_context:
+                        loader.get_text(["empty.txt"])
+
+        error = error_context.exception
+        self.assertEqual(error.status, 422)
+        self.assertEqual(error.code, "NO_TEXT_DETECTED")
+        self.assertEqual(error.message, "No text detected")
+        mock_info.assert_not_called()
+        mock_exception.assert_not_called()
+        sentry_sdk.capture_exception.assert_not_called()
+
+    def test_empty_document_after_fallback_ocr_is_raised_as_loader_error(self):
+        loader = _FallbackFailingBaseLoader(
+            initial_error=EmptyDocument(
+                message="No text detected",
+                code=998,
+            ),
+            fallback_error=EmptyDocument(
+                message="No text extracted from OCR fallback",
+            ),
+            fallback_ocr=True,
+        )
+
+        sentry_sdk = Mock()
+        with patch("polytext.loader.base.logger.info") as mock_info:
+            with patch("polytext.loader.base.logger.exception") as mock_exception:
+                with patch.dict("sys.modules", {"sentry_sdk": sentry_sdk}):
+                    with self.assertRaises(LoaderError) as error_context:
+                        loader.get_text(["empty.pdf"])
+
+        error = error_context.exception
+        self.assertEqual(error.status, 422)
+        self.assertEqual(error.code, "NO_TEXT_DETECTED")
+        self.assertEqual(error.message, "No text detected")
+        mock_info.assert_not_called()
+        mock_exception.assert_not_called()
+        sentry_sdk.capture_exception.assert_not_called()
+
+    def test_conversion_error_is_raised_as_loader_error(self):
+        conversion_error = ConversionError("LibreOffice failed")
+        loader = _FakeBaseLoader(conversion_error)
+
+        sentry_sdk = Mock()
+        with patch("polytext.loader.base.logger.info") as mock_info:
+            with patch("polytext.loader.base.logger.exception") as mock_exception:
+                with patch.dict("sys.modules", {"sentry_sdk": sentry_sdk}):
+                    with self.assertRaises(LoaderError) as error_context:
+                        loader.get_text(["document.docx"])
+
+        error = error_context.exception
+        self.assertEqual(error.status, 422)
+        self.assertEqual(error.code, "CONVERSION_ERROR")
+        self.assertEqual(error.message, "LibreOffice failed")
+        mock_info.assert_not_called()
+        mock_exception.assert_not_called()
+        sentry_sdk.capture_exception.assert_called_once_with(conversion_error)
+
+
+if __name__ == "__main__":
+    unittest.main()

{polytext-0.2.3 → polytext-0.2.5}/tests/test_get_audio_transcript_from_gcs.py

@@ -33,7 +33,7 @@ def main():
     # Define document data
     file_path = "gcs://opit-da-test-ml-ai-store-bucket/learning_resources/course_id=406/module_id=2658/id=31427/8434.mp4"
 
-    local_file_path = "/Users/marcodelgiudice/Projects/polytext/audio_8_barbero_0_5_ore.m4a"
+    local_file_path = "/Users/marcodelgiudice/Downloads/mq0264a5-5a073da227de0ee462bd6de8731d586a1dcc635f.pdf"
 
     # Call get_text method
     start = time.time()

{polytext-0.2.3 → polytext-0.2.5}/tests/test_get_ocr_from_image.py

@@ -38,7 +38,7 @@ def main():
 
     # local_file_path = "/Users/marcodelgiudice/Projects/polytext/IMG_9695.jpg"
     # local_file_path = "/Users/marcodelgiudice/Projects/polytext/IMG_9701.jpg"
-    local_file_path = "/Users/marcodelgiudice/Projects/polytext/chimicaformula.png"
+    local_file_path = "/Users/marcodelgiudice/Projects/polytext/gm1.png"
 
     try:
         start = time.time()

polytext-0.2.5/tests/test_pdf_conversion_error.py

@@ -0,0 +1,43 @@
+import subprocess
+import tempfile
+import unittest
+from unittest.mock import patch
+
+from polytext.converter.pdf import DocumentConverter
+from polytext.exceptions import ConversionError
+
+
+class TestPdfConversionError(unittest.TestCase):
+    @patch.object(DocumentConverter, "check_libreoffice_installed", return_value=True)
+    @patch("polytext.converter.pdf.subprocess.run")
+    @patch("polytext.converter.pdf.subprocess.check_call")
+    def test_conversion_error_includes_libreoffice_output(
+        self,
+        mock_check_call,
+        mock_run,
+        _mock_check_libreoffice,
+    ):
+        libreoffice_error = subprocess.CalledProcessError(
+            returncode=1,
+            cmd=["libreoffice", "--convert-to", "pdf"],
+            output="convert input.docx -> output.pdf",
+            stderr="Unspecified Application Error",
+        )
+        mock_check_call.side_effect = libreoffice_error
+        mock_run.side_effect = libreoffice_error
+
+        with tempfile.NamedTemporaryFile(suffix=".docx") as input_file:
+            with tempfile.NamedTemporaryFile(suffix=".pdf") as output_file:
+                with self.assertRaises(ConversionError) as error_context:
+                    DocumentConverter().convert_to_pdf(
+                        input_file=input_file.name,
+                        original_file=input_file.name,
+                        output_file=output_file.name,
+                    )
+
+        self.assertIn("Unspecified Application Error", error_context.exception.message)
+        self.assertIn("convert input.docx -> output.pdf", error_context.exception.message)
+
+
+if __name__ == "__main__":
+    unittest.main()

{polytext-0.2.3 → polytext-0.2.5}/tests/test_youtube_transcript.py

@@ -32,9 +32,9 @@ url = 'https://www.youtube.com/watch?v=L4as3tks4Js'  # basement alberto angela
 
 # url = 'https://www.youtube.com/watch?v=UabBYexBD4k'  # INM RAG 11 minuti, completato in 26 secondi con successo con gemini-3.1-flash-lite
 
-url = 'https://www.youtube.com/watch?v=96jN2OCOfLs'  # Vibe coding 30 minuti, completato in 150 secondi con successo con gemini-3-flash-preview (160k token in input, 7k in output), 3.1-flash-lite ha raggiunto i max tokens in output (50k) probabile repetition
+#url = 'https://www.youtube.com/watch?v=96jN2OCOfLs'  # Vibe coding 30 minuti, completato in 150 secondi con successo con gemini-3-flash-preview (160k token in input, 7k in output), 3.1-flash-lite ha raggiunto i max tokens in output (50k) probabile repetition
 
-# url = 'https://www.youtube.com/watch?v=HGfsGvmRaaw'  # barbero2 50 minuti, fallito, RECITATION in tutti e 3 i modelli (275k token in input)
+url = 'https://www.youtube.com/watch?v=HGfsGvmRaaw'  # barbero2 50 minuti, fallito, RECITATION in tutti e 3 i modelli (275k token in input)
 
 # url = 'https://www.youtube.com/watch?v=CM2CkNU9xR0'  # google antigravity 27 minuti, completato in 39 secondi con successo con gemini-3.1-flash-lite (146k token in input, 6k token in output)
 

polytext-0.2.3/tests/test_base_loader_error_mapping.py

@@ -1,79 +0,0 @@
-import unittest
-from unittest.mock import patch
-
-from polytext.exceptions import EmptyDocument, LoaderError
-from polytext.loader.base import BaseLoader
-
-
-class _FailingLoader:
-    def __init__(self, error):
-        self.error = error
-
-    def load(self, input_path):
-        raise self.error
-
-
-class _FakeBaseLoader(BaseLoader):
-    def __init__(self, error, **kwargs):
-        super().__init__(**kwargs)
-        self.error = error
-
-    def initiate_storage(self, input):
-        return {}
-
-    def init_loader_class(self, input, storage_client, llm_api_key, is_document_fallback=False, **kwargs):
-        return _FailingLoader(self.error)
-
-
-class TestBaseLoaderErrorMapping(unittest.TestCase):
-    def test_llm_output_empty_document_codes_are_raised_as_loader_errors(self):
-        cases = [
-            (995, "INVALID_ARGUMENT"),
-            (996, "RECITATION"),
-            (997, "REPETITIVE_OUTPUT"),
-            (999, "MAX_TOKENS"),
-        ]
-
-        for empty_document_code, expected_loader_code in cases:
-            with self.subTest(empty_document_code=empty_document_code):
-                loader = _FakeBaseLoader(
-                    EmptyDocument(
-                        message=f"diagnostic failure {empty_document_code}",
-                        code=empty_document_code,
-                    )
-                )
-
-                with patch("polytext.loader.base.logger.exception") as mock_exception:
-                    with self.assertRaises(LoaderError) as error_context:
-                        loader.get_text(["dummy.txt"])
-
-                error = error_context.exception
-                self.assertEqual(error.status, 422)
-                self.assertEqual(error.code, expected_loader_code)
-                self.assertEqual(error.message, f"diagnostic failure {empty_document_code}")
-                mock_exception.assert_called_once()
-                self.assertIn("Raising LoaderError", mock_exception.call_args.args[0])
-                self.assertEqual(mock_exception.call_args.args[1], expected_loader_code)
-                self.assertEqual(mock_exception.call_args.args[2], empty_document_code)
-                self.assertEqual(mock_exception.call_args.args[3], f"diagnostic failure {empty_document_code}")
-
-    def test_empty_or_too_short_documents_still_return_empty_response(self):
-        loader = _FakeBaseLoader(
-            EmptyDocument(
-                message="Document text with less than 400 characters",
-                code=998,
-            )
-        )
-
-        with patch("polytext.loader.base.logger.exception") as mock_exception:
-            response = loader.get_text(["empty.txt"])
-
-        self.assertEqual(response["text"], "")
-        self.assertEqual(response["completion_tokens"], 0)
-        self.assertEqual(response["prompt_tokens"], 0)
-        self.assertEqual(response["output_list"][0]["input"], "empty.txt")
-        mock_exception.assert_not_called()
-
-
-if __name__ == "__main__":
-    unittest.main()