polytext 0.2.8__tar.gz → 0.2.8b1__tar.gz

{polytext-0.2.8 → polytext-0.2.8b1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: polytext
-Version: 0.2.8
+Version: 0.2.8b1
 Summary: Python utilities to simplify document files management
 Home-page: https://github.com/docsity/polytext
 Author: Matteo Senardi

{polytext-0.2.8 → polytext-0.2.8b1}/polytext/converter/audio_to_text.py

@@ -93,10 +93,10 @@ def add_line_break_after_each_sentence(text: str) -> str:
             continue
 
         normalized_line = re.sub(r"\s+", " ", stripped_line)
-        normalized_line = re.sub(r"([.!?])\s+", r"\1\n", normalized_line)
+        normalized_line = re.sub(r"([.!?])\s+", r"\1\\n ", normalized_line)
         formatted_lines.append(normalized_line)
 
-    return "\n".join(formatted_lines).strip()
+    return "\\n ".join(formatted_lines).strip()
 
 
 def create_ascii_safe_upload_copy(audio_file: str) -> tuple[str, str | None]:

{polytext-0.2.8 → polytext-0.2.8b1}/polytext/converter/beautiful_text.py

@@ -156,7 +156,7 @@ class BeautifulTextConverter:
         finalize_nodes()
         return chapters
 
-    def convert(self, raw_text: str, save_transcript_chunks: bool = False, active_chapters: bool = False) -> dict:
+    def convert(self, raw_text: str, save_transcript_chunks: bool = False, active_chapters: bool = True) -> dict:
         cleaned_input = (raw_text or "").strip()
         if not cleaned_input:
             result = {
@@ -204,6 +204,6 @@ class BeautifulTextConverter:
             "text_chunks": cleaned_chunks if save_transcript_chunks else "not provided",
         }
         if active_chapters:
-            result["markdown_json"] = self._convert_markdown_to_json(final_text)
+            # result["markdown_json"] = self._convert_markdown_to_json(final_text)
             result["chapters"] = self._build_chapters(final_text)
         return result

{polytext-0.2.8 → polytext-0.2.8b1}/polytext/converter/document_ocr_to_text.py

@@ -34,7 +34,7 @@ OCR_TAIL_REPETITION_THRESHOLD = float(os.getenv("OCR_TAIL_REPETITION_THRESHOLD",
 OCR_FALLBACK_SOURCE_PATTERN = os.getenv("OCR_FALLBACK_SOURCE_PATTERN", "flash-lite-preview")
 OCR_FALLBACK_MODEL = os.getenv("OCR_FALLBACK_MODEL", "gemini-3-flash-preview")
 OCR_FALLBACK_TEMPERATURE = float(os.getenv("OCR_FALLBACK_TEMPERATURE", "1.0"))
-OCR_FINAL_FALLBACK_MODEL = os.getenv("OCR_FINAL_FALLBACK_MODEL", "gemini-2.0-flash")
+OCR_FINAL_FALLBACK_MODEL = os.getenv("OCR_FINAL_FALLBACK_MODEL", "gemini-3.6-flash")
 OCR_PROMPT_VARIANT_DEFAULT = "default"
 OCR_PROMPT_VARIANT_NON_LITERAL_FALLBACK = "non_literal_fallback"
 OCR_RETRIABLE_OUTPUT_ERROR_CODES = (996, 997, 999)
@@ -113,6 +113,7 @@ def get_document_ocr(
     ocr_model: str | None = None,
     max_output_tokens: int | None = None,
     include_image_descriptions: bool = False,
+    allow_partial_ocr_failures: bool = False,
 ):
     """
     Convenience function to extract text from an image file using OCR, optionally formatted as Markdown.
@@ -136,6 +137,9 @@ def get_document_ocr(
         include_image_descriptions (bool, optional): If True, OCR prompts include
             brief functional descriptions for meaningful non-text images.
             Defaults to False.
+        allow_partial_ocr_failures (bool, optional): If True, pages that still
+            fail OCR after all retries are recorded inline instead of aborting
+            the whole document extraction. Defaults to False.
 
     Returns:
         dict: Dictionary containing the OCR results and metadata.
@@ -149,6 +153,7 @@ def get_document_ocr(
         timeout_minutes=timeout_minutes,
         max_output_tokens=max_output_tokens,
         include_image_descriptions=include_image_descriptions,
+        allow_partial_ocr_failures=allow_partial_ocr_failures,
     )
     return converter.get_document_ocr(document_for_ocr)
 
@@ -157,7 +162,8 @@ class DocumentOCRToTextConverter:
                 markdown_output=True, llm_api_key=None, target_size=1, temp_dir="temp",
                  page_range=None, timeout_minutes: int = None, fallback_stage: int = 0,
                  max_output_tokens: int | None = None, include_image_descriptions: bool = False,
-                 prompt_variant: str = OCR_PROMPT_VARIANT_DEFAULT):
+                 prompt_variant: str = OCR_PROMPT_VARIANT_DEFAULT,
+                 allow_partial_ocr_failures: bool = False):
         """
         Initialize the DocumentOCRToTextConverter class with specified OCR model and formatting options.
 
@@ -182,6 +188,9 @@ class DocumentOCRToTextConverter:
                 Defaults to False.
             prompt_variant (str, optional): Prompt variant used by this attempt.
                 Defaults to "default".
+            allow_partial_ocr_failures (bool, optional): If True, pages that still
+                fail OCR after all retries are recorded inline instead of aborting
+                the whole document extraction. Defaults to False.
 
         Raises:
             OSError: If temp directory creation fails
@@ -196,6 +205,7 @@ class DocumentOCRToTextConverter:
         self.timeout_minutes = timeout_minutes
         self.include_image_descriptions = include_image_descriptions
         self.prompt_variant = prompt_variant
+        self.allow_partial_ocr_failures = allow_partial_ocr_failures
         requested_output_tokens = OCR_MAX_OUTPUT_TOKENS if max_output_tokens is None else max_output_tokens
         self.max_output_tokens = max(requested_output_tokens, OCR_MIN_OUTPUT_TOKENS)
         self.fallback_stage = fallback_stage
@@ -283,6 +293,7 @@ class DocumentOCRToTextConverter:
             max_output_tokens=self.max_output_tokens,
             include_image_descriptions=self.include_image_descriptions,
             prompt_variant=resolved_prompt_variant,
+            allow_partial_ocr_failures=self.allow_partial_ocr_failures,
         )
         result = fallback_converter.get_ocr(
             file_for_ocr=file_for_ocr,
@@ -541,7 +552,25 @@ class DocumentOCRToTextConverter:
                 pix.save(temp_image_path)
 
                 # Perform OCR on the page
-                ocr_result = self.get_ocr(temp_image_path)
+                try:
+                    ocr_result = self.get_ocr(temp_image_path)
+                except EmptyDocument as error:
+                    if not self.allow_partial_ocr_failures:
+                        raise
+                    logger.warning(
+                        "Document OCR failed on page %s after retries; keeping partial document because allow_partial_ocr_failures=True: %s",
+                        page_num + 1,
+                        error.message,
+                    )
+                    ocr_result = {
+                        "text": "",
+                        "completion_tokens": 0,
+                        "prompt_tokens": 0,
+                        "completion_model": self.ocr_model,
+                        "completion_model_provider": self.ocr_model_provider,
+                        "text_chunks": "not provided",
+                        "page_error": True,
+                    }
                 return page_num, ocr_result
 
             finally:

{polytext-0.2.8 → polytext-0.2.8b1}/polytext/loader/base.py

@@ -90,7 +90,9 @@ def _raise_empty_document_loader_error(error: EmptyDocument) -> None:
 class BaseLoader:
     def __init__(self, markdown_output=True, llm_api_key=None, provider: str = "google", temp_dir: str = "temp",
                  ocr_model: str = "gpt-5-mini", timeout_minutes: int | None = None,
-                 include_image_descriptions: bool | None = None, **kwargs):
+                 include_image_descriptions: bool | None = None,
+                 force_ocr: bool = False,
+                 **kwargs):
         """
         Initialize the BaseLoader with cloud storage and LLM configurations.
 
@@ -108,6 +110,8 @@ class BaseLoader:
             include_image_descriptions (bool | None, optional): If True, OCR prompts
                 include brief functional descriptions for meaningful non-text images.
                 If None, defaults from OCR_INCLUDE_IMAGE_DESCRIPTIONS. Defaults to None.
+            force_ocr (bool, optional): If True, supported document files are routed to
+                OCRLoader instead of the standard DocumentLoader. Defaults to False.
              **kwargs: Additional keyword arguments to pass to the underlying loader or extraction logic.
                 - target_size (int, optional): Target file size in bytes. Defaults to 1MB
                 - source (str): Source of the document. Must be either "cloud" or "local"
@@ -131,6 +135,7 @@ class BaseLoader:
             if include_image_descriptions is None
             else include_image_descriptions
         )
+        self.force_ocr = force_ocr
         self.kwargs = kwargs
         self.target_size = kwargs.get("target_size", 1)
         self.source = kwargs.get("source", "cloud")
@@ -259,8 +264,14 @@ class BaseLoader:
         cleanup_result = converter.convert(
             raw_text=raw_result["text"],
             save_transcript_chunks=kwargs.get("save_transcript_chunks", self.save_transcript_chunks),
-            active_chapters=kwargs.get("active_chapters", False),
+            active_chapters=kwargs.get("active_chapters", True),
         )
+        if not cleanup_result.get("chapters"):
+            raise LoaderError(
+                message="No chapters detected",
+                status=422,
+                code="NO_CHAPTERS_DETECTED",
+            )
 
         total_completion_tokens = raw_result.get("completion_tokens", 0) + cleanup_result.get("completion_tokens", 0)
         total_prompt_tokens = raw_result.get("prompt_tokens", 0) + cleanup_result.get("prompt_tokens", 0)
@@ -436,6 +447,20 @@ class BaseLoader:
             )
         elif mime_type:
             if file_extension in [".pdf", ".xlsx", ".docx", ".txt", ".csv", ".odt", ".pptx", ".xls", ".doc", ".ppt", ".rtf"]:
+                if self.force_ocr:
+
+                    return DocumentOCRLoader(
+                        llm_api_key=llm_api_key,
+                        markdown_output=self.markdown_output,
+                        temp_dir=self.temp_dir,
+                        timeout_minutes=self.timeout_minutes,
+                        ocr_provider=self.provider,
+                        ocr_model=self.ocr_model,
+                        include_image_descriptions=self.include_image_descriptions,
+                        allow_partial_ocr_failures=True,
+                        **kwargs,
+                    )
+
                 return DocumentLoader(markdown_output=self.markdown_output, temp_dir=self.temp_dir, timeout_minutes=self.timeout_minutes, **kwargs)
             elif mime_type.startswith("audio/"):
                 audio_kwargs = {**kwargs, "is_output_audio_raw": self.is_output_audio_raw}

{polytext-0.2.8 → polytext-0.2.8b1}/polytext/loader/document_ocr.py

@@ -46,6 +46,7 @@ class DocumentOCRLoader:
                  ocr_provider: str = "google",
                  ocr_model: str | None = None,
                  include_image_descriptions: bool = False,
+                 allow_partial_ocr_failures: bool = False,
                  **kwargs
                  ):
         """
@@ -82,6 +83,9 @@ class DocumentOCRLoader:
             include_image_descriptions (bool, optional): If True, OCR prompts include
                 brief functional descriptions for meaningful non-text images.
                 Defaults to False.
+            allow_partial_ocr_failures (bool, optional): If True, pages that still
+                fail OCR after all retries are recorded inline instead of aborting
+                the whole document extraction. Defaults to False.
             **kwargs:
                 max_output_tokens (int, optional): Maximum Gemini output tokens for
                     Google document OCR generation.
@@ -105,6 +109,7 @@ class DocumentOCRLoader:
         self.ocr_provider = (ocr_provider or "google").lower()
         self.ocr_model = ocr_model
         self.include_image_descriptions = include_image_descriptions
+        self.allow_partial_ocr_failures = allow_partial_ocr_failures
         self.max_output_tokens = kwargs.get("max_output_tokens")
 
         # Set up custom temp directory
@@ -254,6 +259,7 @@ class DocumentOCRLoader:
                 timeout_minutes=self.timeout_minutes,
                 ocr_model=self.ocr_model or None,
                 include_image_descriptions=self.include_image_descriptions,
+                allow_partial_ocr_failures=self.allow_partial_ocr_failures,
             )
         else:
             result_dict = ocr_fn(
@@ -270,6 +276,7 @@ class DocumentOCRLoader:
                 ),
                 max_output_tokens=self.max_output_tokens,
                 include_image_descriptions=self.include_image_descriptions,
+                allow_partial_ocr_failures=self.allow_partial_ocr_failures,
             )
 
         result_dict["type"] = self.type

polytext-0.2.8b1/polytext/prompts/beautiful_text.py

@@ -0,0 +1,61 @@
+BEAUTIFUL_TEXT_PROMPT = """
+You are cleaning a spoken transcript into faithful Markdown.
+
+This is NOT summarization.
+This is NOT rewriting.
+This is NOT editorial adaptation.
+This is a cleaned transcript.
+
+Your task is to remove only clearly meaningless noise while preserving the original spoken wording,
+sequence, tone, rhythm, and reasoning as closely as possible.
+
+REMOVE ONLY:
+- obvious filler sounds such as "eh", "uhm", "mmh" when they clearly add no meaning
+- accidental duplicated words such as "di di", "che che", "da da"
+- clearly aborted false starts that add no meaning
+- irrelevant overlap fragments between speakers
+
+IF THERE IS ANY DOUBT, KEEP THE ORIGINAL WORDING.
+
+PRESERVE STRICTLY:
+- the original wording
+- the original order of ideas
+- the original paragraph flow
+- the original tone and conversational style
+- repetitions that still carry emphasis, rhythm, hesitation, or meaning
+- colloquial phrasing and spoken transitions when meaningful
+
+DO NOT:
+- summarize
+- compress
+- simplify
+- polish into formal written prose
+- merge multiple spoken sentences into a shorter reformulation
+- turn the transcript into an article, essay, report, or explanatory text
+- replace words with better synonyms
+- add explanations, transitions, or inferred content
+- add interpretive conclusions
+
+FORMATTING:
+- output Markdown only
+- preserve the transcript as a cleaned spoken transcript, not as a rewritten article
+- use paragraphs, but do not heavily reorganize the flow
+- headings may be generated editorially, but only to label the topic of the following block
+- headings must be short, neutral, and strictly supported by the text below
+- the final Markdown must contain at least one heading
+- if the text is short or has weak structure, add one minimal heading only
+- do not convert prose into bullet lists unless the speaker is explicitly enumerating points
+- use emphasis sparingly
+- you may add light Markdown emphasis to improve scanability, but only locally and without rewriting the sentence
+- use **bold** for clearly salient entities already present in the source, such as names, products, platforms, institutions, laws, or central concepts
+- use *italics* sparingly for contextual labels, foreign expressions, or technical terms when this remains clearly faithful to the source
+- do not apply emphasis to large portions of text
+- do not use emphasis as a substitute for rewriting, summarizing, or restructuring
+- do not add emphasis just to make the text nicer
+- do not add code fences or commentary
+
+FINAL CHECK:
+- every output sentence must remain closely traceable to the input
+- prefer awkward fidelity over elegant rewriting
+- headings may be editorially generated, but body text must remain maximally faithful
+"""

{polytext-0.2.8 → polytext-0.2.8b1}/polytext.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: polytext
-Version: 0.2.8
+Version: 0.2.8b1
 Summary: Python utilities to simplify document files management
 Home-page: https://github.com/docsity/polytext
 Author: Matteo Senardi

{polytext-0.2.8 → polytext-0.2.8b1}/setup.py

@@ -51,7 +51,7 @@ def get_requirements(*requirements_file):
 
 setup(
     name='polytext',
-    version='0.2.8',
+    version='0.2.8b1',
     url='https://github.com/docsity/polytext',
     # download_url='https://github.com/pualien/py-polytext/archive/0.1.23.tar.gz',
     license='MIT',

{polytext-0.2.8 → polytext-0.2.8b1}/tests/test_audio_transcription_model_migration.py

@@ -137,7 +137,7 @@ class TestAudioTranscriptionModelMigration(unittest.TestCase):
 
         self.assertEqual(
             formatted,
-            "Prima frase.\nSeconda frase?\nTerza frase!\n## Titolo\nQuarta frase.\nQuinta frase.",
+            "Prima frase.\\n Seconda frase?\\n Terza frase!\\n ## Titolo\\n Quarta frase.\\n Quinta frase.",
         )
 
     def test_normalize_no_human_speech_marker_returns_empty_for_marker_only(self):

{polytext-0.2.8 → polytext-0.2.8b1}/tests/test_base_loader_error_mapping.py

@@ -1,4 +1,5 @@
 import unittest
+from types import SimpleNamespace
 from unittest.mock import Mock, patch
 
 from polytext.exceptions import ConversionError, EmptyDocument, LoaderError
@@ -40,6 +41,24 @@ class _FallbackFailingBaseLoader(BaseLoader):
         return _FailingLoader(self.initial_error)
 
 
+class _BeautifulTextLoader(BaseLoader):
+    def __init__(self, raw_result=None, **kwargs):
+        super().__init__(**kwargs)
+        self.raw_result = raw_result or {
+            "text": "raw text",
+            "completion_tokens": 0,
+            "prompt_tokens": 0,
+            "completion_model": "not provided",
+            "completion_model_provider": "not provided",
+            "text_chunks": "not provided",
+            "type": "text",
+            "input": "dummy input",
+        }
+
+    def extract_raw_text_for_beautiful_text(self, input_value: str, **kwargs) -> dict:
+        return self.raw_result
+
+
 class TestBaseLoaderErrorMapping(unittest.TestCase):
     def test_llm_output_empty_document_codes_are_raised_as_loader_errors(self):
         cases = [
@@ -143,6 +162,34 @@ class TestBaseLoaderErrorMapping(unittest.TestCase):
         mock_exception.assert_not_called()
         sentry_sdk.capture_exception.assert_called_once_with(conversion_error)
 
+    def test_beautiful_text_without_detected_chapters_is_raised_as_loader_error(self):
+        loader = _BeautifulTextLoader()
+        cleanup_result = {
+            "text": "Short text without headings.",
+            "completion_tokens": 1,
+            "prompt_tokens": 1,
+            "completion_model": "gemini-3.1-flash-lite",
+            "completion_model_provider": "google",
+            "text_chunks": "not provided",
+            "markdown_json": {"root": ["Short text without headings."]},
+            "chapters": [],
+        }
+
+        fake_converter_class = Mock()
+        fake_converter_class.return_value.convert.return_value = cleanup_result
+
+        with patch.dict(
+            "sys.modules",
+            {"polytext.converter.beautiful_text": SimpleNamespace(BeautifulTextConverter=fake_converter_class)},
+        ):
+            with self.assertRaises(LoaderError) as error_context:
+                loader.get_beautiful_text(["dummy input"], active_chapters=True)
+
+        error = error_context.exception
+        self.assertEqual(error.status, 422)
+        self.assertEqual(error.code, "NO_CHAPTERS_DETECTED")
+        self.assertEqual(error.message, "No chapters detected")
+
 
 if __name__ == "__main__":
     unittest.main()

polytext-0.2.8/polytext/prompts/beautiful_text.py

@@ -1,43 +0,0 @@
-BEAUTIFUL_TEXT_PROMPT = """
-You are an editor specialized in cleaning spoken transcripts and raw text into faithful Markdown.
-This is not summarization. This is not rewriting. This is a cleaned transcript or cleaned source text.
-
-Your task is to remove only accidental noise while preserving the speaker's or author's original words,
-phrasing, reasoning, tone, and sequence of ideas as faithfully as possible.
-
-REMOVE ONLY:
-- non-meaningful fillers such as "eh", "uhm", "diciamo", "eccetera eccetera", "no?" when used only as filler
-- redundant "quindi", "appunto", "comunque" when they are only conversational padding
-- accidental repeated words such as "di di", "da da", "che che"
-- false starts and self-corrections only when they do not carry meaning
-- irrelevant overlap fragments between speakers
-
-PRESERVE COMPLETELY:
-- the original wording and sentence structure, even if colloquial
-- technical terms and proper nouns exactly
-- the original tone and register
-- reasoning, opinions, nuances, and meaningful uncertainty
-- the logical order of the discussion
-
-DO NOT:
-- rewrite sentences in a more elegant style
-- replace words with synonyms
-- summarize, compress, or simplify concepts
-- add explanations, transitions, or missing content
-- correct the speaker's opinions or inaccuracies
-- make the language more formal than the original
-
-FORMATTING:
-- output Markdown only
-- use paragraphs to separate thematic blocks
-- add headings only when the speaker explicitly introduces a new topic
-- use bullet lists or numbered lists only when the source explicitly enumerates items or when the sequence is clearly list-shaped
-- use emphasis sparingly and only when grounded in the original text
-- use **bold** for key information and important concepts, and *italics* for subtle emphasis or contextual terms in every chapter and paragraph whenever they improve readability and understanding
-- do not add code fences
-- do not add introductions or commentary
-
-FINAL CHECK:
-- every sentence in the output must be traceable to an equivalent sentence in the input
-- if a sentence cannot be grounded in the input, remove it
-"""