polytext 0.2.8b1__tar.gz → 0.2.8b3__tar.gz

{polytext-0.2.8b1 → polytext-0.2.8b3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: polytext
-Version: 0.2.8b1
+Version: 0.2.8b3
 Summary: Python utilities to simplify document files management
 Home-page: https://github.com/docsity/polytext
 Author: Matteo Senardi

{polytext-0.2.8b1 → polytext-0.2.8b3}/polytext/converter/audio_to_text.py

@@ -80,23 +80,30 @@ def add_line_break_after_each_sentence(text: str) -> str:
     if not text:
         return text
 
+    line_feed = chr(10)
     lines = text.splitlines()
     formatted_lines = []
 
     for line in lines:
         stripped_line = line.strip()
+
         if not stripped_line:
             formatted_lines.append("")
             continue
+
         if re.match(r"^#{1,6}\s+", stripped_line):
             formatted_lines.append(stripped_line)
             continue
 
         normalized_line = re.sub(r"\s+", " ", stripped_line)
-        normalized_line = re.sub(r"([.!?])\s+", r"\1\\n ", normalized_line)
+        normalized_line = re.sub(
+            r"([.!?])\s+",
+            lambda match: f"{match.group(1)}{line_feed} ",
+            normalized_line,
+        )
         formatted_lines.append(normalized_line)
 
-    return "\\n ".join(formatted_lines).strip()
+    return line_feed.join(formatted_lines).strip()
 
 
 def create_ascii_safe_upload_copy(audio_file: str) -> tuple[str, str | None]:
@@ -537,8 +544,6 @@ class AudioToTextConverter:
                 )
 
             response_text, marker_only = normalize_no_human_speech_marker(response_text)
-            if not marker_only:
-                response_text = self.format_audio_output_text(response_text)
 
             response_dict = {
                 "transcript": "" if marker_only else response_text,

{polytext-0.2.8b1 → polytext-0.2.8b3}/polytext/converter/document_ocr_to_text.py

@@ -13,6 +13,7 @@ from google.api_core import exceptions as google_exceptions
 from ..prompts.ocr import (
     OCR_TO_MARKDOWN_NON_LITERAL_FALLBACK_PROMPT,
     OCR_TO_MARKDOWN_PROMPT,
+    OCR_TO_PLAIN_TEXT_NON_LITERAL_FALLBACK_PROMPT,
     OCR_TO_PLAIN_TEXT_PROMPT,
     build_ocr_prompt,
 )
@@ -34,7 +35,7 @@ OCR_TAIL_REPETITION_THRESHOLD = float(os.getenv("OCR_TAIL_REPETITION_THRESHOLD",
 OCR_FALLBACK_SOURCE_PATTERN = os.getenv("OCR_FALLBACK_SOURCE_PATTERN", "flash-lite-preview")
 OCR_FALLBACK_MODEL = os.getenv("OCR_FALLBACK_MODEL", "gemini-3-flash-preview")
 OCR_FALLBACK_TEMPERATURE = float(os.getenv("OCR_FALLBACK_TEMPERATURE", "1.0"))
-OCR_FINAL_FALLBACK_MODEL = os.getenv("OCR_FINAL_FALLBACK_MODEL", "gemini-3.6-flash")
+OCR_FINAL_FALLBACK_MODEL = os.getenv("OCR_FINAL_FALLBACK_MODEL", "gemini-3.5-flash")
 OCR_PROMPT_VARIANT_DEFAULT = "default"
 OCR_PROMPT_VARIANT_NON_LITERAL_FALLBACK = "non_literal_fallback"
 OCR_RETRIABLE_OUTPUT_ERROR_CODES = (996, 997, 999)
@@ -222,6 +223,8 @@ class DocumentOCRToTextConverter:
     def _build_prompt_template(self) -> str:
         if self.markdown_output and self.prompt_variant == OCR_PROMPT_VARIANT_NON_LITERAL_FALLBACK:
             base_prompt = OCR_TO_MARKDOWN_NON_LITERAL_FALLBACK_PROMPT
+        elif not self.markdown_output and self.prompt_variant == OCR_PROMPT_VARIANT_NON_LITERAL_FALLBACK:
+            base_prompt = OCR_TO_PLAIN_TEXT_NON_LITERAL_FALLBACK_PROMPT
         elif self.markdown_output:
             base_prompt = OCR_TO_MARKDOWN_PROMPT
         else:
@@ -234,8 +237,6 @@ class DocumentOCRToTextConverter:
     def should_prompt_fallback_retry(self, error: EmptyDocument) -> bool:
         if self.fallback_stage != 0:
             return False
-        if not self.markdown_output:
-            return False
         if self.prompt_variant == OCR_PROMPT_VARIANT_NON_LITERAL_FALLBACK:
             return False
         return error.code in OCR_RETRIABLE_OUTPUT_ERROR_CODES
@@ -570,6 +571,7 @@ class DocumentOCRToTextConverter:
                         "completion_model_provider": self.ocr_model_provider,
                         "text_chunks": "not provided",
                         "page_error": True,
+                        "page_error_reason": error.message,
                     }
                 return page_num, ocr_result
 
@@ -603,11 +605,17 @@ class DocumentOCRToTextConverter:
             all_text = []
             total_completion_tokens = 0
             total_prompt_tokens = 0
+            failed_pages = []
 
-            for _, ocr_result in results:
+            for page_num, ocr_result in results:
                 all_text.append(f"{ocr_result['text']}\n")
                 total_completion_tokens += ocr_result['completion_tokens']
                 total_prompt_tokens += ocr_result['prompt_tokens']
+                if ocr_result.get("page_error"):
+                    failed_pages.append({
+                        "page": page_num + 1,
+                        "reason": ocr_result.get("page_error_reason", "unknown"),
+                    })
 
             pdf.close()
 
@@ -618,6 +626,8 @@ class DocumentOCRToTextConverter:
                 "completion_model": self.ocr_model,
                 "completion_model_provider": self.ocr_model_provider,
                 "text_chunks": "not provided",
+                "ocr_failed_pages": [item["page"] for item in failed_pages],
+                "ocr_failed_pages_detail": failed_pages,
             }
 
             return final_result_dict

{polytext-0.2.8b1 → polytext-0.2.8b3}/polytext/converter/ocr_to_text.py

@@ -13,6 +13,7 @@ from google.api_core import exceptions as google_exceptions
 from ..prompts.ocr import (
     OCR_TO_MARKDOWN_NON_LITERAL_FALLBACK_PROMPT,
     OCR_TO_MARKDOWN_PROMPT,
+    OCR_TO_PLAIN_TEXT_NON_LITERAL_FALLBACK_PROMPT,
     OCR_TO_PLAIN_TEXT_PROMPT,
     build_ocr_prompt,
 )
@@ -207,6 +208,8 @@ class OCRToTextConverter:
     def _build_prompt_template(self) -> str:
         if self.markdown_output and self.prompt_variant == OCR_PROMPT_VARIANT_NON_LITERAL_FALLBACK:
             base_prompt = OCR_TO_MARKDOWN_NON_LITERAL_FALLBACK_PROMPT
+        elif not self.markdown_output and self.prompt_variant == OCR_PROMPT_VARIANT_NON_LITERAL_FALLBACK:
+            base_prompt = OCR_TO_PLAIN_TEXT_NON_LITERAL_FALLBACK_PROMPT
         elif self.markdown_output:
             base_prompt = OCR_TO_MARKDOWN_PROMPT
         else:
@@ -219,8 +222,6 @@ class OCRToTextConverter:
     def should_prompt_fallback_retry(self, error: EmptyDocument) -> bool:
         if self.fallback_stage != 0:
             return False
-        if not self.markdown_output:
-            return False
         if self.prompt_variant == OCR_PROMPT_VARIANT_NON_LITERAL_FALLBACK:
             return False
         return error.code in OCR_RETRIABLE_OUTPUT_ERROR_CODES

{polytext-0.2.8b1 → polytext-0.2.8b3}/polytext/loader/base.py

@@ -416,8 +416,7 @@ class BaseLoader:
             if path_without_query:
                 _, file_extension = os.path.splitext(path_without_query)
         else:  # If is local file path (without schema)
-            if os.path.exists(input):
-                _, file_extension = os.path.splitext(input)
+            _, file_extension = os.path.splitext(input)
 
         if file_extension:
             file_extension = file_extension.lower()
@@ -447,9 +446,11 @@ class BaseLoader:
             )
         elif mime_type:
             if file_extension in [".pdf", ".xlsx", ".docx", ".txt", ".csv", ".odt", ".pptx", ".xls", ".doc", ".ppt", ".rtf"]:
+                document_kwargs = {k: v for k, v in kwargs.items() if k != "source"}
                 if self.force_ocr:
 
                     return DocumentOCRLoader(
+                        source=self.source,
                         llm_api_key=llm_api_key,
                         markdown_output=self.markdown_output,
                         temp_dir=self.temp_dir,
@@ -458,10 +459,16 @@ class BaseLoader:
                         ocr_model=self.ocr_model,
                         include_image_descriptions=self.include_image_descriptions,
                         allow_partial_ocr_failures=True,
-                        **kwargs,
+                        **document_kwargs,
                     )
 
-                return DocumentLoader(markdown_output=self.markdown_output, temp_dir=self.temp_dir, timeout_minutes=self.timeout_minutes, **kwargs)
+                return DocumentLoader(
+                    source=self.source,
+                    markdown_output=self.markdown_output,
+                    temp_dir=self.temp_dir,
+                    timeout_minutes=self.timeout_minutes,
+                    **document_kwargs,
+                )
             elif mime_type.startswith("audio/"):
                 audio_kwargs = {**kwargs, "is_output_audio_raw": self.is_output_audio_raw}
                 return AudioLoader(llm_api_key=llm_api_key, markdown_output=self.markdown_output, temp_dir=self.temp_dir, timeout_minutes=self.timeout_minutes, **audio_kwargs)
@@ -590,14 +597,18 @@ class BaseLoader:
 
         result_dict["text"] = clean_extracted_text_whitespace(remove_markdown_strip(result_dict["text"]))
 
-        result_dict = {
+        final_result = {
             "text": result_dict["text"],
             "completion_tokens": result_dict["completion_tokens"],
             "prompt_tokens": result_dict["prompt_tokens"],
             "output_list": [result_dict],
         }
 
-        return result_dict
+        for metadata_key in ("ocr_failed_pages", "ocr_failed_pages_detail"):
+            if metadata_key in result_dict:
+                final_result[metadata_key] = result_dict[metadata_key]
+
+        return final_result
 
 # Helper methods
     @staticmethod

{polytext-0.2.8b1 → polytext-0.2.8b3}/polytext/prompts/ocr.py

@@ -28,6 +28,15 @@ Maintain paragraph breaks and formatting.
 Your output must be a plain text.
 """
 
+OCR_TO_PLAIN_TEXT_NON_LITERAL_FALLBACK_PROMPT = """
+Rephrase and reorganize the text content in a coherent plain text structure.
+Do not transcribe the text verbatim, but preserve the meaning of the original content without omitting anything, even apparently minor details.
+Pay special attention to tables, columns, headers, and any structured content.
+Maintain paragraph breaks and formatting.
+Your output must be a plain text.
+In case no readable text is present, write exactly "no readable text present".
+"""
+
 OCR_IMAGE_DESCRIPTION_INSTRUCTIONS = """
 Image description instructions:
 

{polytext-0.2.8b1 → polytext-0.2.8b3}/polytext.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: polytext
-Version: 0.2.8b1
+Version: 0.2.8b3
 Summary: Python utilities to simplify document files management
 Home-page: https://github.com/docsity/polytext
 Author: Matteo Senardi

{polytext-0.2.8b1 → polytext-0.2.8b3}/setup.py

@@ -51,7 +51,7 @@ def get_requirements(*requirements_file):
 
 setup(
     name='polytext',
-    version='0.2.8b1',
+    version='0.2.8b3',
     url='https://github.com/docsity/polytext',
     # download_url='https://github.com/pualien/py-polytext/archive/0.1.23.tar.gz',
     license='MIT',