polytext 0.2.8b3__tar.gz → 0.2.8b5__tar.gz

{polytext-0.2.8b3 → polytext-0.2.8b5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: polytext
-Version: 0.2.8b3
+Version: 0.2.8b5
 Summary: Python utilities to simplify document files management
 Home-page: https://github.com/docsity/polytext
 Author: Matteo Senardi
@@ -25,7 +25,7 @@ Requires-Dist: markdown-to-json==2.1.2
 Requires-Dist: python-docx==1.1.2
 Requires-Dist: google-api-core>=2.24.2
 Requires-Dist: google-cloud-storage<3.0.0,>=2.17
-Requires-Dist: google-genai>=1.16.1
+Requires-Dist: google-genai==2.22.0
 Requires-Dist: openai==2.26.0
 Requires-Dist: boto3>=1.42.64
 Requires-Dist: botocore>=1.42.64

{polytext-0.2.8b3 → polytext-0.2.8b5}/polytext/converter/audio_to_text.py

@@ -14,6 +14,7 @@ from google.genai import types
 from google.genai import errors as genai_errors
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from google.api_core import exceptions as google_exceptions
+from pydub import AudioSegment
 
 from ..exceptions import EmptyDocument
 from ..prompts.transcription import (
@@ -25,13 +26,22 @@ from ..prompts.transcription import (
 )
 from ..processor.audio_chunker import AudioChunker
 from ..processor.text_merger import TextMerger
-from .gemini_quality_guards import extract_finish_reason, tail_has_excessive_repetition
+from .gemini_quality_guards import (
+    extract_finish_reason,
+    has_excessive_consecutive_word_repetition,
+    tail_has_excessive_repetition,
+)
 
 logger = logging.getLogger(__name__)
 
 SUPPORTED_MIME_TYPES = {
     'audio/x-aac', 'audio/flac', 'audio/mp3', 'audio/m4a', 'audio/mpeg',
-    'audio/mpga', 'audio/mp4', 'audio/opus', 'audio/pcm', 'audio/wav', 'audio/webm'
+    'audio/mpga', 'audio/mp4', 'audio/opus', 'audio/pcm', 'audio/wav', 'audio/x-wav', 'audio/webm'
+}
+
+GEMINI_AUDIO_MIME_ALIASES = {
+    'audio/x-aac': 'audio/aac',
+    'audio/x-wav': 'audio/wav',
 }
 
 INJECTION_GUARD_SYSTEM_INSTRUCTION = (
@@ -48,16 +58,20 @@ INJECTION_GUARD_SYSTEM_INSTRUCTION = (
 )
 
 AUDIO_MIN_OUTPUT_TOKENS = 500
+AUDIO_DEFAULT_MAX_OUTPUT_TOKENS = 4096
+AUDIO_MAX_ADAPTIVE_SPLIT_DEPTH = 1
+AUDIO_ADAPTIVE_SPLIT_OVERLAP_MS = 2000
+AUDIO_LONG_DURATION_THRESHOLD_MS = 80 * 60 * 1000
 AUDIO_TAIL_REPETITION_LINES = int(os.getenv("AUDIO_TAIL_REPETITION_LINES", "200"))
 AUDIO_TAIL_REPETITION_THRESHOLD = float(os.getenv("AUDIO_TAIL_REPETITION_THRESHOLD", "0.35"))
 AUDIO_FALLBACK_SOURCE_PATTERN = os.getenv("AUDIO_FALLBACK_SOURCE_PATTERN", "flash-lite")
-AUDIO_FALLBACK_MODEL = os.getenv("AUDIO_FALLBACK_MODEL", "gemini-3-flash-preview")
+AUDIO_FALLBACK_MODEL = os.getenv("AUDIO_FALLBACK_MODEL", "gemini-3.5-flash-lite")
 AUDIO_FALLBACK_TEMPERATURE = float(os.getenv("AUDIO_FALLBACK_TEMPERATURE", "1.0"))
 AUDIO_FINAL_FALLBACK_MODEL = os.getenv("AUDIO_FINAL_FALLBACK_MODEL", "gemini-3.5-flash")
 AUDIO_FILE_UPLOAD_THRESHOLD_BYTES = 20 * 1024 * 1024
 AUDIO_PROMPT_VARIANT_DEFAULT = "default"
 AUDIO_PROMPT_VARIANT_NON_LITERAL_FALLBACK = "non_literal_fallback"
-AUDIO_RETRIABLE_OUTPUT_ERROR_CODES = (996, 997, 999)
+AUDIO_RETRIABLE_OUTPUT_ERROR_CODES = (996, 997, 998, 999)
 NO_HUMAN_SPEECH_MARKER = "no human speech detected"
 
 
@@ -123,33 +137,32 @@ def create_ascii_safe_upload_copy(audio_file: str) -> tuple[str, str | None]:
 
 def compress_and_convert_audio(input_path: str, bitrate_quality: int = 9) -> str:
     """
-    Compress and convert an audio file to MP3 using ffmpeg.
+    Normalize an audio file to lossless 16 kHz mono WAV using ffmpeg.
 
     Args:
         input_path (str): Path to the original audio file
-        bitrate_quality (int, optional): Variable bitrate quality from 0-9 (9 being lowest). Defaults to 9
+        bitrate_quality (int, optional): Retained for backward compatibility; WAV output is lossless.
 
     Returns:
-        str: Path to the temporary compressed/converted MP3 file
+        str: Path to the temporary normalized WAV file
 
     Raises:
         RuntimeError: If FFmpeg compression/conversion fails
 
     Notes:
-        - Creates a temporary MP3 file that should be deleted after use
-        - Converts audio to mono and 16kHz sample rate for smaller file size
+        - Creates a temporary WAV file that should be deleted after use
+        - Converts audio to 16-bit PCM mono at 16kHz
         - Uses maximum available CPU threads for faster processing
     """
     # Create temporary file for audio output
-    fd, temp_audio_path = tempfile.mkstemp(suffix='.mp3')
+    fd, temp_audio_path = tempfile.mkstemp(suffix='.wav')
     os.close(fd)
 
-    logger.info(f"Compressing audio to bitrate quality: {bitrate_quality}")
+    logger.info("Normalizing audio to lossless 16 kHz mono WAV")
     try:
         ffmpeg.input(input_path).output(
             temp_audio_path,
-            q=bitrate_quality,  # Variable bitrate quality (0-9, 9 being lowest)
-            acodec='libmp3lame',
+            acodec='pcm_s16le',
             ac=1,  # Convert to mono
             ar=16000,  # Lower sample rate
             vn=None,
@@ -162,7 +175,7 @@ def compress_and_convert_audio(input_path: str, bitrate_quality: int = 9) -> str
             os.unlink(temp_audio_path)
         raise
 
-    logger.info(f"Successfully converted and compressed audio: {temp_audio_path}")
+    logger.info(f"Successfully normalized audio: {temp_audio_path}")
     return temp_audio_path
 
 
@@ -189,7 +202,7 @@ def transcribe_full_audio(audio_file, markdown_output: bool = False,
         bitrate_quality (int, optional): Variable bitrate quality from 0-9 (9 being lowest). Defaults to 9
         timeout_minutes (int, optional): Number of minutes to wait for a response. Defaults to None.
         max_llm_tokens (int, optional): Token budget used for audio chunk sizing. Defaults to 4250.
-        max_output_tokens (int | None, optional): Maximum Gemini output tokens. Defaults to `max_llm_tokens`.
+        max_output_tokens (int | None, optional): Maximum Gemini output tokens. Defaults to 4096.
         is_output_audio_raw (bool, optional): If True, use the raw Markdown audio prompt.
             If False, use the formatted Markdown audio prompt. Defaults to True.
 
@@ -211,6 +224,8 @@ class AudioToTextConverter:
                  max_output_tokens: int | None = None, temp_dir: str = "temp",
                  bitrate_quality: int = 9, timeout_minutes: int = None,
                  fallback_stage: int = 0,
+                 adaptive_split_depth: int = 0,
+                 long_audio_protections_enabled: bool = False,
                  prompt_variant: str = AUDIO_PROMPT_VARIANT_DEFAULT,
                  is_output_audio_raw: bool  = True):
         """
@@ -225,12 +240,14 @@ class AudioToTextConverter:
             llm_api_key (str, optional): Override API key for language model. Defaults to None.
             max_llm_tokens (int): Token budget used to size audio chunks. Defaults to 4250.
             max_output_tokens (int | None): Maximum number of output tokens for Gemini generation.
-                Defaults to `max_llm_tokens`.
+                Defaults to 4096.
             temp_dir (str): Directory for temporary files. Defaults to "temp".
             bitrate_quality (int, optional): Variable bitrate quality from 0-9 (9 being lowest). Defaults to 9
             timeout_minutes (int): Number of minutes to wait for a response.
             fallback_stage (int, optional): Internal retry stage used by fallback attempts.
                 Defaults to 0.
+            long_audio_protections_enabled (bool, optional): Apply stricter recovery rules inherited
+                from an original audio longer than 80 minutes. Defaults to False.
             prompt_variant (str, optional): Prompt variant used by this attempt.
                 Defaults to "default".
             is_output_audio_raw (bool, optional): If True, use the raw Markdown audio prompt.
@@ -247,12 +264,18 @@ class AudioToTextConverter:
         self.markdown_output = markdown_output
         self.llm_api_key = llm_api_key
         self.max_llm_tokens = max(max_llm_tokens, AUDIO_MIN_OUTPUT_TOKENS)
-        requested_output_tokens = self.max_llm_tokens if max_output_tokens is None else max_output_tokens
+        requested_output_tokens = (
+            AUDIO_DEFAULT_MAX_OUTPUT_TOKENS
+            if max_output_tokens is None
+            else max_output_tokens
+        )
         self.max_output_tokens = max(requested_output_tokens, AUDIO_MIN_OUTPUT_TOKENS)
         self.chunked_audio = False
         self.bitrate_quality = bitrate_quality
         self.timeout_minutes = timeout_minutes
         self.fallback_stage = fallback_stage
+        self.adaptive_split_depth = adaptive_split_depth
+        self.long_audio_protections_enabled = long_audio_protections_enabled
         self.prompt_variant = prompt_variant
         self.fallback_source_pattern = AUDIO_FALLBACK_SOURCE_PATTERN
         self.fallback_model = AUDIO_FALLBACK_MODEL
@@ -276,6 +299,9 @@ class AudioToTextConverter:
             return AUDIO_TO_MARKDOWN_PROMPT
         return AUDIO_TO_PLAIN_TEXT_PROMPT
 
+    def set_long_audio_protections(self, duration_ms: int) -> None:
+        self.long_audio_protections_enabled = duration_ms > AUDIO_LONG_DURATION_THRESHOLD_MS
+
     def should_prompt_fallback_retry(self, error: EmptyDocument) -> bool:
         if self.fallback_stage != 0:
             return False
@@ -338,6 +364,8 @@ class AudioToTextConverter:
             bitrate_quality=self.bitrate_quality,
             timeout_minutes=self.timeout_minutes,
             fallback_stage=fallback_stage,
+            adaptive_split_depth=self.adaptive_split_depth,
+            long_audio_protections_enabled=self.long_audio_protections_enabled,
             prompt_variant=resolved_prompt_variant,
             is_output_audio_raw=self.is_output_audio_raw,
         )
@@ -353,10 +381,83 @@ class AudioToTextConverter:
         result.setdefault("fallback_to_prompt_variant", resolved_prompt_variant)
         return result
 
+    def transcribe_audio_halves(self, audio_file: str, temperature: float = 0.0) -> dict:
+        """Split one genuinely overlong chunk and transcribe both halves once."""
+        audio = AudioSegment.from_file(audio_file)
+        midpoint = len(audio) // 2
+        ranges = (
+            (0, min(len(audio), midpoint + AUDIO_ADAPTIVE_SPLIT_OVERLAP_MS)),
+            (max(0, midpoint - AUDIO_ADAPTIVE_SPLIT_OVERLAP_MS), len(audio)),
+        )
+        split_paths = []
+        split_results = []
+
+        try:
+            for start_ms, end_ms in ranges:
+                fd, split_path = tempfile.mkstemp(
+                    prefix="adaptive-audio-split-",
+                    suffix=".wav",
+                    dir=self.temp_dir,
+                )
+                os.close(fd)
+                split_paths.append(split_path)
+                (
+                    audio[start_ms:end_ms]
+                    .set_frame_rate(16000)
+                    .set_channels(1)
+                    .set_sample_width(2)
+                    .export(split_path, format="wav", codec="pcm_s16le")
+                    .close()
+                )
+
+                split_converter = AudioToTextConverter(
+                    transcription_model=self.transcription_model,
+                    transcription_model_provider=self.transcription_model_provider,
+                    k=self.k,
+                    min_matches=self.min_matches,
+                    markdown_output=self.markdown_output,
+                    llm_api_key=self.llm_api_key,
+                    max_llm_tokens=self.max_llm_tokens,
+                    max_output_tokens=self.max_output_tokens,
+                    temp_dir=self.temp_dir,
+                    bitrate_quality=self.bitrate_quality,
+                    timeout_minutes=self.timeout_minutes,
+                    fallback_stage=self.fallback_stage,
+                    adaptive_split_depth=self.adaptive_split_depth + 1,
+                    long_audio_protections_enabled=self.long_audio_protections_enabled,
+                    prompt_variant=self.prompt_variant,
+                    is_output_audio_raw=self.is_output_audio_raw,
+                )
+                split_results.append(
+                    split_converter.transcribe_audio(split_path, temperature=temperature)
+                )
+
+            merged_transcript = TextMerger(k=self.k, min_matches=self.min_matches).merge_texts(
+                split_results[0]["transcript"],
+                split_results[1]["transcript"],
+            )
+            return {
+                "transcript": merged_transcript,
+                "completion_tokens": sum(item["completion_tokens"] for item in split_results),
+                "prompt_tokens": sum(item["prompt_tokens"] for item in split_results),
+                "completion_model": self.transcription_model,
+                "completion_model_provider": self.transcription_model_provider,
+                "finish_reason": "ADAPTIVE_SPLIT",
+                "max_output_tokens": self.max_output_tokens,
+                "temperature": temperature,
+                "prompt_variant": self.prompt_variant,
+                "adaptive_split": True,
+                "split_results": split_results,
+            }
+        finally:
+            for split_path in split_paths:
+                if os.path.exists(split_path):
+                    os.remove(split_path)
+
     def build_config(self, output_budget: int, temperature: float = 0.0) -> types.GenerateContentConfig:
         return types.GenerateContentConfig(
             temperature=temperature,
-            thinking_config=types.ThinkingConfig(thinking_budget=0),
+            thinking_config=types.ThinkingConfig(thinking_level="minimal"),
             max_output_tokens=output_budget,
             system_instruction=INJECTION_GUARD_SYSTEM_INSTRUCTION,
             tools=[],
@@ -433,6 +534,7 @@ class AudioToTextConverter:
             except ValueError:
                 logger.exception("Unsupported audio format for %s", audio_file)
                 raise
+        mime_type = GEMINI_AUDIO_MIME_ALIASES.get(mime_type, mime_type)
 
         return client.models.generate_content(
             model=self.transcription_model,
@@ -455,7 +557,6 @@ class AudioToTextConverter:
                 google_exceptions.ServiceUnavailable,
                 google_exceptions.InternalServerError,
                 genai_errors.ServerError,
-                genai_errors.APIError,
         ),
         tries=8,
         delay=1,
@@ -518,6 +619,9 @@ class AudioToTextConverter:
                 tail_lines=AUDIO_TAIL_REPETITION_LINES,
                 threshold=AUDIO_TAIL_REPETITION_THRESHOLD,
             )
+            has_repetitive_word_loop = has_excessive_consecutive_word_repetition(
+                response_text,
+            )
             usage_metadata = getattr(response, "usage_metadata", None)
             completion_tokens = getattr(usage_metadata, "candidates_token_count", 0) or 0
             prompt_tokens = getattr(usage_metadata, "prompt_token_count", 0) or 0
@@ -532,19 +636,84 @@ class AudioToTextConverter:
                 )
 
             if finish_reason and "MAX_TOKENS" in finish_reason:
+                if (
+                    self.long_audio_protections_enabled
+                    and self.adaptive_split_depth < AUDIO_MAX_ADAPTIVE_SPLIT_DEPTH
+                ):
+                    logger.info(
+                        "Splitting long-audio chunk before fallback after MAX_TOKENS response: %s",
+                        audio_file,
+                    )
+                    return self.transcribe_audio_halves(audio_file, temperature=temperature)
+                if has_repetitive_tail or has_repetitive_word_loop:
+                    raise EmptyDocument(
+                        message=f"Transcript discarded because repetitive output reached max tokens for audio: {audio_file}",
+                        code=997,
+                    )
+                if self.adaptive_split_depth < AUDIO_MAX_ADAPTIVE_SPLIT_DEPTH:
+                    logger.info(
+                        "Splitting audio chunk after non-repetitive MAX_TOKENS response: %s",
+                        audio_file,
+                    )
+                    return self.transcribe_audio_halves(audio_file, temperature=temperature)
                 raise EmptyDocument(
                     message=f"Transcript truncated because max output tokens were reached for audio: {audio_file}",
                     code=999,
                 )
 
             if has_repetitive_tail:
+                if (
+                    self.long_audio_protections_enabled
+                    and self.adaptive_split_depth < AUDIO_MAX_ADAPTIVE_SPLIT_DEPTH
+                ):
+                    logger.info(
+                        "Splitting long-audio chunk before fallback after repetitive tail: %s",
+                        audio_file,
+                    )
+                    return self.transcribe_audio_halves(audio_file, temperature=temperature)
                 raise EmptyDocument(
                     message=f"Transcript discarded because repetitive tail was detected for audio: {audio_file}",
                     code=997,
                 )
 
+            if has_repetitive_word_loop:
+                if (
+                    self.long_audio_protections_enabled
+                    and self.adaptive_split_depth < AUDIO_MAX_ADAPTIVE_SPLIT_DEPTH
+                ):
+                    logger.info(
+                        "Splitting long-audio chunk before fallback after repetitive word loop: %s",
+                        audio_file,
+                    )
+                    return self.transcribe_audio_halves(audio_file, temperature=temperature)
+                raise EmptyDocument(
+                    message=f"Transcript discarded because repetitive word loop was detected for audio: {audio_file}",
+                    code=997,
+                )
+
             response_text, marker_only = normalize_no_human_speech_marker(response_text)
 
+            is_insignificant_stop = (
+                finish_reason
+                and "STOP" in finish_reason
+                and (
+                    not response_text.strip()
+                    or (
+                        completion_tokens <= 4
+                        and len(response_text.split()) <= 2
+                    )
+                )
+            )
+            if (
+                self.long_audio_protections_enabled
+                and not marker_only
+                and is_insignificant_stop
+            ):
+                raise EmptyDocument(
+                    message=f"Transcript discarded because STOP returned empty or insignificant output for audio: {audio_file}",
+                    code=998,
+                )
+
             response_dict = {
                 "transcript": "" if marker_only else response_text,
                 "completion_tokens": completion_tokens,
@@ -562,6 +731,24 @@ class AudioToTextConverter:
             )
             return response_dict
         except EmptyDocument as e:
+            if (
+                e.code == 999
+                and self.adaptive_split_depth >= AUDIO_MAX_ADAPTIVE_SPLIT_DEPTH
+                and self.fallback_stage == 0
+                and self.transcription_model != self.fallback_model
+            ):
+                return self.run_fallback(
+                    audio_file=audio_file,
+                    reason=e.message,
+                    fallback_model=self.fallback_model,
+                    fallback_temperature=self.fallback_temperature,
+                    fallback_stage=2 if self.markdown_output else 1,
+                    prompt_variant=(
+                        AUDIO_PROMPT_VARIANT_NON_LITERAL_FALLBACK
+                        if self.markdown_output
+                        else self.prompt_variant
+                    ),
+                )
             if self.should_prompt_fallback_retry(e):
                 return self.run_fallback(
                     audio_file=audio_file,
@@ -654,6 +841,13 @@ class AudioToTextConverter:
         # Create chunker and extract chunks
         logger.info("Creating AudioChunker instance...")
         chunker = AudioChunker(used_file, max_llm_tokens=self.max_llm_tokens)
+        self.set_long_audio_protections(chunker.duration_ms)
+        logger.info(
+            "Long-audio transcription protections enabled: %s (duration: %sms, threshold: %sms)",
+            self.long_audio_protections_enabled,
+            chunker.duration_ms,
+            AUDIO_LONG_DURATION_THRESHOLD_MS,
+        )
         chunks = chunker.extract_chunks()
 
         logger.info(f"chunks: {chunks}")

{polytext-0.2.8b3 → polytext-0.2.8b5}/polytext/converter/beautiful_text.py

@@ -9,8 +9,6 @@ from google.api_core import exceptions as google_exceptions
 from retry import retry
 from concurrent.futures import ThreadPoolExecutor, as_completed
 
-from polytext.processor.transcript_chunker import TranscriptChunker
-from polytext.processor.text_merger import TextMerger
 from polytext.prompts.beautiful_text import BEAUTIFUL_TEXT_PROMPT
 
 logger = logging.getLogger(__name__)
@@ -26,6 +24,7 @@ class BeautifulTextConverter:
         prompt_overhead: int = 1800,
         tokens_per_char: float = 0.25,
         overlap_chars: int = 800,
+        max_target_chars: int = 12000,
     ) -> None:
         self.llm_api_key = llm_api_key
         self.model = model
@@ -34,19 +33,58 @@ class BeautifulTextConverter:
         self.prompt_overhead = prompt_overhead
         self.tokens_per_char = tokens_per_char
         self.overlap_chars = overlap_chars
+        self.max_target_chars = max_target_chars
 
     def get_client(self):
         return genai.Client(api_key=self.llm_api_key) if self.llm_api_key else genai.Client()
 
     def chunk_raw_text(self, raw_text: str) -> list[dict]:
-        chunker = TranscriptChunker(
-            transcript=raw_text,
-            max_llm_tokens=self.max_llm_tokens,
-            prompt_overhead=self.prompt_overhead,
-            tokens_per_char=self.tokens_per_char,
-            overlap_chars=self.overlap_chars,
-        )
-        return chunker.chunk_transcript()
+        text = (raw_text or "").strip()
+        if not text:
+            return []
+
+        chunks = []
+        start = 0
+        index = 0
+
+        while start < len(text):
+            proposed_end = min(start + self.max_target_chars, len(text))
+            end = self._find_chunk_end(text, start, proposed_end)
+            target_text = text[start:end].strip()
+
+            if not target_text:
+                break
+
+            chunks.append(
+                {
+                    "index": index,
+                    "target_text": target_text,
+                }
+            )
+            start = end
+            index += 1
+
+        return chunks
+
+    def _find_chunk_end(self, text: str, start: int, proposed_end: int) -> int:
+        if proposed_end >= len(text):
+            return len(text)
+
+        minimum_end = start + int(self.max_target_chars * 0.65)
+        chunk_window = text[minimum_end:proposed_end]
+        sentence_boundaries = list(re.finditer(r"(?<=[.!?])\s+", chunk_window))
+        if sentence_boundaries:
+            return minimum_end + sentence_boundaries[-1].end()
+
+        paragraph_boundary = text.rfind("\n\n", minimum_end, proposed_end)
+        if paragraph_boundary != -1:
+            return paragraph_boundary + 2
+
+        whitespace_boundary = text.rfind(" ", minimum_end, proposed_end)
+        if whitespace_boundary != -1:
+            return whitespace_boundary + 1
+
+        return proposed_end
 
     @retry(
         (
@@ -60,11 +98,18 @@ class BeautifulTextConverter:
         backoff=2,
         logger=logger,
     )
-    def process_chunk(self, client, chunk_text: str, index: int) -> dict:
+    def process_chunk(self, client, chunk: dict, index: int) -> dict:
         logger.info("Processing beautiful text chunk %s", index + 1)
         start_time = time.time()
 
+        target_text = chunk["target_text"]
+
         config = types.GenerateContentConfig(
+            temperature=0,
+            thinking_config=types.ThinkingConfig(thinking_budget=0),
+            max_output_tokens=self.max_llm_tokens,
+            tools=[],
+            automatic_function_calling=types.AutomaticFunctionCallingConfig(disable=True),
             safety_settings=[
                 types.SafetySetting(
                     category=types.HarmCategory.HARM_CATEGORY_HATE_SPEECH,
@@ -87,7 +132,11 @@ class BeautifulTextConverter:
 
         response = client.models.generate_content(
             model=self.model,
-            contents=[BEAUTIFUL_TEXT_PROMPT, chunk_text],
+            contents=[
+                BEAUTIFUL_TEXT_PROMPT,
+                "TARGET TEXT TO CLEAN",
+                target_text,
+            ],
             config=config,
         )
 
@@ -100,7 +149,7 @@ class BeautifulTextConverter:
         }
 
     def merge_cleaned_chunks(self, chunks: list[str]) -> str:
-        return TextMerger(llm_api_key=self.llm_api_key).merge_chunks(chunks=chunks)
+        return "\n\n".join(chunk.strip() for chunk in chunks if chunk.strip())
 
     def _convert_markdown_to_json(self, markdown_text: str) -> dict:
         if not markdown_text.strip():
@@ -181,7 +230,7 @@ class BeautifulTextConverter:
 
         with ThreadPoolExecutor() as executor:
             future_to_index = {
-                executor.submit(self.process_chunk, client, chunk["text"], chunk["index"]): chunk["index"]
+                executor.submit(self.process_chunk, client, chunk, chunk["index"]): chunk["index"]
                 for chunk in chunks
             }
 

{polytext-0.2.8b3 → polytext-0.2.8b5}/polytext/converter/document_ocr_to_text.py

@@ -115,6 +115,7 @@ def get_document_ocr(
     max_output_tokens: int | None = None,
     include_image_descriptions: bool = False,
     allow_partial_ocr_failures: bool = False,
+    ocr_render_dpi: int | None = None,
 ):
     """
     Convenience function to extract text from an image file using OCR, optionally formatted as Markdown.
@@ -141,6 +142,8 @@ def get_document_ocr(
         allow_partial_ocr_failures (bool, optional): If True, pages that still
             fail OCR after all retries are recorded inline instead of aborting
             the whole document extraction. Defaults to False.
+        ocr_render_dpi (int | None, optional): Render PDF pages at this DPI before
+            OCR. When omitted, preserves PyMuPDF's default rendering behavior.
 
     Returns:
         dict: Dictionary containing the OCR results and metadata.
@@ -155,6 +158,7 @@ def get_document_ocr(
         max_output_tokens=max_output_tokens,
         include_image_descriptions=include_image_descriptions,
         allow_partial_ocr_failures=allow_partial_ocr_failures,
+        ocr_render_dpi=ocr_render_dpi,
     )
     return converter.get_document_ocr(document_for_ocr)
 
@@ -164,7 +168,8 @@ class DocumentOCRToTextConverter:
                  page_range=None, timeout_minutes: int = None, fallback_stage: int = 0,
                  max_output_tokens: int | None = None, include_image_descriptions: bool = False,
                  prompt_variant: str = OCR_PROMPT_VARIANT_DEFAULT,
-                 allow_partial_ocr_failures: bool = False):
+                 allow_partial_ocr_failures: bool = False,
+                 ocr_render_dpi: int | None = None):
         """
         Initialize the DocumentOCRToTextConverter class with specified OCR model and formatting options.
 
@@ -192,6 +197,8 @@ class DocumentOCRToTextConverter:
             allow_partial_ocr_failures (bool, optional): If True, pages that still
                 fail OCR after all retries are recorded inline instead of aborting
                 the whole document extraction. Defaults to False.
+            ocr_render_dpi (int | None, optional): Render PDF pages at this DPI
+                before OCR. Defaults to PyMuPDF's native rendering behavior.
 
         Raises:
             OSError: If temp directory creation fails
@@ -207,6 +214,7 @@ class DocumentOCRToTextConverter:
         self.include_image_descriptions = include_image_descriptions
         self.prompt_variant = prompt_variant
         self.allow_partial_ocr_failures = allow_partial_ocr_failures
+        self.ocr_render_dpi = ocr_render_dpi
         requested_output_tokens = OCR_MAX_OUTPUT_TOKENS if max_output_tokens is None else max_output_tokens
         self.max_output_tokens = max(requested_output_tokens, OCR_MIN_OUTPUT_TOKENS)
         self.fallback_stage = fallback_stage
@@ -242,7 +250,9 @@ class DocumentOCRToTextConverter:
         return error.code in OCR_RETRIABLE_OUTPUT_ERROR_CODES
 
     def should_fallback_temperature_retry(self, error: EmptyDocument, temperature: float) -> bool:
-        expected_stage = 1 if self.markdown_output else 0
+        # The non-literal prompt retry advances every OCR mode to stage 1.
+        # Plain-text document OCR must therefore also try the fallback model at stage 1.
+        expected_stage = 1
         if self.fallback_stage != expected_stage:
             return False
         if error.code not in OCR_RETRIABLE_OUTPUT_ERROR_CODES:
@@ -295,6 +305,7 @@ class DocumentOCRToTextConverter:
             include_image_descriptions=self.include_image_descriptions,
             prompt_variant=resolved_prompt_variant,
             allow_partial_ocr_failures=self.allow_partial_ocr_failures,
+            ocr_render_dpi=self.ocr_render_dpi,
         )
         result = fallback_converter.get_ocr(
             file_for_ocr=file_for_ocr,
@@ -368,6 +379,8 @@ class DocumentOCRToTextConverter:
             config = types.GenerateContentConfig(
                 temperature=temperature,
                 max_output_tokens=self.max_output_tokens,
+                tools=[],
+                automatic_function_calling=types.AutomaticFunctionCallingConfig(disable=True),
                 safety_settings=[
                     types.SafetySetting(
                         category=types.HarmCategory.HARM_CATEGORY_HATE_SPEECH,
@@ -549,7 +562,10 @@ class DocumentOCRToTextConverter:
 
             try:
                 # Convert page to image
-                pix = page.get_pixmap()
+                if self.ocr_render_dpi is None:
+                    pix = page.get_pixmap()
+                else:
+                    pix = page.get_pixmap(dpi=self.ocr_render_dpi, alpha=False)
                 pix.save(temp_image_path)
 
                 # Perform OCR on the page

{polytext-0.2.8b3 → polytext-0.2.8b5}/polytext/converter/gemini_quality_guards.py

@@ -37,6 +37,14 @@ def has_consecutive_repetition(items: list[str], min_run_length: int = 3) -> boo
     return False
 
 
+def has_excessive_consecutive_word_repetition(
+    text: str,
+    min_run_length: int = 12,
+) -> bool:
+    words = re.findall(r"\b\w+\b", (text or "").casefold())
+    return has_consecutive_repetition(words, min_run_length=min_run_length)
+
+
 def tail_has_excessive_repetition(
     text: str,
     tail_lines: int,

{polytext-0.2.8b3 → polytext-0.2.8b5}/polytext/converter/ocr_to_text.py

@@ -352,6 +352,8 @@ class OCRToTextConverter:
             config = types.GenerateContentConfig(
                 temperature=temperature,
                 max_output_tokens=self.max_output_tokens,
+                tools=[],
+                automatic_function_calling=types.AutomaticFunctionCallingConfig(disable=True),
                 safety_settings=[
                     types.SafetySetting(
                         category=types.HarmCategory.HARM_CATEGORY_HATE_SPEECH,