polytext 0.2.5__tar.gz → 0.2.6__tar.gz

{polytext-0.2.5 → polytext-0.2.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: polytext
-Version: 0.2.5
+Version: 0.2.6
 Summary: Python utilities to simplify document files management
 Home-page: https://github.com/docsity/polytext
 Author: Matteo Senardi
@@ -21,6 +21,7 @@ Requires-Dist: PyMuPDF>=1.25.5
 Requires-Dist: pycryptodome==3.23.0
 Requires-Dist: weasyprint==65.1
 Requires-Dist: markdown==3.8
+Requires-Dist: markdown-to-json==2.1.2
 Requires-Dist: python-docx==1.1.2
 Requires-Dist: google-api-core>=2.24.2
 Requires-Dist: google-cloud-storage<3.0.0,>=2.17

polytext-0.2.6/polytext/converter/beautiful_text.py

@@ -0,0 +1,209 @@
+import logging
+import re
+import time
+from importlib import import_module
+
+from google import genai
+from google.genai import types
+from google.api_core import exceptions as google_exceptions
+from retry import retry
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+from polytext.processor.transcript_chunker import TranscriptChunker
+from polytext.processor.text_merger import TextMerger
+from polytext.prompts.beautiful_text import BEAUTIFUL_TEXT_PROMPT
+
+logger = logging.getLogger(__name__)
+
+
+class BeautifulTextConverter:
+    def __init__(
+        self,
+        llm_api_key: str = None,
+        model: str = "gemini-3.1-flash-lite",
+        model_provider: str = "google",
+        max_llm_tokens: int = 8000,
+        prompt_overhead: int = 1800,
+        tokens_per_char: float = 0.25,
+        overlap_chars: int = 800,
+    ) -> None:
+        self.llm_api_key = llm_api_key
+        self.model = model
+        self.model_provider = model_provider
+        self.max_llm_tokens = max_llm_tokens
+        self.prompt_overhead = prompt_overhead
+        self.tokens_per_char = tokens_per_char
+        self.overlap_chars = overlap_chars
+
+    def get_client(self):
+        return genai.Client(api_key=self.llm_api_key) if self.llm_api_key else genai.Client()
+
+    def chunk_raw_text(self, raw_text: str) -> list[dict]:
+        chunker = TranscriptChunker(
+            transcript=raw_text,
+            max_llm_tokens=self.max_llm_tokens,
+            prompt_overhead=self.prompt_overhead,
+            tokens_per_char=self.tokens_per_char,
+            overlap_chars=self.overlap_chars,
+        )
+        return chunker.chunk_transcript()
+
+    @retry(
+        (
+            google_exceptions.DeadlineExceeded,
+            google_exceptions.ResourceExhausted,
+            google_exceptions.ServiceUnavailable,
+            google_exceptions.InternalServerError,
+        ),
+        tries=5,
+        delay=2,
+        backoff=2,
+        logger=logger,
+    )
+    def process_chunk(self, client, chunk_text: str, index: int) -> dict:
+        logger.info("Processing beautiful text chunk %s", index + 1)
+        start_time = time.time()
+
+        config = types.GenerateContentConfig(
+            safety_settings=[
+                types.SafetySetting(
+                    category=types.HarmCategory.HARM_CATEGORY_HATE_SPEECH,
+                    threshold=types.HarmBlockThreshold.BLOCK_NONE,
+                ),
+                types.SafetySetting(
+                    category=types.HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT,
+                    threshold=types.HarmBlockThreshold.BLOCK_NONE,
+                ),
+                types.SafetySetting(
+                    category=types.HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT,
+                    threshold=types.HarmBlockThreshold.BLOCK_NONE,
+                ),
+                types.SafetySetting(
+                    category=types.HarmCategory.HARM_CATEGORY_HARASSMENT,
+                    threshold=types.HarmBlockThreshold.BLOCK_NONE,
+                ),
+            ]
+        )
+
+        response = client.models.generate_content(
+            model=self.model,
+            contents=[BEAUTIFUL_TEXT_PROMPT, chunk_text],
+            config=config,
+        )
+
+        logger.info("Beautiful text chunk %s processed in %.2fs", index + 1, time.time() - start_time)
+
+        return {
+            "transcript": response.text,
+            "completion_tokens": response.usage_metadata.candidates_token_count,
+            "prompt_tokens": response.usage_metadata.prompt_token_count,
+        }
+
+    def merge_cleaned_chunks(self, chunks: list[str]) -> str:
+        return TextMerger(llm_api_key=self.llm_api_key).merge_chunks(chunks=chunks)
+
+    def _convert_markdown_to_json(self, markdown_text: str) -> dict:
+        if not markdown_text.strip():
+            return {}
+
+        try:
+            markdown_to_json = import_module("markdown_to_json")
+        except ImportError as exc:
+            raise ImportError(
+                "markdown-to-json is required when active_chapters=True. "
+                "Install it with: pip install markdown-to-json"
+            ) from exc
+
+        return markdown_to_json.dictify(markdown_text)
+
+    def _build_chapters(self, markdown_text: str) -> list[dict]:
+        heading_pattern = re.compile(r"^(#{1,6})\s+(.*?)\s*$")
+        chapters = []
+        stack: list[dict] = []
+
+        def finalize_nodes(target_depth: int = 0) -> None:
+            while len(stack) > target_depth:
+                node_state = stack.pop()
+                node_state["node"]["content"] = "\n".join(node_state["content_lines"]).strip()
+
+        for line in markdown_text.splitlines():
+            heading_match = heading_pattern.match(line)
+            if heading_match:
+                level = len(heading_match.group(1))
+                title = heading_match.group(2).strip()
+
+                while stack and stack[-1]["node"]["level"] >= level:
+                    finalize_nodes(len(stack) - 1)
+
+                chapter_node = {
+                    "title": title,
+                    "level": level,
+                    "content": "",
+                    "children": [],
+                }
+
+                if stack:
+                    stack[-1]["node"]["children"].append(chapter_node)
+                else:
+                    chapters.append(chapter_node)
+
+                stack.append({"node": chapter_node, "content_lines": []})
+                continue
+
+            if stack:
+                stack[-1]["content_lines"].append(line)
+
+        finalize_nodes()
+        return chapters
+
+    def convert(self, raw_text: str, save_transcript_chunks: bool = False, active_chapters: bool = False) -> dict:
+        cleaned_input = (raw_text or "").strip()
+        if not cleaned_input:
+            result = {
+                "text": "",
+                "completion_tokens": 0,
+                "prompt_tokens": 0,
+                "completion_model": self.model,
+                "completion_model_provider": self.model_provider,
+                "text_chunks": [] if save_transcript_chunks else "not provided",
+            }
+            if active_chapters:
+                result["markdown_json"] = {}
+                result["chapters"] = []
+            return result
+
+        chunks = self.chunk_raw_text(cleaned_input)
+        client = self.get_client()
+
+        results = []
+        total_completion_tokens = 0
+        total_prompt_tokens = 0
+
+        with ThreadPoolExecutor() as executor:
+            future_to_index = {
+                executor.submit(self.process_chunk, client, chunk["text"], chunk["index"]): chunk["index"]
+                for chunk in chunks
+            }
+
+            for future in as_completed(future_to_index):
+                index = future_to_index[future]
+                result = future.result()
+                results.append((index, result["transcript"]))
+                total_completion_tokens += result["completion_tokens"]
+                total_prompt_tokens += result["prompt_tokens"]
+
+        cleaned_chunks = [text for index, text in sorted(results, key=lambda item: item[0])]
+        final_text = self.merge_cleaned_chunks(cleaned_chunks)
+
+        result = {
+            "text": final_text,
+            "completion_tokens": total_completion_tokens,
+            "prompt_tokens": total_prompt_tokens,
+            "completion_model": self.model,
+            "completion_model_provider": self.model_provider,
+            "text_chunks": cleaned_chunks if save_transcript_chunks else "not provided",
+        }
+        if active_chapters:
+            result["markdown_json"] = self._convert_markdown_to_json(final_text)
+            result["chapters"] = self._build_chapters(final_text)
+        return result

{polytext-0.2.5 → polytext-0.2.6}/polytext/loader/base.py

@@ -33,6 +33,8 @@ import boto3
 from google.cloud import storage
 from google.genai import errors as genai_errors
 
+from ..converter.beautiful_text import BeautifulTextConverter
+
 
 dotenv.load_dotenv()
 
@@ -235,6 +237,59 @@ class BaseLoader:
 
         return response
 
+    def get_beautiful_text(self, input_list: list[str], **kwargs):
+        if not isinstance(input_list, list) or not all(isinstance(item, str) for item in input_list):
+            raise TypeError("Parameter 'input' must be a list of strings.")
+        if not input_list:
+            raise ValueError("Input list is empty.")
+        if len(input_list) != 1:
+            raise ValueError("get_beautiful_text expects exactly one input.")
+
+        kwargs = {**self.kwargs, **kwargs}
+        raw_result = self.extract_raw_text_for_beautiful_text(input_value=input_list[0], **kwargs)
+
+        converter = BeautifulTextConverter(llm_api_key=self.llm_api_key)
+        cleanup_result = converter.convert(
+            raw_text=raw_result["text"],
+            save_transcript_chunks=kwargs.get("save_transcript_chunks", self.save_transcript_chunks),
+            active_chapters=kwargs.get("active_chapters", False),
+        )
+
+        total_completion_tokens = raw_result.get("completion_tokens", 0) + cleanup_result.get("completion_tokens", 0)
+        total_prompt_tokens = raw_result.get("prompt_tokens", 0) + cleanup_result.get("prompt_tokens", 0)
+
+        result_item = {
+            "text": cleanup_result["text"],
+            "completion_tokens": total_completion_tokens,
+            "prompt_tokens": total_prompt_tokens,
+            "completion_model": cleanup_result.get("completion_model", "not provided"),
+            "completion_model_provider": cleanup_result.get("completion_model_provider", "not provided"),
+            "text_chunks": cleanup_result.get("text_chunks", "not provided"),
+            "type": raw_result.get("type", "text"),
+            "input": input_list[0],
+        }
+        if "markdown_json" in cleanup_result:
+            result_item["markdown_json"] = cleanup_result["markdown_json"]
+        if "chapters" in cleanup_result:
+            result_item["chapters"] = cleanup_result["chapters"]
+
+        response = {
+            "text": result_item["text"],
+            "completion_tokens": result_item["completion_tokens"],
+            "prompt_tokens": result_item["prompt_tokens"],
+            "completion_model": result_item["completion_model"],
+            "completion_model_provider": result_item["completion_model_provider"],
+            "text_chunks": result_item["text_chunks"],
+            "type": result_item["type"],
+            "input": result_item["input"],
+            "output_list": [result_item],
+        }
+        if "markdown_json" in result_item:
+            response["markdown_json"] = result_item["markdown_json"]
+        if "chapters" in result_item:
+            response["chapters"] = result_item["chapters"]
+        return response
+
     def initiate_storage(self, input: str) -> dict:
         """
             Initializes and returns a client and relevant details for various cloud storage services or web URLs.
@@ -518,6 +573,114 @@ class BaseLoader:
             return True
         return False
 
+    @staticmethod
+    def is_remote_input(s: str) -> bool:
+        return s.startswith(("s3://", "gcs://", "http://", "https://", "www.", "www.youtube"))
+
+    @staticmethod
+    def is_text_file_extension(path_value: str) -> bool:
+        return Path(path_value).suffix.lower() in {".txt", ".text", ".md", ".markdown"}
+
+    @staticmethod
+    def is_beautiful_text_supported_file_extension(path_value: str) -> bool:
+        return Path(path_value).suffix.lower() in {
+            ".txt",
+            ".text",
+            ".md",
+            ".markdown",
+            ".pdf",
+            ".xlsx",
+            ".docx",
+            ".csv",
+            ".odt",
+            ".pptx",
+            ".xls",
+            ".doc",
+            ".ppt",
+            ".rtf",
+            ".ipynb",
+            ".xml",
+            ".xbrl",
+        }
+
+    def extract_raw_text_for_beautiful_text(self, input_value: str, **kwargs) -> dict:
+        cleaned_input = input_value.strip()
+
+        if "\n" in cleaned_input or (not self.is_local_path(cleaned_input) and not self.is_remote_input(cleaned_input)):
+            return {
+                "text": cleaned_input,
+                "completion_tokens": 0,
+                "prompt_tokens": 0,
+                "completion_model": "not provided",
+                "completion_model_provider": "not provided",
+                "text_chunks": "not provided",
+                "type": "text",
+                "input": input_value,
+            }
+
+        local_path = Path(cleaned_input)
+
+        if local_path.exists():
+            if local_path.is_file() and self.is_text_file_extension(cleaned_input):
+                return {
+                    "text": local_path.read_text(encoding="utf-8"),
+                    "completion_tokens": 0,
+                    "prompt_tokens": 0,
+                    "completion_model": "not provided",
+                    "completion_model_provider": "not provided",
+                    "text_chunks": "not provided",
+                    "type": "text",
+                    "input": input_value,
+                }
+
+        if self.is_local_path(cleaned_input) and not self.is_remote_input(cleaned_input):
+            if not local_path.exists():
+                raise FileNotFoundError(f"Input not found or format not recognized: {input_value}")
+
+            if not self.is_beautiful_text_supported_file_extension(cleaned_input):
+                raise ValueError(
+                    "get_beautiful_text supports only text or document inputs such as txt, md, pdf, docx, xlsx, csv, ipynb, xml, or xbrl."
+                )
+
+        if self.is_remote_input(cleaned_input):
+            if cleaned_input.startswith(("http://", "https://", "www.", "www.youtube")):
+                raise ValueError(
+                    "get_beautiful_text does not support web pages, YouTube, audio, video, or image URLs. Pass text directly or a text/document file path."
+                )
+
+            if not self.is_beautiful_text_supported_file_extension(cleaned_input):
+                raise ValueError(
+                    "get_beautiful_text supports only text or document file inputs such as txt, md, pdf, docx, xlsx, csv, ipynb, xml, or xbrl."
+                )
+
+        storage_client = self.initiate_storage(input=input_value)
+        loader_class = self.init_loader_class(
+            input=input_value,
+            storage_client=storage_client,
+            llm_api_key=self.llm_api_key,
+            **kwargs,
+        )
+
+        unsupported_loader_types = (AudioLoader, VideoLoader, OCRLoader, HtmlLoader, YoutubeTranscriptLoaderWithLlm)
+        if isinstance(loader_class, unsupported_loader_types):
+            raise ValueError(
+                "get_beautiful_text supports only text or document inputs, not audio, video, image, HTML, or YouTube sources."
+            )
+
+        extracted = self.run_loader_class(loader_class=loader_class, input_list=[input_value])
+        output_item = extracted.get("output_list", [{}])[0]
+
+        return {
+            "text": extracted.get("text", ""),
+            "completion_tokens": extracted.get("completion_tokens", 0),
+            "prompt_tokens": extracted.get("prompt_tokens", 0),
+            "completion_model": output_item.get("completion_model", "not provided"),
+            "completion_model_provider": output_item.get("completion_model_provider", "not provided"),
+            "text_chunks": output_item.get("text_chunks", "not provided"),
+            "type": output_item.get("type", "not provided"),
+            "input": output_item.get("input", input_value),
+        }
+
     def validate_user_text(self, text: str) -> bool:
         """
         Validate a text string. Raises EmptyDocument if the text is too short.

polytext-0.2.6/polytext/prompts/beautiful_text.py

@@ -0,0 +1,43 @@
+BEAUTIFUL_TEXT_PROMPT = """
+You are an editor specialized in cleaning spoken transcripts and raw text into faithful Markdown.
+This is not summarization. This is not rewriting. This is a cleaned transcript or cleaned source text.
+
+Your task is to remove only accidental noise while preserving the speaker's or author's original words,
+phrasing, reasoning, tone, and sequence of ideas as faithfully as possible.
+
+REMOVE ONLY:
+- non-meaningful fillers such as "eh", "uhm", "diciamo", "eccetera eccetera", "no?" when used only as filler
+- redundant "quindi", "appunto", "comunque" when they are only conversational padding
+- accidental repeated words such as "di di", "da da", "che che"
+- false starts and self-corrections only when they do not carry meaning
+- irrelevant overlap fragments between speakers
+
+PRESERVE COMPLETELY:
+- the original wording and sentence structure, even if colloquial
+- technical terms and proper nouns exactly
+- the original tone and register
+- reasoning, opinions, nuances, and meaningful uncertainty
+- the logical order of the discussion
+
+DO NOT:
+- rewrite sentences in a more elegant style
+- replace words with synonyms
+- summarize, compress, or simplify concepts
+- add explanations, transitions, or missing content
+- correct the speaker's opinions or inaccuracies
+- make the language more formal than the original
+
+FORMATTING:
+- output Markdown only
+- use paragraphs to separate thematic blocks
+- add headings only when the speaker explicitly introduces a new topic
+- use bullet lists or numbered lists only when the source explicitly enumerates items or when the sequence is clearly list-shaped
+- use emphasis sparingly and only when grounded in the original text
+- use **bold** for key information and important concepts, and *italics* for subtle emphasis or contextual terms in every chapter and paragraph whenever they improve readability and understanding
+- do not add code fences
+- do not add introductions or commentary
+
+FINAL CHECK:
+- every sentence in the output must be traceable to an equivalent sentence in the input
+- if a sentence cannot be grounded in the input, remove it
+"""

{polytext-0.2.5 → polytext-0.2.6}/polytext.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: polytext
-Version: 0.2.5
+Version: 0.2.6
 Summary: Python utilities to simplify document files management
 Home-page: https://github.com/docsity/polytext
 Author: Matteo Senardi
@@ -21,6 +21,7 @@ Requires-Dist: PyMuPDF>=1.25.5
 Requires-Dist: pycryptodome==3.23.0
 Requires-Dist: weasyprint==65.1
 Requires-Dist: markdown==3.8
+Requires-Dist: markdown-to-json==2.1.2
 Requires-Dist: python-docx==1.1.2
 Requires-Dist: google-api-core>=2.24.2
 Requires-Dist: google-cloud-storage<3.0.0,>=2.17

{polytext-0.2.5 → polytext-0.2.6}/polytext.egg-info/SOURCES.txt

@@ -12,6 +12,7 @@ polytext.egg-info/top_level.txt
 polytext/converter/__init__.py
 polytext/converter/audio_to_text.py
 polytext/converter/base.py
+polytext/converter/beautiful_text.py
 polytext/converter/document_ocr_to_text.py
 polytext/converter/document_ocr_to_text_azure_oai.py
 polytext/converter/gemini_quality_guards.py
@@ -47,6 +48,7 @@ polytext/processor/audio_chunker.py
 polytext/processor/text_merger.py
 polytext/processor/transcript_chunker.py
 polytext/prompts/__init__.py
+polytext/prompts/beautiful_text.py
 polytext/prompts/ocr.py
 polytext/prompts/text_merging.py
 polytext/prompts/text_to_md.py
@@ -57,6 +59,7 @@ tests/test_audio_chunker.py
 tests/test_audio_comparison_helpers.py
 tests/test_audio_transcription_model_migration.py
 tests/test_base_loader_error_mapping.py
+tests/test_beautiful_text_manual.py
 tests/test_compare_audio_models.py
 tests/test_compare_document_ocr_to_text_models.py
 tests/test_compare_ocr_to_text_models.py

{polytext-0.2.5 → polytext-0.2.6}/polytext.egg-info/requires.txt

@@ -3,6 +3,7 @@ PyMuPDF>=1.25.5
 pycryptodome==3.23.0
 weasyprint==65.1
 markdown==3.8
+markdown-to-json==2.1.2
 python-docx==1.1.2
 google-api-core>=2.24.2
 google-cloud-storage<3.0.0,>=2.17

{polytext-0.2.5 → polytext-0.2.6}/setup.py

@@ -51,7 +51,7 @@ def get_requirements(*requirements_file):
 
 setup(
     name='polytext',
-    version='0.2.5',
+    version='0.2.6',
     url='https://github.com/docsity/polytext',
     # download_url='https://github.com/pualien/py-polytext/archive/0.1.23.tar.gz',
     license='MIT',

polytext-0.2.6/tests/test_beautiful_text_manual.py

@@ -0,0 +1,68 @@
+import logging
+import os
+import sys
+
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+
+from dotenv import load_dotenv
+
+from polytext.loader.base import BaseLoader
+
+load_dotenv("..env")
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger("polytext")
+
+
+PLAIN_TEXT_SAMPLE = """"""
+
+
+def run_beautiful_text(input_value: str, source: str = "local", activate_full_process: bool = False) -> dict:
+    loader = BaseLoader(
+        markdown_output=True,
+        save_transcript_chunks=True,
+        source=source,
+    )
+    result_dict_original = None
+    if activate_full_process:
+        local_file_path = "/Users/username/Projects/polytext/mk5konma-f4c711ba3480d81e35d60ddee5cea1cf15c690d6.mp4" #  "s3://docsity-data/documents/original/2011/06/05/La_disciplina_degli_enti_locali_negli_statuti_regionali.pdf"  # "https://www.youtube.com/watch?v=Njrr5etGFGg&list=PLufwFtblC0Rta8GZlxVJ7DWqVnrENbrCV&index=4" # "/Users/andreasolfanelli/Projects/polytext/giovedì alle 10-15(2).aac"
+
+        logger.info("FULL PROCESS ACTIVE - using local file path: %s", local_file_path)
+        result_dict_original = loader.get_text(input_list=[local_file_path])
+        logger.info("***** END FULL PROCESS ACTIVE ******")
+        print(result_dict_original)
+
+    result_dict = loader.get_beautiful_text(input_list=[result_dict_original["text"] if activate_full_process else input_value], active_chapters=True)
+
+    logger.info("Input: %s", input_value)
+    logger.info("Type: %s", result_dict.get("type"))
+    logger.info("Completion model: %s", result_dict.get("completion_model"))
+    logger.info("Completion tokens: %s", result_dict.get("completion_tokens"))
+    logger.info("Prompt tokens: %s", result_dict.get("prompt_tokens"))
+    logger.info("Preview:\n%s", result_dict.get("text", "")[:2000])
+
+    return result_dict
+
+
+def main():
+    mode = "full_process"
+
+    if mode == "plain_text":
+        return run_beautiful_text(PLAIN_TEXT_SAMPLE, source="local")
+
+    if mode == "local_file":
+        local_file_path = "/Users/username/Projects/polytext/summary_note_64_84_level_1_develop_type_G20Y40.pdf" # "/Users/andreasolfanelli/Projects/polytext/2.-Principi-fondamentali.pdf" # "/Users/andreasolfanelli/Projects/polytext/1.materiale del'istituto Lezione-2_La-norma-giuridica--caratteristiche-e-interpretazione--e-sanzioni.pdf"
+        return run_beautiful_text(local_file_path, source="local")
+
+    if mode == "s3_file":
+        s3_file_path = "s3://docsity-data/documents/original/2011/06/05/La_disciplina_degli_enti_locali_negli_statuti_regionali.pdf" # "s3://docsity-ai-develop/da_ml_ai_summary_output/lang=it/y=2026/m=05/d=12/upload_date=2026-05-12/summary_note_70_96_level_1_develop_type_G20Y40.pdf"
+        return run_beautiful_text(s3_file_path, source="cloud")
+
+    if mode == "full_process":
+        return run_beautiful_text("", source="local", activate_full_process=True)
+
+    raise ValueError(f"Unsupported mode: {mode}")
+
+
+if __name__ == "__main__":
+    main()