chatterer 0.1.22__tar.gz → 0.1.24__tar.gz

{chatterer-0.1.22 → chatterer-0.1.24}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: chatterer
-Version: 0.1.22
+Version: 0.1.24
 Summary: The highest-level interface for various LLM APIs.
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
@@ -11,10 +11,9 @@ Requires-Dist: pillow>=11.1.0
 Requires-Dist: regex>=2024.11.6
 Requires-Dist: rich>=13.9.4
 Requires-Dist: colorama>=0.4.6
-Requires-Dist: spargear>=0.2.0
+Requires-Dist: spargear>=0.2.7
 Provides-Extra: dev
-Requires-Dist: neo4j-extension>=0.1.14; extra == "dev"
-Requires-Dist: ipykernel>=6.29.5; extra == "dev"
+Requires-Dist: pyright>=1.1.401; extra == "dev"
 Provides-Extra: conversion
 Requires-Dist: youtube-transcript-api>=1.0.3; extra == "conversion"
 Requires-Dist: chatterer[browser]; extra == "conversion"
@@ -34,12 +33,10 @@ Requires-Dist: mistune>=3.1.3; extra == "markdown"
 Provides-Extra: video
 Requires-Dist: pydub>=0.25.1; extra == "video"
 Provides-Extra: langchain
-Requires-Dist: chatterer[langchain-providers]; extra == "langchain"
+Requires-Dist: langchain-anthropic>=0.3.10; extra == "langchain"
+Requires-Dist: langchain-google-genai>=2.1.1; extra == "langchain"
+Requires-Dist: langchain-ollama>=0.3.0; extra == "langchain"
 Requires-Dist: langchain-experimental>=0.3.4; extra == "langchain"
-Provides-Extra: langchain-providers
-Requires-Dist: langchain-anthropic>=0.3.10; extra == "langchain-providers"
-Requires-Dist: langchain-google-genai>=2.1.1; extra == "langchain-providers"
-Requires-Dist: langchain-ollama>=0.3.0; extra == "langchain-providers"
 Provides-Extra: all
 Requires-Dist: chatterer[dev]; extra == "all"
 Requires-Dist: chatterer[langchain]; extra == "all"

chatterer-0.1.24/chatterer/examples/__main__.py

@@ -0,0 +1,75 @@
+from spargear import SubcommandArguments, SubcommandSpec
+
+
+def any2md():
+    from .any2md import Arguments
+
+    return Arguments
+
+
+def pdf2md():
+    from .pdf2md import Arguments
+
+    return Arguments
+
+
+def pdf2txt():
+    from .pdf2txt import Arguments
+
+    return Arguments
+
+
+def ppt():
+    from .ppt import Arguments
+
+    return Arguments
+
+
+def pw():
+    from .pw import Arguments
+
+    return Arguments
+
+
+def snippet():
+    from .snippet import Arguments
+
+    return Arguments
+
+
+def transcribe():
+    from .transcribe import Arguments
+
+    return Arguments
+
+
+def upstage():
+    from .upstage import Arguments
+
+    return Arguments
+
+
+def web2md():
+    from .web2md import Arguments
+
+    return Arguments
+
+
+class Arguments(SubcommandArguments):
+    any2md = SubcommandSpec(name="any2md", argument_class_factory=any2md)
+    pdf2md = SubcommandSpec(name="pdf2md", argument_class_factory=pdf2md)
+    pdf2txt = SubcommandSpec(name="pdf2txt", argument_class_factory=pdf2txt)
+    ppt = SubcommandSpec(name="ppt", argument_class_factory=ppt)
+    pw = SubcommandSpec(name="pw", argument_class_factory=pw)
+    snippet = SubcommandSpec(name="snippet", argument_class_factory=snippet)
+    transcribe = SubcommandSpec(name="transcribe", argument_class_factory=transcribe)
+    upstage = SubcommandSpec(name="upstage", argument_class_factory=upstage)
+    web2md = SubcommandSpec(name="web2md", argument_class_factory=web2md)
+
+
+def main():
+    Arguments().execute()
+
+
+if __name__ == "__main__":
+    main()

chatterer-0.1.22/chatterer/examples/anything_to_markdown.py → chatterer-0.1.24/chatterer/examples/any2md.py

@@ -3,7 +3,7 @@ from pathlib import Path
 from typing import Optional, TypedDict
 
 import openai
-from spargear import BaseArguments
+from spargear import RunnableArguments
 
 from chatterer import anything_to_markdown
 
@@ -16,10 +16,10 @@ class AnythingToMarkdownReturns(TypedDict):
     out_text: str
 
 
-class AnythingToMarkdownArguments(BaseArguments):
+class Arguments(RunnableArguments[AnythingToMarkdownReturns]):
     """Command line arguments for converting various file types to markdown."""
 
-    input: str
+    SOURCE: str
     """Input file to convert to markdown. Can be a file path or a URL."""
     output: Optional[str] = None
     """Output path for the converted markdown file. If not provided, the input file's suffix is replaced with .md"""
@@ -43,7 +43,7 @@ class AnythingToMarkdownArguments(BaseArguments):
     def run(self) -> AnythingToMarkdownReturns:
         if not self.prevent_save_file:
             if not self.output:
-                output = Path(self.input).with_suffix(".md")
+                output = Path(self.SOURCE).with_suffix(".md")
             else:
                 output = Path(self.output)
         else:
@@ -57,7 +57,7 @@ class AnythingToMarkdownArguments(BaseArguments):
             llm_model = None
 
         text: str = anything_to_markdown(
-            self.input,
+            self.SOURCE,
             llm_client=llm_client,
             llm_model=llm_model,
             style_map=self.style_map,
@@ -67,18 +67,18 @@ class AnythingToMarkdownArguments(BaseArguments):
         if output:
             output.parent.mkdir(parents=True, exist_ok=True)
             output.write_text(text, encoding=self.encoding)
-            logger.info(f"Converted `{self.input}` to markdown and saved to `{output}`.")
+            logger.info(f"Converted `{self.SOURCE}` to markdown and saved to `{output}`.")
         else:
-            logger.info(f"Converted `{self.input}` to markdown.")
+            logger.info(f"Converted `{self.SOURCE}` to markdown.")
         return {
-            "input": self.input,
+            "input": self.SOURCE,
             "output": str(output) if output is not None else None,
             "out_text": text,
         }
 
 
 def main() -> None:
-    AnythingToMarkdownArguments().run()
+    Arguments().run()
 
 
 if __name__ == "__main__":

chatterer-0.1.24/chatterer/examples/pdf2md.py

@@ -0,0 +1,338 @@
+#!/usr/bin/env python3
+"""
+PDF to Markdown Converter CLI
+
+A command-line tool for converting PDF documents to Markdown using multimodal LLMs.
+Supports both sequential and parallel processing modes with async capabilities.
+"""
+
+import asyncio
+import logging
+import sys
+import time
+from pathlib import Path
+from typing import List, Literal, Optional, TypedDict
+
+from spargear import ArgumentSpec, RunnableArguments
+
+from chatterer import Chatterer
+from chatterer.tools.convert_pdf_to_markdown import PdfToMarkdown
+
+
+class ConversionResult(TypedDict, total=False):
+    """Type definition for conversion results."""
+
+    input: str
+    output: str
+    result: str
+    processing_time: float
+    characters: int
+    error: str
+
+
+# Setup enhanced logging
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", datefmt="%H:%M:%S")
+logger = logging.getLogger(__name__)
+
+
+class Arguments(RunnableArguments[List[ConversionResult]]):
+    """Command-line arguments for PDF to Markdown conversion."""
+
+    PDF_OR_DIRECTORY_PATH: str
+    """Input PDF file or directory containing PDF files to convert to markdown."""
+
+    output: Optional[str] = None
+    """Output path. For a file, path to the output markdown file. For a directory, output directory for .md files."""
+
+    page: Optional[str] = None
+    """Zero-based page indices to convert (e.g., '0,2,4-8'). If None, converts all pages."""
+
+    recursive: bool = False
+    """If input is a directory, search for PDFs recursively."""
+
+    mode: Literal["sequential", "parallel"] = "parallel"
+    """Processing mode: 'sequential' for strict continuity, 'parallel' for faster processing."""
+
+    sync: bool = False
+    """Enable synchronous processing for sequential mode. If set to True, will run in sync mode."""
+
+    max_concurrent: int = 10
+    """Maximum number of concurrent LLM requests when using async mode."""
+
+    image_zoom: float = 2.0
+    """Zoom factor for rendering PDF pages as images (higher zoom = higher resolution)."""
+
+    image_format: Literal["png", "jpg", "jpeg"] = "png"
+    """Image format for PDF page rendering."""
+
+    image_quality: int = 95
+    """JPEG quality when using jpg/jpeg format (1-100)."""
+
+    context_tail_lines: int = 10
+    """Number of lines from previous page's markdown to use as context (sequential mode only)."""
+
+    verbose: bool = False
+    """Enable verbose logging output."""
+
+    chatterer: ArgumentSpec[Chatterer] = ArgumentSpec(
+        ["--chatterer"],
+        default_factory=lambda: Chatterer.from_provider("google:gemini-2.5-flash-preview-05-20"),
+        help="Chatterer instance configuration (e.g., 'google:gemini-2.5-flash-preview-05-20').",
+        type=Chatterer.from_provider,
+    )
+
+    def __post_init__(self) -> None:
+        """Validate and adjust arguments after initialization."""
+        if self.verbose:
+            logging.getLogger().setLevel(logging.DEBUG)
+
+        if not self.sync and self.mode == "sequential":
+            logger.warning("Async mode is only available with parallel mode. Switching to parallel mode.")
+            self.mode = "parallel"
+
+        if self.max_concurrent < 1:
+            logger.warning("max_concurrent must be >= 1. Setting to 1.")
+            self.max_concurrent = 1
+        elif self.max_concurrent > 10:
+            logger.warning("max_concurrent > 10 may cause rate limiting. Consider reducing.")
+
+    def run(self) -> List[ConversionResult]:
+        """Execute the PDF to Markdown conversion."""
+        if not self.sync:
+            return asyncio.run(self._run_async())
+        else:
+            return self._run_sync()
+
+    def _run_sync(self) -> List[ConversionResult]:
+        """Execute synchronous conversion."""
+        pdf_files, output_base, is_dir = self._prepare_files()
+
+        converter = PdfToMarkdown(
+            chatterer=self.chatterer.unwrap(),
+            image_zoom=self.image_zoom,
+            image_format=self.image_format,
+            image_jpg_quality=self.image_quality,
+            context_tail_lines=self.context_tail_lines,
+        )
+
+        results: List[ConversionResult] = []
+        total_start_time = time.time()
+
+        logger.info(f"🚀 Starting {self.mode} conversion of {len(pdf_files)} PDF(s)...")
+
+        for i, pdf in enumerate(pdf_files, 1):
+            output_path = (output_base / f"{pdf.stem}.md") if is_dir else output_base
+
+            logger.info(f"📄 Processing {i}/{len(pdf_files)}: {pdf.name}")
+            start_time = time.time()
+
+            # Progress callback for individual PDF
+            def progress_callback(current: int, total: int) -> None:
+                progress = (current / total) * 100
+                logger.info(f"  └─ Progress: {current}/{total} pages ({progress:.1f}%)")
+
+            try:
+                markdown = converter.convert(
+                    pdf_input=str(pdf),
+                    page_indices=self.page,
+                    mode=self.mode,
+                    progress_callback=progress_callback,
+                )
+
+                # Save result
+                output_path.parent.mkdir(parents=True, exist_ok=True)
+                output_path.write_text(markdown, encoding="utf-8")
+
+                elapsed = time.time() - start_time
+                chars_per_sec = len(markdown) / elapsed if elapsed > 0 else 0
+
+                logger.info(f"  ✅ Completed in {elapsed:.1f}s ({chars_per_sec:.0f} chars/s)")
+                logger.info(f"  📝 Generated {len(markdown):,} characters → {output_path}")
+
+                results.append({
+                    "input": pdf.as_posix(),
+                    "output": output_path.as_posix(),
+                    "result": markdown,
+                    "processing_time": elapsed,
+                    "characters": len(markdown),
+                })
+
+            except Exception as e:
+                logger.error(f"  ❌ Failed to process {pdf.name}: {e}")
+                results.append({
+                    "input": pdf.as_posix(),
+                    "output": "",
+                    "result": "",
+                    "error": str(e),
+                })
+
+        total_elapsed = time.time() - total_start_time
+        total_chars = sum(len(r.get("result", "")) for r in results)
+        successful_conversions = sum(1 for r in results if "error" not in r)
+
+        logger.info("🎉 Conversion complete!")
+        logger.info(f"  📊 Total time: {total_elapsed:.1f}s")
+        logger.info(f"  📈 Success rate: {successful_conversions}/{len(pdf_files)} ({(successful_conversions / len(pdf_files) * 100):.1f}%)")
+        logger.info(f"  📝 Total output: {total_chars:,} characters")
+        logger.info(f"  ⚡ Average speed: {total_chars / total_elapsed:.0f} chars/s")
+
+        return results
+
+    async def _run_async(self) -> List[ConversionResult]:
+        """Execute asynchronous conversion with parallel processing."""
+        pdf_files, output_base, is_dir = self._prepare_files()
+
+        converter = PdfToMarkdown(
+            chatterer=self.chatterer.unwrap(),
+            image_zoom=self.image_zoom,
+            image_format=self.image_format,
+            image_jpg_quality=self.image_quality,
+            context_tail_lines=self.context_tail_lines,
+        )
+
+        total_start_time = time.time()
+
+        logger.info(f"🚀 Starting ASYNC parallel conversion of {len(pdf_files)} PDF(s)...")
+        logger.info(f"⚡ Max concurrent: {self.max_concurrent} LLM requests")
+
+        # Process PDFs concurrently
+        semaphore = asyncio.Semaphore(self.max_concurrent)
+
+        async def process_pdf(pdf: Path, index: int) -> ConversionResult:
+            async with semaphore:
+                output_path = (output_base / f"{pdf.stem}.md") if is_dir else output_base
+
+                logger.info(f"📄 Processing {index}/{len(pdf_files)}: {pdf.name}")
+                start_time = time.time()
+
+                # Progress callback for individual PDF
+                def progress_callback(current: int, total: int) -> None:
+                    progress = (current / total) * 100
+                    logger.info(f"  └─ {pdf.name}: {current}/{total} pages ({progress:.1f}%)")
+
+                try:
+                    markdown = await converter.aconvert(
+                        pdf_input=str(pdf),
+                        page_indices=self.page,
+                        progress_callback=progress_callback,
+                        max_concurrent=self.max_concurrent,  # Limit per-PDF concurrency
+                    )
+
+                    # Save result
+                    output_path.parent.mkdir(parents=True, exist_ok=True)
+                    output_path.write_text(markdown, encoding="utf-8")
+
+                    elapsed = time.time() - start_time
+                    chars_per_sec = len(markdown) / elapsed if elapsed > 0 else 0
+
+                    logger.info(f"  ✅ {pdf.name} completed in {elapsed:.1f}s ({chars_per_sec:.0f} chars/s)")
+                    logger.info(f"  📝 Generated {len(markdown):,} characters → {output_path}")
+
+                    return {
+                        "input": pdf.as_posix(),
+                        "output": output_path.as_posix(),
+                        "result": markdown,
+                        "processing_time": elapsed,
+                        "characters": len(markdown),
+                    }
+
+                except Exception as e:
+                    logger.error(f"  ❌ Failed to process {pdf.name}: {e}")
+                    return {
+                        "input": pdf.as_posix(),
+                        "output": "",
+                        "result": "",
+                        "error": str(e),
+                    }
+
+        # Execute all PDF processing tasks
+        tasks = [process_pdf(pdf, i) for i, pdf in enumerate(pdf_files, 1)]
+        raw_results = await asyncio.gather(*tasks, return_exceptions=True)
+
+        # Handle exceptions in results
+        final_results: List[ConversionResult] = []
+        for result in raw_results:
+            if isinstance(result, Exception):
+                logger.error(f"Task failed with exception: {result}")
+                final_results.append(ConversionResult(input="", output="", result="", error=str(result)))
+            else:
+                # Type narrowing: result is ConversionResult after isinstance check
+                final_results.append(result)  # type: ignore[arg-type]
+
+        total_elapsed = time.time() - total_start_time
+        total_chars = sum(len(r.get("result", "")) for r in final_results)
+        successful_conversions = sum(1 for r in final_results if "error" not in r)
+
+        logger.info("🎉 ASYNC conversion complete!")
+        logger.info(f"  📊 Total time: {total_elapsed:.1f}s")
+        logger.info(f"  📈 Success rate: {successful_conversions}/{len(pdf_files)} ({(successful_conversions / len(pdf_files) * 100):.1f}%)")
+        logger.info(f"  📝 Total output: {total_chars:,} characters")
+        logger.info(f"  ⚡ Average speed: {total_chars / total_elapsed:.0f} chars/s")
+        logger.info(f"  🚀 Speedup: ~{len(pdf_files) / max(1, total_elapsed / 60):.1f}x faster than sequential")
+
+        return final_results
+
+    def _prepare_files(self) -> tuple[List[Path], Path, bool]:
+        """Prepare input and output file paths."""
+        input_path = Path(self.PDF_OR_DIRECTORY_PATH).resolve()
+        pdf_files: List[Path] = []
+        is_dir = False
+
+        # Determine input files
+        if input_path.is_file():
+            if input_path.suffix.lower() != ".pdf":
+                logger.error(f"❌ Input file must be a PDF: {input_path}")
+                sys.exit(1)
+            pdf_files.append(input_path)
+        elif input_path.is_dir():
+            is_dir = True
+            pattern = "**/*.pdf" if self.recursive else "*.pdf"
+            pdf_files = sorted([f for f in input_path.glob(pattern) if f.is_file()])
+            if not pdf_files:
+                logger.warning(f"⚠️  No PDF files found in {input_path}")
+                sys.exit(0)
+        else:
+            logger.error(f"❌ Input path does not exist: {input_path}")
+            sys.exit(1)
+
+        # Determine output path
+        if self.output:
+            output_base = Path(self.output).resolve()
+        elif is_dir:
+            output_base = input_path
+        else:
+            output_base = input_path.with_suffix(".md")
+
+        # Create output directories
+        if is_dir:
+            output_base.mkdir(parents=True, exist_ok=True)
+        else:
+            output_base.parent.mkdir(parents=True, exist_ok=True)
+
+        logger.info(f"📂 Input: {input_path}")
+        logger.info(f"📁 Output: {output_base}")
+        logger.info(f"📄 Found {len(pdf_files)} PDF file(s)")
+
+        return pdf_files, output_base, is_dir
+
+
+def main() -> None:
+    """Main entry point for the CLI application."""
+    args = None
+    try:
+        args = Arguments()
+        args.run()
+    except KeyboardInterrupt:
+        logger.info("🛑 Conversion interrupted by user")
+        sys.exit(130)
+    except Exception as e:
+        logger.error(f"❌ Unexpected error: {e}")
+        if args and hasattr(args, "verbose") and args.verbose:
+            import traceback
+
+            traceback.print_exc()
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

chatterer-0.1.22/chatterer/examples/pdf_to_text.py → chatterer-0.1.24/chatterer/examples/pdf2txt.py

@@ -3,15 +3,15 @@ import sys
 from pathlib import Path
 from typing import Optional
 
-from spargear import BaseArguments
+from spargear import RunnableArguments
 
 from chatterer.tools.convert_to_text import pdf_to_text
 
 logger = logging.getLogger(__name__)
 
 
-class PdfToTextArgs(BaseArguments):
-    input: Path
+class Arguments(RunnableArguments[None]):
+    PDF_PATH: Path
     """Path to the PDF file to convert to text."""
     output: Optional[Path]
     """Path to the output text file. If not provided, defaults to the input file with a .txt suffix."""
@@ -19,7 +19,7 @@ class PdfToTextArgs(BaseArguments):
     """Comma-separated list of zero-based page indices to extract from the PDF. Supports ranges, e.g., '0,2,4-8'."""
 
     def run(self) -> None:
-        input = self.input.resolve()
+        input = self.PDF_PATH.resolve()
         out = self.output or input.with_suffix(".txt")
         if not input.is_file():
             sys.exit(1)
@@ -47,7 +47,7 @@ def parse_page_indices(pages_str: str) -> list[int]:
 
 
 def main() -> None:
-    PdfToTextArgs().run()
+    Arguments().run()
 
 
 if __name__ == "__main__":

chatterer-0.1.22/chatterer/examples/make_ppt.py → chatterer-0.1.24/chatterer/examples/ppt.py

@@ -3,7 +3,7 @@ import sys
 from pathlib import Path
 from typing import NotRequired, TypedDict
 
-from spargear import BaseArguments
+from spargear import RunnableArguments
 
 from chatterer import BaseMessage, Chatterer, HumanMessage, SystemMessage
 
@@ -155,7 +155,7 @@ Now, generate the final `presentation.html` file using impress.js and the provid
 # --- Argument Parsing ---
 
 
-class MakePptArguments(BaseArguments):
+class Arguments(RunnableArguments[None]):
     """
     Arguments for the presentation generation process.
     """
@@ -179,9 +179,7 @@ class MakePptArguments(BaseArguments):
     """Prompt for organizing slides into a presentation script"""
 
     # LLM Settings
-    provider: str = (
-        "openai:gpt-4.1"  # Example: "openai:gpt-4o", "anthropic:claude-3-haiku-20240307", "google:gemini-1.5-flash"
-    )
+    provider: str = "openai:gpt-4.1"  # Example: "openai:gpt-4o", "anthropic:claude-3-haiku-20240307", "google:gemini-1.5-flash"
     """Name of the language model to use (provider:model_name)"""
 
     # Other settings
@@ -293,7 +291,7 @@ class GeneratedSlide(TypedDict):
     script: NotRequired[str]
 
 
-def run_presentation_agent(args: MakePptArguments):
+def run_presentation_agent(args: Arguments):
     """Executes the presentation generation agent loop."""
 
     if args.verbose:
@@ -481,7 +479,7 @@ Remember to follow all instructions in the role prompt, especially regarding HTM
 
 
 def main() -> None:
-    MakePptArguments().run()
+    Arguments().run()
 
 
 if __name__ == "__main__":