chatterer 0.1.18__py3-none-any.whl → 0.1.19__py3-none-any.whl

chatterer/examples/pdf_to_markdown.py

@@ -1,107 +1,107 @@
-def resolve_import_path_and_get_logger():
-    # ruff: noqa: E402
-    import logging
-    import sys
-
-    if __name__ == "__main__" and "." not in sys.path:
-        sys.path.append(".")
-
-    logger = logging.getLogger(__name__)
-    return logger
-
-
-logger = resolve_import_path_and_get_logger()
-import sys
-from pathlib import Path
-from typing import Optional
-
-from spargear import ArgumentSpec, BaseArguments
-
-from chatterer import Chatterer, PdfToMarkdown
-
-
-class PdfToMarkdownArgs(BaseArguments):
-    in_path: ArgumentSpec[str] = ArgumentSpec(
-        ["in-path"], help="Path to the input PDF file or a directory containing PDF files."
-    )
-    out_path: Optional[str] = None
-    """Output path. For a file, path to the output markdown file. For a directory, output directory for .md files."""
-    chatterer: ArgumentSpec[Chatterer] = ArgumentSpec(
-        ["--chatterer"],
-        default=None,
-        help="Chatterer instance for communication.",
-        type=Chatterer.from_provider,
-        required=True,
-    )
-    pages: Optional[str] = None
-    """Page indices to convert (e.g., '1,3,5-9')."""
-    recursive: bool = False
-    """If input is a directory, search for PDFs recursively."""
-
-    def run(self) -> list[dict[str, str]]:
-        in_path = Path(self.in_path.unwrap()).resolve()
-        page_indices = parse_page_indices(self.pages) if self.pages else None
-        pdf_files: list[Path] = []
-        is_dir = False
-        if in_path.is_file():
-            if in_path.suffix.lower() != ".pdf":
-                sys.exit(1)
-            pdf_files.append(in_path)
-        elif in_path.is_dir():
-            is_dir = True
-            pattern = "*.pdf"
-            pdf_files = sorted([
-                f for f in (in_path.rglob(pattern) if self.recursive else in_path.glob(pattern)) if f.is_file()
-            ])
-            if not pdf_files:
-                sys.exit(0)
-        else:
-            sys.exit(1)
-        if self.out_path:
-            out_base = Path(self.out_path).resolve()
-        elif is_dir:
-            out_base = in_path
-        else:
-            out_base = in_path.with_suffix(".md")
-
-        if is_dir:
-            out_base.mkdir(parents=True, exist_ok=True)
-        else:
-            out_base.parent.mkdir(parents=True, exist_ok=True)
-
-        converter = PdfToMarkdown(chatterer=self.chatterer.unwrap())
-        results: list[dict[str, str]] = []
-        for pdf in pdf_files:
-            out_path = (out_base / (pdf.stem + ".md")) if is_dir else out_base
-            md = converter.convert(str(pdf), page_indices)
-            out_path.parent.mkdir(parents=True, exist_ok=True)
-            out_path.write_text(md, encoding="utf-8")
-            results.append({"input": pdf.as_posix(), "output": out_path.as_posix(), "result": md})
-        logger.info(f"Converted {len(pdf_files)} PDF(s) to markdown and saved to `{out_base}`.")
-        return results
-
-
-def parse_page_indices(pages_str: str) -> list[int] | None:
-    if not pages_str:
-        return None
-    indices: set[int] = set()
-    for part in pages_str.split(","):
-        part = part.strip()
-        if not part:
-            continue
-        if "-" in part:
-            start_str, end_str = part.split("-", 1)
-            start = int(start_str.strip())
-            end = int(end_str.strip())
-            if start > end:
-                raise ValueError
-            indices.update(range(start, end + 1))
-        else:
-            indices.add(int(part))
-    if not indices:
-        raise ValueError
-    return sorted(indices)
-
-
-if __name__ == "__main__":
-    PdfToMarkdownArgs().run()
+def resolve_import_path_and_get_logger():
+    # ruff: noqa: E402
+    import logging
+    import sys
+
+    if __name__ == "__main__" and "." not in sys.path:
+        sys.path.append(".")
+
+    logger = logging.getLogger(__name__)
+    return logger
+
+
+logger = resolve_import_path_and_get_logger()
+import sys
+from pathlib import Path
+from typing import Optional
+
+from spargear import ArgumentSpec, BaseArguments
+
+from chatterer import Chatterer, PdfToMarkdown
+
+
+class PdfToMarkdownArgs(BaseArguments):
+    in_path: ArgumentSpec[str] = ArgumentSpec(["in-path"], help="Path to the input PDF file or a directory containing PDF files.")
+    out_path: Optional[str] = None
+    """Output path. For a file, path to the output markdown file. For a directory, output directory for .md files."""
+    chatterer: ArgumentSpec[Chatterer] = ArgumentSpec(
+        ["--chatterer"],
+        default=None,
+        help="Chatterer instance for communication.",
+        type=Chatterer.from_provider,
+        required=True,
+    )
+    pages: Optional[str] = None
+    """Page indices to convert (e.g., '1,3,5-9')."""
+    recursive: bool = False
+    """If input is a directory, search for PDFs recursively."""
+
+    def run(self) -> list[dict[str, str]]:
+        in_path = Path(self.in_path.unwrap()).resolve()
+        page_indices = parse_page_indices(self.pages) if self.pages else None
+        pdf_files: list[Path] = []
+        is_dir = False
+        if in_path.is_file():
+            if in_path.suffix.lower() != ".pdf":
+                sys.exit(1)
+            pdf_files.append(in_path)
+        elif in_path.is_dir():
+            is_dir = True
+            pattern = "*.pdf"
+            pdf_files = sorted([f for f in (in_path.rglob(pattern) if self.recursive else in_path.glob(pattern)) if f.is_file()])
+            if not pdf_files:
+                sys.exit(0)
+        else:
+            sys.exit(1)
+        if self.out_path:
+            out_base = Path(self.out_path).resolve()
+        elif is_dir:
+            out_base = in_path
+        else:
+            out_base = in_path.with_suffix(".md")
+
+        if is_dir:
+            out_base.mkdir(parents=True, exist_ok=True)
+        else:
+            out_base.parent.mkdir(parents=True, exist_ok=True)
+
+        converter = PdfToMarkdown(chatterer=self.chatterer.unwrap())
+        results: list[dict[str, str]] = []
+        for pdf in pdf_files:
+            out_path = (out_base / (pdf.stem + ".md")) if is_dir else out_base
+            md = converter.convert(str(pdf), page_indices)
+            out_path.parent.mkdir(parents=True, exist_ok=True)
+            out_path.write_text(md, encoding="utf-8")
+            results.append({"input": pdf.as_posix(), "output": out_path.as_posix(), "result": md})
+        logger.info(f"Converted {len(pdf_files)} PDF(s) to markdown and saved to `{out_base}`.")
+        return results
+
+
+def parse_page_indices(pages_str: str) -> list[int] | None:
+    if not pages_str:
+        return None
+    indices: set[int] = set()
+    for part in pages_str.split(","):
+        part = part.strip()
+        if not part:
+            continue
+        if "-" in part:
+            start_str, end_str = part.split("-", 1)
+            start = int(start_str.strip())
+            end = int(end_str.strip())
+            if start > end:
+                raise ValueError
+            indices.update(range(start, end + 1))
+        else:
+            indices.add(int(part))
+    if not indices:
+        raise ValueError
+    return sorted(indices)
+
+
+def main() -> None:
+    PdfToMarkdownArgs().run()
+
+
+if __name__ == "__main__":
+    main()

chatterer/examples/pdf_to_text.py

@@ -1,56 +1,60 @@
-def resolve_import_path_and_get_logger():
-    # ruff: noqa: E402
-    import logging
-    import sys
-
-    if __name__ == "__main__" and "." not in sys.path:
-        sys.path.append(".")
-
-    logger = logging.getLogger(__name__)
-    return logger
-
-
-logger = resolve_import_path_and_get_logger()
-import sys
-from pathlib import Path
-
-from spargear import ArgumentSpec, BaseArguments
-
-from chatterer.tools.convert_to_text import pdf_to_text
-
-
-class PdfToTextArgs(BaseArguments):
-    in_path: ArgumentSpec[Path] = ArgumentSpec(["in-path"], help="Path to the PDF file.")
-    out_path: ArgumentSpec[Path] = ArgumentSpec(["--out-path"], default=None, help="Output file path.")
-    pages: ArgumentSpec[str] = ArgumentSpec(["--pages"], default=None, help="Page indices to extract, e.g. '1,3,5-9'.")
-
-    def run(self) -> None:
-        input = self.in_path.unwrap().resolve()
-        out = self.out_path.value or input.with_suffix(".txt")
-        if not input.is_file():
-            sys.exit(1)
-        out.write_text(
-            pdf_to_text(input, parse_page_indices(pages_arg) if (pages_arg := self.pages.value) else None),
-            encoding="utf-8",
-        )
-        logger.info(f"Extracted text from `{input}` to `{out}`")
-
-
-def parse_page_indices(pages_str: str) -> list[int]:
-    indices: set[int] = set()
-    for part in pages_str.split(","):
-        part = part.strip()
-        if "-" in part:
-            start_str, end_str = part.split("-", 1)
-            start = int(start_str)
-            end = int(end_str)
-            if start > end:
-                raise ValueError
-            indices.update(range(start, end + 1))
-        else:
-            indices.add(int(part))
-    return sorted(indices)
-
-
-if __name__ == "__main__":
-    PdfToTextArgs().run()
+def resolve_import_path_and_get_logger():
+    # ruff: noqa: E402
+    import logging
+    import sys
+
+    if __name__ == "__main__" and "." not in sys.path:
+        sys.path.append(".")
+
+    logger = logging.getLogger(__name__)
+    return logger
+
+
+logger = resolve_import_path_and_get_logger()
+import sys
+from pathlib import Path
+
+from spargear import ArgumentSpec, BaseArguments
+
+from chatterer.tools.convert_to_text import pdf_to_text
+
+
+class PdfToTextArgs(BaseArguments):
+    in_path: ArgumentSpec[Path] = ArgumentSpec(["in-path"], help="Path to the PDF file.")
+    out_path: ArgumentSpec[Path] = ArgumentSpec(["--out-path"], default=None, help="Output file path.")
+    pages: ArgumentSpec[str] = ArgumentSpec(["--pages"], default=None, help="Page indices to extract, e.g. '1,3,5-9'.")
+
+    def run(self) -> None:
+        input = self.in_path.unwrap().resolve()
+        out = self.out_path.value or input.with_suffix(".txt")
+        if not input.is_file():
+            sys.exit(1)
+        out.write_text(
+            pdf_to_text(input, parse_page_indices(pages_arg) if (pages_arg := self.pages.value) else None),
+            encoding="utf-8",
+        )
+        logger.info(f"Extracted text from `{input}` to `{out}`")
+
+
+def parse_page_indices(pages_str: str) -> list[int]:
+    indices: set[int] = set()
+    for part in pages_str.split(","):
+        part = part.strip()
+        if "-" in part:
+            start_str, end_str = part.split("-", 1)
+            start = int(start_str)
+            end = int(end_str)
+            if start > end:
+                raise ValueError
+            indices.update(range(start, end + 1))
+        else:
+            indices.add(int(part))
+    return sorted(indices)
+
+
+def main() -> None:
+    PdfToTextArgs().run()
+
+
+if __name__ == "__main__":
+    main()

chatterer/examples/transcription_api.py

@@ -1,123 +1,127 @@
-# pyright: reportUnknownVariableType=false, reportUnknownMemberType=false, reportArgumentType=false, reportMissingTypeStubs=false
-
-from io import BytesIO
-from pathlib import Path
-from typing import cast
-
-from openai import OpenAI
-from pydub import AudioSegment
-from spargear import ArgumentSpec, BaseArguments
-
-# Maximum chunk length in seconds
-MAX_CHUNK_DURATION = 600
-
-
-class TranscriptionApiArguments(BaseArguments):
-    in_path = ArgumentSpec(
-        ["in-path"],
-        type=Path,
-        help="The audio file to transcribe.",
-    )
-    out_path = ArgumentSpec(
-        ["--out-path"],
-        type=Path,
-        default=None,
-        help="Path to save the transcription output.",
-    )
-    model: ArgumentSpec[str] = ArgumentSpec(
-        ["--model"],
-        default="gpt-4o-transcribe",
-        help="The model to use for transcription.",
-    )
-    api_key: ArgumentSpec[str] = ArgumentSpec(
-        ["--api-key"],
-        default=None,
-        help="The API key for authentication.",
-    )
-    base_url: ArgumentSpec[str] = ArgumentSpec(
-        ["--base-url"],
-        default="https://api.openai.com/v1",
-        help="The base URL for the API.",
-    )
-
-    def run(self) -> None:
-        audio_path = self.in_path.unwrap()
-        model = self.model.unwrap()
-
-        client = OpenAI(api_key=self.api_key.value, base_url=self.base_url.value)
-
-        audio = load_audio_segment(audio_path)
-
-        segments = split_audio(audio, MAX_CHUNK_DURATION)
-        print(f"[i] Audio duration: {len(audio) / 1000:.1f}s; splitting into {len(segments)} segment(s)")
-
-        transcripts: list[str] = []
-        for idx, seg in enumerate(segments, start=1):
-            print(f"[i] Transcribing segment {idx}/{len(segments)}...")
-            transcripts.append(transcribe_segment(seg, client, model))
-
-        full_transcript = "\n\n".join(transcripts)
-        output_path: Path = self.out_path.value or audio_path.with_suffix(".txt")
-        output_path.write_text(full_transcript, encoding="utf-8")
-        print(f"[✓] Transcription saved to: {output_path}")
-
-
-def load_audio_segment(file_path: Path) -> AudioSegment:
-    """
-    Load an audio file as an AudioSegment. Convert to mp3 format in-memory if needed.
-    """
-    ext = file_path.suffix.lower()[1:]
-    audio = AudioSegment.from_file(file_path.as_posix(), format=ext if ext != "mp3" else None)
-    if ext != "mp3":
-        buffer = BytesIO()
-        audio.export(buffer, format="mp3")
-        buffer.seek(0)
-        audio = AudioSegment.from_file(buffer, format="mp3")
-    return audio
-
-
-def split_audio(audio: AudioSegment, max_duration_s: int) -> list[AudioSegment]:
-    """
-    Split the AudioSegment into chunks no longer than max_duration_s seconds.
-    """
-    chunk_length_ms = (max_duration_s - 1) * 1000
-    duration_ms = len(audio)
-    segments: list[AudioSegment] = []
-    segment_idx: int = 0
-    for start_ms in range(0, duration_ms, chunk_length_ms):
-        end_ms = min(start_ms + chunk_length_ms, duration_ms)
-        segment = cast(AudioSegment, audio[start_ms:end_ms])
-        segments.append(segment)
-        # with open(f"segment_{segment_idx}.mp3", "wb") as f:
-        #     segment.export(f, format="mp3")
-        segment_idx += 1
-    return segments
-
-
-def transcribe_segment(segment: AudioSegment, client: OpenAI, model: str) -> str:
-    """
-    Transcribe a single AudioSegment chunk and return its text.
-    """
-    buffer = BytesIO()
-    segment.export(buffer, format="mp3")
-    buffer.seek(0)
-    mp3_bytes = buffer.read()
-    response = client.audio.transcriptions.create(
-        model=model,
-        prompt="Transcribe whole text from audio.",
-        file=("audio.mp3", mp3_bytes),
-        response_format="text",
-        stream=True,
-    )
-    for res in response:
-        if res.type == "transcript.text.delta":
-            print(res.delta, end="", flush=True)
-        if res.type == "transcript.text.done":
-            print()
-            return res.text
-    else:
-        raise RuntimeError("No transcription result found.")
-
-
-if __name__ == "__main__":
-    TranscriptionApiArguments().run()
+# pyright: reportUnknownVariableType=false, reportUnknownMemberType=false, reportArgumentType=false, reportMissingTypeStubs=false
+
+from io import BytesIO
+from pathlib import Path
+from typing import cast
+
+from openai import OpenAI
+from pydub import AudioSegment
+from spargear import ArgumentSpec, BaseArguments
+
+# Maximum chunk length in seconds
+MAX_CHUNK_DURATION = 600
+
+
+class TranscriptionApiArguments(BaseArguments):
+    in_path = ArgumentSpec(
+        ["in-path"],
+        type=Path,
+        help="The audio file to transcribe.",
+    )
+    out_path = ArgumentSpec(
+        ["--out-path"],
+        type=Path,
+        default=None,
+        help="Path to save the transcription output.",
+    )
+    model: ArgumentSpec[str] = ArgumentSpec(
+        ["--model"],
+        default="gpt-4o-transcribe",
+        help="The model to use for transcription.",
+    )
+    api_key: ArgumentSpec[str] = ArgumentSpec(
+        ["--api-key"],
+        default=None,
+        help="The API key for authentication.",
+    )
+    base_url: ArgumentSpec[str] = ArgumentSpec(
+        ["--base-url"],
+        default="https://api.openai.com/v1",
+        help="The base URL for the API.",
+    )
+
+    def run(self) -> None:
+        audio_path = self.in_path.unwrap()
+        model = self.model.unwrap()
+
+        client = OpenAI(api_key=self.api_key.value, base_url=self.base_url.value)
+
+        audio = load_audio_segment(audio_path)
+
+        segments = split_audio(audio, MAX_CHUNK_DURATION)
+        print(f"[i] Audio duration: {len(audio) / 1000:.1f}s; splitting into {len(segments)} segment(s)")
+
+        transcripts: list[str] = []
+        for idx, seg in enumerate(segments, start=1):
+            print(f"[i] Transcribing segment {idx}/{len(segments)}...")
+            transcripts.append(transcribe_segment(seg, client, model))
+
+        full_transcript = "\n\n".join(transcripts)
+        output_path: Path = self.out_path.value or audio_path.with_suffix(".txt")
+        output_path.write_text(full_transcript, encoding="utf-8")
+        print(f"[✓] Transcription saved to: {output_path}")
+
+
+def load_audio_segment(file_path: Path) -> AudioSegment:
+    """
+    Load an audio file as an AudioSegment. Convert to mp3 format in-memory if needed.
+    """
+    ext = file_path.suffix.lower()[1:]
+    audio = AudioSegment.from_file(file_path.as_posix(), format=ext if ext != "mp3" else None)
+    if ext != "mp3":
+        buffer = BytesIO()
+        audio.export(buffer, format="mp3")
+        buffer.seek(0)
+        audio = AudioSegment.from_file(buffer, format="mp3")
+    return audio
+
+
+def split_audio(audio: AudioSegment, max_duration_s: int) -> list[AudioSegment]:
+    """
+    Split the AudioSegment into chunks no longer than max_duration_s seconds.
+    """
+    chunk_length_ms = (max_duration_s - 1) * 1000
+    duration_ms = len(audio)
+    segments: list[AudioSegment] = []
+    segment_idx: int = 0
+    for start_ms in range(0, duration_ms, chunk_length_ms):
+        end_ms = min(start_ms + chunk_length_ms, duration_ms)
+        segment = cast(AudioSegment, audio[start_ms:end_ms])
+        segments.append(segment)
+        # with open(f"segment_{segment_idx}.mp3", "wb") as f:
+        #     segment.export(f, format="mp3")
+        segment_idx += 1
+    return segments
+
+
+def transcribe_segment(segment: AudioSegment, client: OpenAI, model: str) -> str:
+    """
+    Transcribe a single AudioSegment chunk and return its text.
+    """
+    buffer = BytesIO()
+    segment.export(buffer, format="mp3")
+    buffer.seek(0)
+    mp3_bytes = buffer.read()
+    response = client.audio.transcriptions.create(
+        model=model,
+        prompt="Transcribe whole text from audio.",
+        file=("audio.mp3", mp3_bytes),
+        response_format="text",
+        stream=True,
+    )
+    for res in response:
+        if res.type == "transcript.text.delta":
+            print(res.delta, end="", flush=True)
+        if res.type == "transcript.text.done":
+            print()
+            return res.text
+    else:
+        raise RuntimeError("No transcription result found.")
+
+
+def main() -> None:
+    TranscriptionApiArguments().run()
+
+
+if __name__ == "__main__":
+    main()