chatterer 0.1.24__py3-none-any.whl → 0.1.26__py3-none-any.whl

chatterer/examples/transcribe.py

@@ -1,112 +1,192 @@
-# pyright: reportUnknownVariableType=false, reportUnknownMemberType=false, reportArgumentType=false, reportMissingTypeStubs=false
-
-from io import BytesIO
-from pathlib import Path
-from typing import Optional, cast
-
-from openai import OpenAI
-from pydub import AudioSegment
-from spargear import RunnableArguments
-
-# Maximum chunk length in seconds
-MAX_CHUNK_DURATION = 600
-
-
-class Arguments(RunnableArguments[None]):
-    AUDIO_PATH: Path
-    """The audio file to transcribe."""
-    output: Optional[Path] = None
-    """Path to save the transcription output."""
-    model: str = "gpt-4o-transcribe"
-    """The model to use for transcription."""
-    api_key: Optional[str] = None
-    """The API key for authentication."""
-    base_url: str = "https://api.openai.com/v1"
-    """The base URL for the API."""
-    prompt: str = "Transcribe whole text from audio."
-    """The prompt to use for transcription."""
-
-    def run(self) -> None:
-        model = self.model
-
-        client = OpenAI(api_key=self.api_key, base_url=self.base_url)
-
-        audio = load_audio_segment(self.AUDIO_PATH)
-
-        segments = split_audio(audio, MAX_CHUNK_DURATION)
-        print(f"[i] Audio duration: {len(audio) / 1000:.1f}s; splitting into {len(segments)} segment(s)")
-
-        transcripts: list[str] = []
-        for idx, seg in enumerate(segments, start=1):
-            print(f"[i] Transcribing segment {idx}/{len(segments)}...")
-            transcripts.append(transcribe_segment(seg, client, model, self.prompt))
-
-        full_transcript = "\n\n".join(transcripts)
-        output_path: Path = self.output or self.AUDIO_PATH.with_suffix(".txt")
-        output_path.write_text(full_transcript, encoding="utf-8")
-        print(f"[✓] Transcription saved to: {output_path}")
-
-
-def load_audio_segment(file_path: Path) -> AudioSegment:
-    """
-    Load an audio file as an AudioSegment. Convert to mp3 format in-memory if needed.
-    """
-    ext = file_path.suffix.lower()[1:]
-    audio = AudioSegment.from_file(file_path.as_posix(), format=ext if ext != "mp3" else None)
-    if ext != "mp3":
-        buffer = BytesIO()
-        audio.export(buffer, format="mp3")
-        buffer.seek(0)
-        audio = AudioSegment.from_file(buffer, format="mp3")
-    return audio
-
-
-def split_audio(audio: AudioSegment, max_duration_s: int) -> list[AudioSegment]:
-    """
-    Split the AudioSegment into chunks no longer than max_duration_s seconds.
-    """
-    chunk_length_ms = (max_duration_s - 1) * 1000
-    duration_ms = len(audio)
-    segments: list[AudioSegment] = []
-    segment_idx: int = 0
-    for start_ms in range(0, duration_ms, chunk_length_ms):
-        end_ms = min(start_ms + chunk_length_ms, duration_ms)
-        segment = cast(AudioSegment, audio[start_ms:end_ms])
-        segments.append(segment)
-        # with open(f"segment_{segment_idx}.mp3", "wb") as f:
-        #     segment.export(f, format="mp3")
-        segment_idx += 1
-    return segments
-
-
-def transcribe_segment(segment: AudioSegment, client: OpenAI, model: str, prompt: str) -> str:
-    """
-    Transcribe a single AudioSegment chunk and return its text.
-    """
-    buffer = BytesIO()
-    segment.export(buffer, format="mp3")
-    buffer.seek(0)
-    mp3_bytes = buffer.read()
-    response = client.audio.transcriptions.create(
-        model=model,
-        prompt=prompt,
-        file=("audio.mp3", mp3_bytes),
-        response_format="text",
-        stream=True,
-    )
-    for res in response:
-        if res.type == "transcript.text.delta":
-            print(res.delta, end="", flush=True)
-        if res.type == "transcript.text.done":
-            print()
-            return res.text
-    else:
-        raise RuntimeError("No transcription result found.")
-
-
-def main() -> None:
-    Arguments().run()
-
-
-if __name__ == "__main__":
-    main()
+# pyright: reportUnknownVariableType=false, reportUnknownMemberType=false, reportArgumentType=false, reportMissingTypeStubs=false
+
+from io import BytesIO
+from pathlib import Path
+from typing import List, Optional
+
+from openai import OpenAI
+from pydub import AudioSegment
+from spargear import RunnableArguments
+
+
+# -------------------------------------------------------------------
+# Helper functions for timestamp parsing & segment selection
+# -------------------------------------------------------------------
+def parse_timestamp(ts: str) -> float:
+    """
+    Parse a timestamp string into seconds.
+    Supports:
+      - "SS" or "SS.sss"
+      - "MM:SS" or "MM:SS.sss"
+      - "HH:MM:SS" or "HH:MM:SS.sss"
+    """
+    parts = ts.split(":")
+    seconds = 0.0
+    for idx, part in enumerate(reversed(parts)):
+        if not part:
+            value = 0.0
+        else:
+            value = float(part)
+        if idx == 0:
+            seconds += value
+        elif idx == 1:
+            seconds += value * 60
+        elif idx == 2:
+            seconds += value * 3600
+        else:
+            raise ValueError(f"Timestamp '{ts}' is too long (use H:MM:SS at most)")
+    return seconds
+
+
+def get_selected_audio(audio: AudioSegment, segments_str: str) -> AudioSegment:
+    """
+    Given full audio and a segments string (e.g. "650-750,16:50-17:30,800-"),
+    extract those subranges and concatenate them.
+    """
+    duration_ms = len(audio)
+    duration_s = duration_ms / 1000.0
+    subsegments: List[AudioSegment] = []
+
+    for part in segments_str.split(","):
+        if "-" not in part:
+            raise ValueError(f"Invalid segment '{part}' (must contain '-')")
+        start_str, end_str = part.split("-", 1)
+        start_s = parse_timestamp(start_str) if start_str.strip() else 0.0
+        end_s = parse_timestamp(end_str) if end_str.strip() else duration_s
+
+        # clamp
+        start_s = max(0.0, min(start_s, duration_s))
+        end_s = max(0.0, min(end_s, duration_s))
+        if end_s <= start_s:
+            print(f"[!] Warning: segment '{part}' yields non-positive duration; skipping.")
+            continue
+
+        start_ms = int(start_s * 1000)
+        end_ms = int(end_s * 1000)
+        sub = audio[start_ms:end_ms]
+        subsegments.append(sub)
+        print(f"[i] Selected segment {start_s:.2f}s–{end_s:.2f}s ({end_s - start_s:.2f}s)")
+
+    if not subsegments:
+        raise RuntimeError("No valid segments were specified.")
+    # concatenate
+    combined = subsegments[0]
+    for seg in subsegments[1:]:
+        combined += seg
+    return combined
+
+
+# -------------------------------------------------------------------
+# Main transcription logic
+# -------------------------------------------------------------------
+class Arguments(RunnableArguments[None]):
+    AUDIO_PATH: Path
+    """The audio file to transcribe."""
+    output: Optional[Path] = None
+    """Path to save the transcription output."""
+    model: str = "gpt-4o-transcribe"
+    """The model to use for transcription."""
+    api_key: Optional[str] = None
+    """The API key for authentication."""
+    base_url: str = "https://api.openai.com/v1"
+    """The base URL for the API."""
+    prompt: str = "Transcribe whole text from audio."
+    """The prompt to use for transcription."""
+    segments: Optional[str] = None
+    """
+    Comma-separated list of time ranges to include (e.g. "650-750,16:50-17:30,800-").
+    Each range is start-end; start or end may be omitted.
+    Supports seconds or H:MM:SS formats.
+    """
+    max_chunk_duration: int = 600
+    """Maximum duration of each chunk in seconds."""
+
+    def run(self) -> None:
+        client = OpenAI(api_key=self.api_key, base_url=self.base_url)
+
+        # 1) Load entire audio
+        original_audio = load_audio_segment(self.AUDIO_PATH)
+
+        # 2) If segments specified, extract & combine
+        if self.segments:
+            audio = get_selected_audio(original_audio, self.segments)
+            print(f"[i] Combined audio duration: {len(audio) / 1000:.1f}s (from segments)")
+        else:
+            audio = original_audio
+            print(f"[i] Audio duration: {len(audio) / 1000:.1f}s (full audio)")
+
+        # 3) Split into chunks
+        segments = split_audio(audio, self.max_chunk_duration)
+        print(f"[i] Splitting into {len(segments)} segment(s) for transcription")
+
+        # 4) Transcribe each chunk
+        transcripts: List[str] = []
+        for idx, seg in enumerate(segments, start=1):
+            print(f"[i] Transcribing segment {idx}/{len(segments)}...")
+            transcripts.append(transcribe_segment(seg, client, self.model, self.prompt))
+
+        # 5) Write out
+        full = "\n\n".join(transcripts)
+        out_path = self.output or self.AUDIO_PATH.with_suffix(".txt")
+        out_path.write_text(full, encoding="utf-8")
+        print(f"[✓] Transcription saved to: {out_path}")
+
+
+def load_audio_segment(file_path: Path) -> AudioSegment:
+    """
+    Load an audio file as an AudioSegment. Convert to mp3 format in-memory if needed.
+    """
+    ext = file_path.suffix.lower()[1:]
+    audio = AudioSegment.from_file(file_path.as_posix(), format=None if ext == "mp3" else ext)
+    if ext != "mp3":
+        buffer = BytesIO()
+        audio.export(buffer, format="mp3")
+        buffer.seek(0)
+        audio = AudioSegment.from_file(buffer, format="mp3")
+    return audio
+
+
+def split_audio(audio: AudioSegment, max_duration_s: int) -> List[AudioSegment]:
+    """
+    Split the AudioSegment into chunks no longer than max_duration_s seconds.
+    """
+    chunk_ms = (max_duration_s - 1) * 1000
+    duration_ms = len(audio)
+    segments: List[AudioSegment] = []
+    for start in range(0, duration_ms, chunk_ms):
+        end = min(start + chunk_ms, duration_ms)
+        segments.append(audio[start:end])
+    return segments
+
+
+def transcribe_segment(segment: AudioSegment, client: OpenAI, model: str, prompt: str) -> str:
+    """
+    Transcribe a single AudioSegment chunk and return its text.
+    """
+    buffer = BytesIO()
+    segment.export(buffer, format="mp3")
+    buffer.seek(0)
+    mp3_bytes = buffer.read()
+
+    response = client.audio.transcriptions.create(
+        model=model,
+        prompt=prompt,
+        file=("audio.mp3", mp3_bytes),
+        response_format="text",
+        stream=True,
+    )
+    for res in response:
+        if res.type == "transcript.text.delta":
+            print(res.delta, end="", flush=True)
+        elif res.type == "transcript.text.done":
+            print()
+            return res.text
+    raise RuntimeError("No transcription result found.")
+
+
+def main() -> None:
+    Arguments().run()
+
+
+if __name__ == "__main__":
+    main()

chatterer/examples/upstage.py

@@ -1,89 +1,89 @@
-import logging
-from pathlib import Path
-from typing import Optional
-
-from langchain_core.documents.base import Blob
-from spargear import ArgumentSpec, BaseArguments
-
-from chatterer import Chatterer, UpstageDocumentParseParser
-from chatterer.tools.upstage_document_parser import (
-    DEFAULT_IMAGE_DIR,
-    DOCUMENT_PARSE_BASE_URL,
-    DOCUMENT_PARSE_DEFAULT_MODEL,
-    OCR,
-    Category,
-    OutputFormat,
-    SplitType,
-)
-
-logger = logging.getLogger(__name__)
-
-
-class Arguments(BaseArguments):
-    INPUT_PATH: Path
-    """Input file to parse. Can be a PDF, image, or other supported formats."""
-    output: Optional[Path] = None
-    """Output file path for the parsed content. Defaults to input file with .md suffix if not provided."""
-    api_key: Optional[str] = None
-    """API key for the Upstage API."""
-    base_url: str = DOCUMENT_PARSE_BASE_URL
-    """Base URL for the Upstage API."""
-    model: str = DOCUMENT_PARSE_DEFAULT_MODEL
-    """Model to use for parsing."""
-    split: SplitType = "none"
-    """Split type for the parsed content."""
-    ocr: OCR = "auto"
-    """OCR type for parsing."""
-    output_format: OutputFormat = "markdown"
-    """Output format for the parsed content."""
-    coordinates: bool = False
-    """Whether to include coordinates in the output."""
-    base64_encoding: list[Category] = ["figure"]
-    """Base64 encoding for specific categories in the parsed content."""
-    image_description_instruction: str = "Describe the image in detail."
-    """Instruction for generating image descriptions."""
-    image_dir: str = DEFAULT_IMAGE_DIR
-    """Directory to save images extracted from the document."""
-    chatterer: ArgumentSpec[Chatterer] = ArgumentSpec(
-        ["--chatterer"],
-        default=None,
-        help="Chatterer instance for communication.",
-        type=Chatterer.from_provider,
-    )
-
-    def run(self) -> None:
-        input = self.INPUT_PATH.resolve()
-        out = self.output or input.with_suffix(".md")
-
-        parser = UpstageDocumentParseParser(
-            api_key=self.api_key,
-            base_url=self.base_url,
-            model=self.model,
-            split=self.split,
-            ocr=self.ocr,
-            output_format=self.output_format,
-            coordinates=self.coordinates,
-            base64_encoding=self.base64_encoding,
-            image_description_instruction=self.image_description_instruction,
-            image_dir=self.image_dir,
-            chatterer=self.chatterer.value,
-        )
-        docs = parser.parse(Blob.from_path(input))  # pyright: ignore[reportUnknownMemberType]
-
-        if self.image_dir:
-            for path, image in parser.image_data.items():
-                (path := Path(path)).parent.mkdir(parents=True, exist_ok=True)
-                path.write_bytes(image)
-                logger.info(f"Saved image to `{path}`")
-
-        markdown: str = "\n\n".join(f"<!--- page {i} -->\n{doc.page_content}" for i, doc in enumerate(docs, 1))
-        out.write_text(markdown, encoding="utf-8")
-        logger.info(f"Parsed `{input}` to `{out}`")
-
-
-def main() -> None:
-    Arguments().run()
-
-
-if __name__ == "__main__":
-    main()
+import logging
+from pathlib import Path
+from typing import Optional
+
+from langchain_core.documents.base import Blob
+from spargear import ArgumentSpec, BaseArguments
+
+from chatterer import Chatterer, UpstageDocumentParseParser
+from chatterer.tools.upstage_document_parser import (
+    DEFAULT_IMAGE_DIR,
+    DOCUMENT_PARSE_BASE_URL,
+    DOCUMENT_PARSE_DEFAULT_MODEL,
+    OCR,
+    Category,
+    OutputFormat,
+    SplitType,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class Arguments(BaseArguments):
+    INPUT_PATH: Path
+    """Input file to parse. Can be a PDF, image, or other supported formats."""
+    output: Optional[Path] = None
+    """Output file path for the parsed content. Defaults to input file with .md suffix if not provided."""
+    api_key: Optional[str] = None
+    """API key for the Upstage API."""
+    base_url: str = DOCUMENT_PARSE_BASE_URL
+    """Base URL for the Upstage API."""
+    model: str = DOCUMENT_PARSE_DEFAULT_MODEL
+    """Model to use for parsing."""
+    split: SplitType = "none"
+    """Split type for the parsed content."""
+    ocr: OCR = "auto"
+    """OCR type for parsing."""
+    output_format: OutputFormat = "markdown"
+    """Output format for the parsed content."""
+    coordinates: bool = False
+    """Whether to include coordinates in the output."""
+    base64_encoding: list[Category] = ["figure"]
+    """Base64 encoding for specific categories in the parsed content."""
+    image_description_instruction: str = "Describe the image in detail."
+    """Instruction for generating image descriptions."""
+    image_dir: str = DEFAULT_IMAGE_DIR
+    """Directory to save images extracted from the document."""
+    chatterer: ArgumentSpec[Chatterer] = ArgumentSpec(
+        ["--chatterer"],
+        default=None,
+        help="Chatterer instance for communication.",
+        type=Chatterer.from_provider,
+    )
+
+    def run(self) -> None:
+        input = self.INPUT_PATH.resolve()
+        out = self.output or input.with_suffix(".md")
+
+        parser = UpstageDocumentParseParser(
+            api_key=self.api_key,
+            base_url=self.base_url,
+            model=self.model,
+            split=self.split,
+            ocr=self.ocr,
+            output_format=self.output_format,
+            coordinates=self.coordinates,
+            base64_encoding=self.base64_encoding,
+            image_description_instruction=self.image_description_instruction,
+            image_dir=self.image_dir,
+            chatterer=self.chatterer.value,
+        )
+        docs = parser.parse(Blob.from_path(input))  # pyright: ignore[reportUnknownMemberType]
+
+        if self.image_dir:
+            for path, image in parser.image_data.items():
+                (path := Path(path)).parent.mkdir(parents=True, exist_ok=True)
+                path.write_bytes(image)
+                logger.info(f"Saved image to `{path}`")
+
+        markdown: str = "\n\n".join(f"<!--- page {i} -->\n{doc.page_content}" for i, doc in enumerate(docs, 1))
+        out.write_text(markdown, encoding="utf-8")
+        logger.info(f"Parsed `{input}` to `{out}`")
+
+
+def main() -> None:
+    Arguments().run()
+
+
+if __name__ == "__main__":
+    main()

chatterer/examples/web2md.py

@@ -1,66 +1,80 @@
-from pathlib import Path
-from typing import Literal
-
-from spargear import ArgumentSpec, RunnableArguments
-
-from chatterer import Chatterer, MarkdownLink, PlayWrightBot
-
-
-class Arguments(RunnableArguments[None]):
-    URL: str
-    """The URL to crawl."""
-    output: str = Path(__file__).with_suffix(".md").as_posix()
-    """The output file path for the markdown file."""
-    chatterer: ArgumentSpec[Chatterer] = ArgumentSpec(
-        ["--chatterer"],
-        help="The Chatterer backend and model to use for filtering the markdown.",
-        type=Chatterer.from_provider,
-    )
-    engine: Literal["firefox", "chromium", "webkit"] = "firefox"
-    """The browser engine to use."""
-
-    def run(self) -> None:
-        chatterer = self.chatterer.value
-        url: str = self.URL.strip()
-        output: Path = Path(self.output).resolve()
-        with PlayWrightBot(chatterer=chatterer, engine=self.engine) as bot:
-            md = bot.url_to_md(url)
-            output.write_text(md, encoding="utf-8")
-            if chatterer is not None:
-                md_llm = bot.url_to_md_with_llm(url.strip())
-                output.write_text(md_llm, encoding="utf-8")
-            links = MarkdownLink.from_markdown(md, referer_url=url)
-            for link in links:
-                if link.type == "link":
-                    print(f"- [{truncate_string(link.url)}] {truncate_string(link.inline_text)} ({truncate_string(link.inline_title)})")
-                elif link.type == "image":
-                    print(f"- ![{truncate_string(link.url)}] ({truncate_string(link.inline_text)})")
-
-    async def arun(self) -> None:
-        chatterer = self.chatterer.value
-        url: str = self.URL.strip()
-        output: Path = Path(self.output).resolve()
-        async with PlayWrightBot(chatterer=chatterer, engine=self.engine) as bot:
-            md = await bot.aurl_to_md(url)
-            output.write_text(md, encoding="utf-8")
-            if chatterer is not None:
-                md_llm = await bot.aurl_to_md_with_llm(url.strip())
-                output.write_text(md_llm, encoding="utf-8")
-            links = MarkdownLink.from_markdown(md, referer_url=url)
-            for link in links:
-                if link.type == "link":
-                    print(f"- [{truncate_string(link.url)}] {truncate_string(link.inline_text)} ({truncate_string(link.inline_title)})")
-                elif link.type == "image":
-                    print(f"- ![{truncate_string(link.url)}] ({truncate_string(link.inline_text)})")
-
-
-def truncate_string(s: str) -> str:
-    return s[:50] + "..." if len(s) > 50 else s
-
-
-def main() -> None:
-    Arguments().run()
-
-
-if __name__ == "__main__":
-    main()
+from datetime import datetime
+from pathlib import Path
+from typing import Literal
+
+from spargear import ArgumentSpec, RunnableArguments
+
+from chatterer import Chatterer, MarkdownLink, PlayWrightBot
+
+
+def ouput_path_factory() -> Path:
+    """Factory function to generate a default output path for the markdown file."""
+    return Path(datetime.now().strftime("%Y%m%d_%H%M%S") + "_web2md.md").resolve()
+
+
+class Arguments(RunnableArguments[None]):
+    URL: str
+    """The URL to crawl."""
+    output: ArgumentSpec[Path] = ArgumentSpec(
+        ["--output", "-o"],
+        default_factory=ouput_path_factory,
+        help="The output file path for the markdown file.",
+    )
+    """The output file path for the markdown file."""
+    chatterer: ArgumentSpec[Chatterer] = ArgumentSpec(
+        ["--chatterer"],
+        help="The Chatterer backend and model to use for filtering the markdown.",
+        type=Chatterer.from_provider,
+    )
+    engine: Literal["firefox", "chromium", "webkit"] = "firefox"
+    """The browser engine to use."""
+
+    def run(self) -> None:
+        chatterer = self.chatterer.value
+        url: str = self.URL.strip()
+        output: Path = self.output.unwrap().resolve()
+        with PlayWrightBot(chatterer=chatterer, engine=self.engine) as bot:
+            md = bot.url_to_md(url)
+            output.write_text(md, encoding="utf-8")
+            if chatterer is not None:
+                md_llm = bot.url_to_md_with_llm(url.strip())
+                output.write_text(md_llm, encoding="utf-8")
+            links = MarkdownLink.from_markdown(md, referer_url=url)
+            for link in links:
+                if link.type == "link":
+                    print(
+                        f"- [{truncate_string(link.url)}] {truncate_string(link.inline_text)} ({truncate_string(link.inline_title)})"
+                    )
+                elif link.type == "image":
+                    print(f"- ![{truncate_string(link.url)}] ({truncate_string(link.inline_text)})")
+
+    async def arun(self) -> None:
+        chatterer = self.chatterer.value
+        url: str = self.URL.strip()
+        output: Path = self.output.unwrap().resolve()
+        async with PlayWrightBot(chatterer=chatterer, engine=self.engine) as bot:
+            md = await bot.aurl_to_md(url)
+            output.write_text(md, encoding="utf-8")
+            if chatterer is not None:
+                md_llm = await bot.aurl_to_md_with_llm(url.strip())
+                output.write_text(md_llm, encoding="utf-8")
+            links = MarkdownLink.from_markdown(md, referer_url=url)
+            for link in links:
+                if link.type == "link":
+                    print(
+                        f"- [{truncate_string(link.url)}] {truncate_string(link.inline_text)} ({truncate_string(link.inline_title)})"
+                    )
+                elif link.type == "image":
+                    print(f"- ![{truncate_string(link.url)}] ({truncate_string(link.inline_text)})")
+
+
+def truncate_string(s: str) -> str:
+    return s[:50] + "..." if len(s) > 50 else s
+
+
+def main() -> None:
+    Arguments().run()
+
+
+if __name__ == "__main__":
+    main()