chatterer 0.1.18__py3-none-any.whl → 0.1.19__py3-none-any.whl

chatterer/examples/upstage_parser.py

@@ -1,100 +1,95 @@
-def resolve_import_path_and_get_logger():
-    # ruff: noqa: E402
-    import logging
-    import sys
-
-    if __name__ == "__main__" and "." not in sys.path:
-        sys.path.append(".")
-
-    logger = logging.getLogger(__name__)
-    return logger
-
-
-logger = resolve_import_path_and_get_logger()
-from pathlib import Path
-
-from langchain_core.documents.base import Blob
-from spargear import ArgumentSpec, BaseArguments
-
-from chatterer import Chatterer, UpstageDocumentParseParser
-from chatterer.tools.upstage_document_parser import (
-    DEFAULT_IMAGE_DIR,
-    DOCUMENT_PARSE_BASE_URL,
-    DOCUMENT_PARSE_DEFAULT_MODEL,
-    OCR,
-    Category,
-    OutputFormat,
-    SplitType,
-)
-
-
-class UpstageParserArguments(BaseArguments):
-    in_path: ArgumentSpec[Path] = ArgumentSpec(["in-path"], help="Path to the input file.")
-    out_path: ArgumentSpec[Path] = ArgumentSpec(["--out-path"], default=None, help="Output file path.")
-    api_key: ArgumentSpec[str] = ArgumentSpec(["--api-key"], default=None, help="API key for the Upstage API.")
-    base_url: ArgumentSpec[str] = ArgumentSpec(
-        ["--base-url"], default=DOCUMENT_PARSE_BASE_URL, help="Base URL for the Upstage API."
-    )
-    model: ArgumentSpec[str] = ArgumentSpec(
-        ["--model"], default=DOCUMENT_PARSE_DEFAULT_MODEL, help="Model to use for parsing."
-    )
-    split: ArgumentSpec[SplitType] = ArgumentSpec(["--split"], default="none", help="Split type for parsing.")
-    ocr: ArgumentSpec[OCR] = ArgumentSpec(["--ocr"], default="auto", help="OCR type for parsing.")
-    output_format: ArgumentSpec[OutputFormat] = ArgumentSpec(
-        ["--output-format"], default="markdown", help="Output format."
-    )
-    coordinates: ArgumentSpec[bool] = ArgumentSpec(["--coordinates"], action="store_true", help="Include coordinates.")
-    base64_encoding: ArgumentSpec[list[Category]] = ArgumentSpec(
-        ["--base64-encoding"], default=["figure"], help="Base64 encoding for specific categories."
-    )
-    image_description_instruction: ArgumentSpec[str] = ArgumentSpec(
-        ["--image-description-instruction"],
-        default="Describe the image in detail.",
-        help="Instruction for image description.",
-    )
-    image_dir: ArgumentSpec[str] = ArgumentSpec(
-        ["--image-dir"],
-        default=DEFAULT_IMAGE_DIR,
-        help="Directory for image paths.",
-    )
-    chatterer: ArgumentSpec[Chatterer] = ArgumentSpec(
-        ["--chatterer"],
-        default=None,
-        help="Chatterer instance for communication.",
-        type=Chatterer.from_provider,
-    )
-
-    def run(self) -> None:
-        UpstageParserArguments.load()
-        input = UpstageParserArguments.in_path.unwrap().resolve()
-        out = UpstageParserArguments.out_path.value or input.with_suffix(".md")
-
-        parser = UpstageDocumentParseParser(
-            api_key=UpstageParserArguments.api_key.value,
-            base_url=UpstageParserArguments.base_url.unwrap(),
-            model=UpstageParserArguments.model.unwrap(),
-            split=UpstageParserArguments.split.unwrap(),
-            ocr=UpstageParserArguments.ocr.unwrap(),
-            output_format=UpstageParserArguments.output_format.unwrap(),
-            coordinates=UpstageParserArguments.coordinates.unwrap(),
-            base64_encoding=UpstageParserArguments.base64_encoding.unwrap(),
-            image_description_instruction=UpstageParserArguments.image_description_instruction.unwrap(),
-            image_dir=UpstageParserArguments.image_dir.value,
-            chatterer=UpstageParserArguments.chatterer.value,
-        )
-
-        docs = parser.parse(Blob.from_path(input))  # pyright: ignore[reportUnknownMemberType]
-
-        if UpstageParserArguments.image_dir.value:
-            for path, image in parser.image_data.items():
-                (path := Path(path)).parent.mkdir(parents=True, exist_ok=True)
-                path.write_bytes(image)
-                logger.info(f"Saved image to `{path}`")
-
-        markdown: str = "\n\n".join(f"<!--- page {i} -->\n{doc.page_content}" for i, doc in enumerate(docs, 1))
-        out.write_text(markdown, encoding="utf-8")
-        logger.info(f"Parsed `{input}` to `{out}`")
-
-
-if __name__ == "__main__":
-    UpstageParserArguments().run()
+def resolve_import_path_and_get_logger():
+    # ruff: noqa: E402
+    import logging
+    import sys
+
+    if __name__ == "__main__" and "." not in sys.path:
+        sys.path.append(".")
+
+    logger = logging.getLogger(__name__)
+    return logger
+
+
+logger = resolve_import_path_and_get_logger()
+from pathlib import Path
+
+from langchain_core.documents.base import Blob
+from spargear import ArgumentSpec, BaseArguments
+
+from chatterer import Chatterer, UpstageDocumentParseParser
+from chatterer.tools.upstage_document_parser import (
+    DEFAULT_IMAGE_DIR,
+    DOCUMENT_PARSE_BASE_URL,
+    DOCUMENT_PARSE_DEFAULT_MODEL,
+    OCR,
+    Category,
+    OutputFormat,
+    SplitType,
+)
+
+
+class UpstageParserArguments(BaseArguments):
+    in_path: ArgumentSpec[Path] = ArgumentSpec(["in-path"], help="Path to the input file.")
+    out_path: ArgumentSpec[Path] = ArgumentSpec(["--out-path"], default=None, help="Output file path.")
+    api_key: ArgumentSpec[str] = ArgumentSpec(["--api-key"], default=None, help="API key for the Upstage API.")
+    base_url: ArgumentSpec[str] = ArgumentSpec(["--base-url"], default=DOCUMENT_PARSE_BASE_URL, help="Base URL for the Upstage API.")
+    model: ArgumentSpec[str] = ArgumentSpec(["--model"], default=DOCUMENT_PARSE_DEFAULT_MODEL, help="Model to use for parsing.")
+    split: ArgumentSpec[SplitType] = ArgumentSpec(["--split"], default="none", help="Split type for parsing.")
+    ocr: ArgumentSpec[OCR] = ArgumentSpec(["--ocr"], default="auto", help="OCR type for parsing.")
+    output_format: ArgumentSpec[OutputFormat] = ArgumentSpec(["--output-format"], default="markdown", help="Output format.")
+    coordinates: ArgumentSpec[bool] = ArgumentSpec(["--coordinates"], action="store_true", help="Include coordinates.")
+    base64_encoding: ArgumentSpec[list[Category]] = ArgumentSpec(["--base64-encoding"], default=["figure"], help="Base64 encoding for specific categories.")
+    image_description_instruction: ArgumentSpec[str] = ArgumentSpec(
+        ["--image-description-instruction"],
+        default="Describe the image in detail.",
+        help="Instruction for image description.",
+    )
+    image_dir: ArgumentSpec[str] = ArgumentSpec(
+        ["--image-dir"],
+        default=DEFAULT_IMAGE_DIR,
+        help="Directory for image paths.",
+    )
+    chatterer: ArgumentSpec[Chatterer] = ArgumentSpec(
+        ["--chatterer"],
+        default=None,
+        help="Chatterer instance for communication.",
+        type=Chatterer.from_provider,
+    )
+
+    def run(self) -> None:
+        input = UpstageParserArguments.in_path.unwrap().resolve()
+        out = UpstageParserArguments.out_path.value or input.with_suffix(".md")
+
+        parser = UpstageDocumentParseParser(
+            api_key=UpstageParserArguments.api_key.value,
+            base_url=UpstageParserArguments.base_url.unwrap(),
+            model=UpstageParserArguments.model.unwrap(),
+            split=UpstageParserArguments.split.unwrap(),
+            ocr=UpstageParserArguments.ocr.unwrap(),
+            output_format=UpstageParserArguments.output_format.unwrap(),
+            coordinates=UpstageParserArguments.coordinates.unwrap(),
+            base64_encoding=UpstageParserArguments.base64_encoding.unwrap(),
+            image_description_instruction=UpstageParserArguments.image_description_instruction.unwrap(),
+            image_dir=UpstageParserArguments.image_dir.value,
+            chatterer=UpstageParserArguments.chatterer.value,
+        )
+
+        docs = parser.parse(Blob.from_path(input))  # pyright: ignore[reportUnknownMemberType]
+
+        if UpstageParserArguments.image_dir.value:
+            for path, image in parser.image_data.items():
+                (path := Path(path)).parent.mkdir(parents=True, exist_ok=True)
+                path.write_bytes(image)
+                logger.info(f"Saved image to `{path}`")
+
+        markdown: str = "\n\n".join(f"<!--- page {i} -->\n{doc.page_content}" for i, doc in enumerate(docs, 1))
+        out.write_text(markdown, encoding="utf-8")
+        logger.info(f"Parsed `{input}` to `{out}`")
+
+
+def main() -> None:
+    UpstageParserArguments().run()
+
+
+if __name__ == "__main__":
+    main()

chatterer/examples/webpage_to_markdown.py

@@ -1,79 +1,79 @@
-def resolve_import_path_and_get_logger():
-    # ruff: noqa: E402
-    import logging
-    import sys
-
-    if __name__ == "__main__" and "." not in sys.path:
-        sys.path.append(".")
-
-    logger = logging.getLogger(__name__)
-    return logger
-
-
-logger = resolve_import_path_and_get_logger()
-from pathlib import Path
-from typing import Literal
-
-from spargear import ArgumentSpec, BaseArguments
-
-from chatterer import Chatterer, MarkdownLink, PlayWrightBot
-
-
-class WebpageToMarkdownArgs(BaseArguments):
-    url: ArgumentSpec[str] = ArgumentSpec(["url"], help="The URL to crawl.")
-    out_path: str = Path(__file__).with_suffix(".md").as_posix()
-    """The output file path for the markdown file."""
-    chatterer: ArgumentSpec[Chatterer] = ArgumentSpec(
-        ["--llm"],
-        default=None,
-        type=Chatterer.from_provider,
-        help="The Chatterer backend and model to use for filtering the markdown.",
-    )
-    engine: Literal["firefox", "chromium", "webkit"] = "firefox"
-    """The browser engine to use."""
-
-    def run(self) -> None:
-        chatterer = self.chatterer.value
-        url: str = self.url.unwrap().strip()
-        out_path: Path = Path(self.out_path).resolve()
-        with PlayWrightBot(chatterer=chatterer, engine=self.engine) as bot:
-            md = bot.url_to_md(url)
-            out_path.write_text(md, encoding="utf-8")
-            if chatterer is not None:
-                md_llm = bot.url_to_md_with_llm(url.strip())
-                out_path.write_text(md_llm, encoding="utf-8")
-            links = MarkdownLink.from_markdown(md, referer_url=url)
-            for link in links:
-                if link.type == "link":
-                    print(
-                        f"- [{truncate_string(link.url)}] {truncate_string(link.inline_text)} ({truncate_string(link.inline_title)})"
-                    )
-                elif link.type == "image":
-                    print(f"- ![{truncate_string(link.url)}] ({truncate_string(link.inline_text)})")
-
-    async def arun(self) -> None:
-        chatterer = self.chatterer.value
-        url: str = self.url.unwrap().strip()
-        out_path: Path = Path(self.out_path).resolve()
-        async with PlayWrightBot(chatterer=chatterer, engine=self.engine) as bot:
-            md = await bot.aurl_to_md(url)
-            out_path.write_text(md, encoding="utf-8")
-            if chatterer is not None:
-                md_llm = await bot.aurl_to_md_with_llm(url.strip())
-                out_path.write_text(md_llm, encoding="utf-8")
-            links = MarkdownLink.from_markdown(md, referer_url=url)
-            for link in links:
-                if link.type == "link":
-                    print(
-                        f"- [{truncate_string(link.url)}] {truncate_string(link.inline_text)} ({truncate_string(link.inline_title)})"
-                    )
-                elif link.type == "image":
-                    print(f"- ![{truncate_string(link.url)}] ({truncate_string(link.inline_text)})")
-
-
-def truncate_string(s: str) -> str:
-    return s[:50] + "..." if len(s) > 50 else s
-
-
-if __name__ == "__main__":
-    WebpageToMarkdownArgs().run()
+def resolve_import_path_and_get_logger():
+    # ruff: noqa: E402
+    import logging
+    import sys
+
+    if __name__ == "__main__" and "." not in sys.path:
+        sys.path.append(".")
+
+    logger = logging.getLogger(__name__)
+    return logger
+
+
+logger = resolve_import_path_and_get_logger()
+from pathlib import Path
+from typing import Literal
+
+from spargear import ArgumentSpec, BaseArguments
+
+from chatterer import Chatterer, MarkdownLink, PlayWrightBot
+
+
+class WebpageToMarkdownArgs(BaseArguments):
+    url: ArgumentSpec[str] = ArgumentSpec(["url"], help="The URL to crawl.")
+    out_path: str = Path(__file__).with_suffix(".md").as_posix()
+    """The output file path for the markdown file."""
+    chatterer: ArgumentSpec[Chatterer] = ArgumentSpec(
+        ["--llm"],
+        default=None,
+        type=Chatterer.from_provider,
+        help="The Chatterer backend and model to use for filtering the markdown.",
+    )
+    engine: Literal["firefox", "chromium", "webkit"] = "firefox"
+    """The browser engine to use."""
+
+    def run(self) -> None:
+        chatterer = self.chatterer.value
+        url: str = self.url.unwrap().strip()
+        out_path: Path = Path(self.out_path).resolve()
+        with PlayWrightBot(chatterer=chatterer, engine=self.engine) as bot:
+            md = bot.url_to_md(url)
+            out_path.write_text(md, encoding="utf-8")
+            if chatterer is not None:
+                md_llm = bot.url_to_md_with_llm(url.strip())
+                out_path.write_text(md_llm, encoding="utf-8")
+            links = MarkdownLink.from_markdown(md, referer_url=url)
+            for link in links:
+                if link.type == "link":
+                    print(f"- [{truncate_string(link.url)}] {truncate_string(link.inline_text)} ({truncate_string(link.inline_title)})")
+                elif link.type == "image":
+                    print(f"- ![{truncate_string(link.url)}] ({truncate_string(link.inline_text)})")
+
+    async def arun(self) -> None:
+        chatterer = self.chatterer.value
+        url: str = self.url.unwrap().strip()
+        out_path: Path = Path(self.out_path).resolve()
+        async with PlayWrightBot(chatterer=chatterer, engine=self.engine) as bot:
+            md = await bot.aurl_to_md(url)
+            out_path.write_text(md, encoding="utf-8")
+            if chatterer is not None:
+                md_llm = await bot.aurl_to_md_with_llm(url.strip())
+                out_path.write_text(md_llm, encoding="utf-8")
+            links = MarkdownLink.from_markdown(md, referer_url=url)
+            for link in links:
+                if link.type == "link":
+                    print(f"- [{truncate_string(link.url)}] {truncate_string(link.inline_text)} ({truncate_string(link.inline_title)})")
+                elif link.type == "image":
+                    print(f"- ![{truncate_string(link.url)}] ({truncate_string(link.inline_text)})")
+
+
+def truncate_string(s: str) -> str:
+    return s[:50] + "..." if len(s) > 50 else s
+
+
+def main() -> None:
+    WebpageToMarkdownArgs().run()
+
+
+if __name__ == "__main__":
+    main()