chatterer 0.1.11__py3-none-any.whl → 0.1.13__py3-none-any.whl

chatterer/__init__.py

@@ -16,12 +16,13 @@ from .tools import (
     anything_to_markdown,
     citation_chunker,
     get_default_html_to_markdown_options,
+    get_youtube_video_details,
     get_youtube_video_subtitle,
     html_to_markdown,
+    init_upstage_document_parser,
     init_webpage_to_markdown,
     pdf_to_text,
     pyscripts_to_snippets,
-    get_youtube_video_details,
 )
 from .utils import (
     Base64Image,
@@ -57,4 +58,5 @@ __all__ = [
     "get_youtube_video_subtitle",
     "get_youtube_video_details",
     "interactive_shell",
+    "init_upstage_document_parser",
 ]

chatterer/common_types/__init__.py

@@ -0,0 +1,21 @@
+from .io import (
+    BytesReadable,
+    BytesWritable,
+    FileDescriptorOrPath,
+    PathOrReadable,
+    Readable,
+    StringReadable,
+    StringWritable,
+    Writable,
+)
+
+__all__ = [
+    "BytesReadable",
+    "BytesWritable",
+    "FileDescriptorOrPath",
+    "PathOrReadable",
+    "Readable",
+    "StringReadable",
+    "StringWritable",
+    "Writable",
+]

chatterer/common_types/io.py

@@ -0,0 +1,19 @@
+import os
+from io import BufferedReader, BufferedWriter, BytesIO, StringIO, TextIOWrapper
+from typing import TypeAlias
+
+# Type aliases for callback functions and file descriptors
+FileDescriptorOrPath: TypeAlias = int | str | bytes | os.PathLike[str] | os.PathLike[bytes]
+
+# Type aliases for different types of IO objects
+BytesReadable: TypeAlias = BytesIO | BufferedReader
+BytesWritable: TypeAlias = BytesIO | BufferedWriter
+StringReadable: TypeAlias = StringIO | TextIOWrapper
+StringWritable: TypeAlias = StringIO | TextIOWrapper
+
+# Combined type aliases for readable and writable objects
+Readable: TypeAlias = BytesReadable | StringReadable
+Writable: TypeAlias = BytesWritable | StringWritable
+
+# Type alias for path or readable object
+PathOrReadable: TypeAlias = FileDescriptorOrPath | Readable

chatterer/language_model.py

@@ -465,6 +465,68 @@ def interactive_shell(
             description="Whether further Python tool calling is needed to answer user query."
         )
 
+    def respond(messages: list[BaseMessage]) -> str:
+        # AI 응답 스트리밍 출력
+        console.print("[bold blue]AI:[/bold blue] ", end="")
+        response = ""
+        for chunk in chatterer.generate_stream(messages=messages):
+            response += chunk
+            console.print(chunk, end="")
+        console.print()  # 응답 후 줄바꿈 추가
+        return response.strip()
+
+    def code_session_returning_end_of_turn() -> bool:
+        code_session_messages: list[BaseMessage] = []
+        while True:
+            code_execution: CodeExecutionResult = chatterer.invoke_code_execution(
+                messages=context,
+                repl_tool=repl_tool,
+                prompt_for_code_invoke=prompt_for_code_invoke,
+                function_signatures=function_signatures,
+                function_reference_prefix=function_reference_prefix,
+                function_reference_seperator=function_reference_seperator,
+                config=config,
+                stop=stop,
+                **kwargs,
+            )
+            if code_execution.code.strip() in ("", "quit", "exit", "pass"):
+                return False
+
+            last_tool_use_message = AIMessage(
+                content=f"Executed code:\n```python\n{code_execution.code}\n```\nOutput:\n{code_execution.output}".strip()
+            )
+            code_session_messages.append(last_tool_use_message)
+            console.print("[bold yellow]Executed code:[/bold yellow]")
+            console.print(f"[code]{code_execution.code}[/code]")
+            console.print("[bold yellow]Output:[/bold yellow]")
+            console.print(code_execution.output)
+
+            decision = chatterer.generate_pydantic(
+                response_model=IsFurtherCodeExecutionNeeded,
+                messages=augment_prompt_for_toolcall(
+                    function_signatures=function_signatures,
+                    messages=context + code_session_messages,
+                    prompt_for_code_invoke=prompt_for_code_invoke,
+                    function_reference_prefix=function_reference_prefix,
+                    function_reference_seperator=function_reference_seperator,
+                ),
+            )
+            review_on_code_execution = decision.review_on_code_execution.strip()
+            next_action = decision.next_action.strip()
+            console.print("[bold blue]AI:[/bold blue]")
+            console.print(f"-[bold yellow]Review on code execution:[/bold yellow] {review_on_code_execution}")
+            console.print(f"-[bold yellow]Next Action:[/bold yellow] {next_action}")
+            code_session_messages.append(
+                AIMessage(
+                    content=f"- Review upon code execution: {review_on_code_execution}\n- Next Action: {next_action}".strip()
+                )
+            )
+            if not decision.is_further_code_execution_needed:
+                response: str = respond(context + code_session_messages)
+                context.append(last_tool_use_message)
+                context.append(AIMessage(content=response))
+                return True
+
     # REPL 도구 초기화
     if repl_tool is None:
         repl_tool = get_default_repl_tool()
@@ -504,77 +566,11 @@ def interactive_shell(
         )
 
         # 코드 실행 처리
-        if decision.is_code_execution_needed:
-            code_result = chatterer.invoke_code_execution(
-                messages=context,
-                repl_tool=repl_tool,
-                prompt_for_code_invoke=prompt_for_code_invoke,
-                function_signatures=function_signatures,
-                function_reference_prefix=function_reference_prefix,
-                function_reference_seperator=function_reference_seperator,
-                config=config,
-                stop=stop,
-                **kwargs,
-            )
-
-            if code_result.code.strip() == "pass":
-                tool_use_message = None
-            else:
-                code_session_messages: list[BaseMessage] = []
-                while True:
-                    code_execution_message = AIMessage(
-                        content=f"Executed code:\n```python\n{code_result.code}\n```\nOutput:\n{code_result.output}".strip()
-                    )
-                    code_session_messages.append(code_execution_message)
-                    console.print("[bold yellow]Executed code:[/bold yellow]")
-                    console.print(f"[code]{code_result.code}[/code]")
-                    console.print("[bold yellow]Output:[/bold yellow]")
-                    console.print(code_result.output)
-
-                    decision = chatterer.generate_pydantic(
-                        response_model=IsFurtherCodeExecutionNeeded,
-                        messages=augment_prompt_for_toolcall(
-                            function_signatures=function_signatures,
-                            messages=context + code_session_messages,
-                            prompt_for_code_invoke=prompt_for_code_invoke,
-                            function_reference_prefix=function_reference_prefix,
-                            function_reference_seperator=function_reference_seperator,
-                        ),
-                    )
-                    review_on_code_execution = decision.review_on_code_execution.strip()
-                    next_action = decision.next_action.strip()
-                    console.print("[bold blue]AI:[/bold blue]")
-                    console.print(f"-[bold yellow]Review on code execution:[/bold yellow] {review_on_code_execution}")
-                    console.print(f"-[bold yellow]Next Action:[/bold yellow] {next_action}")
-                    code_session_messages.append(
-                        AIMessage(
-                            content=f"- Review upon code execution: {review_on_code_execution}\n- Next Action: {next_action}".strip()
-                        )
-                    )
-                    if not decision.is_further_code_execution_needed:
-                        tool_use_message = code_execution_message
-                        break
-        else:
-            tool_use_message = None
-
-        # 코드 실행 결과 컨텍스트에 추가
-        if tool_use_message:
-            context.append(tool_use_message)
+        if decision.is_code_execution_needed and code_session_returning_end_of_turn():
+            continue
 
         # AI 응답 스트리밍 출력
-        console.print("[bold blue]AI:[/bold blue] ", end="")
-        response = ""
-        for chunk in chatterer.generate_stream(messages=context):
-            response += chunk
-            console.print(chunk, end="")
-
-        # 전체 응답 처리 후 컨텍스트에 추가
-        lines = response.split("\n")
-        if lines:
-            lines[-1] = lines[-1].rstrip()  # 마지막 줄의 오른쪽 공백 제거
-        response = "\n".join(lines).strip()
-        context.append(AIMessage(content=response))
-        console.print()  # 응답 후 줄바꿈 추가
+        context.append(AIMessage(content=respond(context)))
 
 
 if __name__ == "__main__":

chatterer/tools/__init__.py

@@ -6,7 +6,7 @@ from .convert_to_text import (
     pdf_to_text,
     pyscripts_to_snippets,
 )
-from .youtube import get_youtube_video_subtitle, get_youtube_video_details
+from .youtube import get_youtube_video_details, get_youtube_video_subtitle
 
 
 def init_webpage_to_markdown():
@@ -15,6 +15,12 @@ def init_webpage_to_markdown():
     return webpage_to_markdown
 
 
+def init_upstage_document_parser():
+    from . import upstage_document_parser
+
+    return upstage_document_parser
+
+
 __all__ = [
     "html_to_markdown",
     "anything_to_markdown",
@@ -25,4 +31,5 @@ __all__ = [
     "init_webpage_to_markdown",
     "get_youtube_video_subtitle",
     "get_youtube_video_details",
+    "init_upstage_document_parser",
 ]

chatterer/tools/convert_to_text.py

@@ -3,14 +3,11 @@ import importlib
 import os
 import re
 import site
-from contextlib import contextmanager, suppress
 from fnmatch import fnmatch
-from io import BufferedReader, BufferedWriter, BytesIO, StringIO, TextIOWrapper
 from pathlib import Path
 from typing import (
     TYPE_CHECKING,
     Callable,
-    Iterator,
     NamedTuple,
     NotRequired,
     Optional,
@@ -20,6 +17,9 @@ from typing import (
     TypedDict,
 )
 
+from ..common_types.io import PathOrReadable
+from ..utils.bytesio import read_bytes_stream
+
 if TYPE_CHECKING:
     from bs4 import Tag
     from openai import OpenAI
@@ -38,20 +38,6 @@ type FileTree = dict[str, Optional[FileTree]]
 
 # Type aliases for callback functions and file descriptors
 CodeLanguageCallback: TypeAlias = Callable[["Tag"], Optional[str]]
-FileDescriptorOrPath: TypeAlias = int | str | bytes | os.PathLike[str] | os.PathLike[bytes]
-
-# Type aliases for different types of IO objects
-BytesReadable: TypeAlias = BytesIO | BufferedReader
-BytesWritable: TypeAlias = BytesIO | BufferedWriter
-StringReadable: TypeAlias = StringIO | TextIOWrapper
-StringWritable: TypeAlias = StringIO | TextIOWrapper
-
-# Combined type aliases for readable and writable objects
-Readable: TypeAlias = BytesReadable | StringReadable
-Writable: TypeAlias = BytesWritable | StringWritable
-
-# Type alias for path or readable object
-PathOrReadable: TypeAlias = FileDescriptorOrPath | Readable
 
 
 class HtmlToMarkdownOptions(TypedDict):
@@ -240,7 +226,7 @@ def pdf_to_text(path_or_file: PathOrReadable) -> str:
     """
     from pymupdf import Document  # pyright: ignore[reportMissingTypeStubs]
 
-    with _open_stream(path_or_file) as stream:
+    with read_bytes_stream(path_or_file) as stream:
         if stream is None:
             raise FileNotFoundError(path_or_file)
         return "\n".join(
@@ -430,34 +416,3 @@ def _get_pyscript_paths(path_or_pkgname: str, ban_fn_patterns: Optional[list[str
             if p.is_file()
         ]
     return [p for p in pypaths if not ban_fn_patterns or not _is_banned(p, ban_fn_patterns)]
-
-
-@contextmanager
-def _open_stream(
-    path_or_file: PathOrReadable,
-) -> Iterator[Optional[BytesReadable]]:
-    """
-    Context manager for opening a file or using an existing stream.
-
-    Handles different types of input (file paths, byte streams, string streams)
-    and yields a BytesReadable object that can be used to read binary data.
-
-    Args:
-        path_or_file: File path or readable object.
-
-    Yields:
-        Optional[BytesReadable]: A readable binary stream or None if opening fails.
-    """
-    stream: Optional[BytesReadable] = None
-    try:
-        with suppress(BaseException):
-            if isinstance(path_or_file, BytesReadable):
-                stream = path_or_file
-            elif isinstance(path_or_file, StringReadable):
-                stream = BytesIO(path_or_file.read().encode("utf-8"))
-            else:
-                stream = open(path_or_file, "rb")
-        yield stream
-    finally:
-        if stream is not None:
-            stream.close()

chatterer/tools/upstage_document_parser.py

@@ -0,0 +1,438 @@
+"""Adopted from`langchain_upstage.document_parse"""
+
+import io
+import json
+import logging
+import os
+from typing import Iterator, Literal, Optional, cast
+
+import requests
+from langchain_core.document_loaders import BaseBlobParser, Blob
+from langchain_core.documents import Document
+from pydantic import BaseModel, Field
+from pypdf import PdfReader, PdfWriter
+from pypdf.errors import PdfReadError
+
+from ..common_types.io import BytesReadable
+from ..language_model import DEFAULT_IMAGE_DESCRIPTION_INSTRUCTION, Chatterer
+from ..utils.image import Base64Image
+
+logger = logging.getLogger("pypdf")
+logger.setLevel(logging.ERROR)
+
+DOCUMENT_PARSE_BASE_URL = "https://api.upstage.ai/v1/document-ai/document-parse"
+DEFAULT_NUM_PAGES = 10
+DOCUMENT_PARSE_DEFAULT_MODEL = "document-parse"
+
+OutputFormat = Literal["text", "html", "markdown"]
+OCR = Literal["auto", "force"]
+SplitType = Literal["none", "page", "element"]
+Category = Literal[
+    "paragraph",
+    "table",
+    "figure",
+    "header",
+    "footer",
+    "caption",
+    "equation",
+    "heading1",
+    "list",
+    "index",
+    "footnote",
+    "chart",
+]
+
+
+class Content(BaseModel):
+    text: Optional[str] = None
+    html: Optional[str] = None
+    markdown: Optional[str] = None
+
+
+class Coordinate(BaseModel):
+    x: float
+    y: float
+
+
+class Element(BaseModel):
+    category: Category
+    content: Content
+    coordinates: list[Coordinate] = Field(default_factory=list)
+    base64_encoding: str = ""
+    id: int
+    page: int
+
+    def parse_text(self, parser: "UpstageDocumentParseParser") -> str:
+        output_format: OutputFormat = parser.output_format
+        chatterer: Optional[Chatterer] = parser.chatterer
+        image_description_instruction: str = parser.image_description_instruction
+        output: Optional[str] = None
+        if output_format == "text":
+            output = self.content.text
+        elif output_format == "html":
+            output = self.content.html
+        elif output_format == "markdown":
+            output = self.content.markdown
+        if output is None:
+            raise ValueError(f"Invalid output format: {output_format}")
+
+        if chatterer is not None and self.category == "figure" and self.base64_encoding:
+            image = Base64Image.from_string(f"data:image/jpeg;base64,{self.base64_encoding}")
+            if image is None:
+                raise ValueError(f"Invalid base64 encoding for image: {self.base64_encoding}")
+            ocr_content = output.removeprefix("![image](/image/placeholder)\n")
+            image_description = chatterer.describe_image(
+                image.data_uri,
+                image_description_instruction
+                + f"\nHint: The OCR detected the following text:\n```\n{ocr_content}\n```",
+            )
+            output = f"\n\n<details>\n{image_description}\n</details>\n\n"
+
+        return output
+
+
+def get_from_param_or_env(
+    key: str,
+    param: Optional[str] = None,
+    env_key: Optional[str] = None,
+    default: Optional[str] = None,
+) -> str:
+    """Get a value from a param or an environment variable."""
+    if param is not None:
+        return param
+    elif env_key and env_key in os.environ and os.environ[env_key]:
+        return os.environ[env_key]
+    elif default is not None:
+        return default
+    else:
+        raise ValueError(
+            f"Did not find {key}, please add an environment variable"
+            f" `{env_key}` which contains it, or pass"
+            f"  `{key}` as a named parameter."
+        )
+
+
+class UpstageDocumentParseParser(BaseBlobParser):
+    """Upstage Document Parse Parser.
+
+    To use, you should have the environment variable `UPSTAGE_API_KEY`
+    set with your API key or pass it as a named parameter to the constructor.
+
+    Example:
+        .. code-block:: python
+
+            from langchain_upstage import UpstageDocumentParseParser
+
+            loader = UpstageDocumentParseParser(split="page", output_format="text")
+    """
+
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        base_url: str = DOCUMENT_PARSE_BASE_URL,
+        model: str = DOCUMENT_PARSE_DEFAULT_MODEL,
+        split: SplitType = "none",
+        ocr: OCR = "auto",
+        output_format: OutputFormat = "markdown",
+        coordinates: bool = True,
+        base64_encoding: list[Category] = [],
+        chatterer: Optional[Chatterer] = None,
+        image_description_instruction: str = DEFAULT_IMAGE_DESCRIPTION_INSTRUCTION,
+    ) -> None:
+        """
+        Initializes an instance of the Upstage class.
+
+        Args:
+            api_key (str, optional): The API key for accessing the Upstage API.
+                                     Defaults to None, in which case it will be
+                                     fetched from the environment variable
+                                     `UPSTAGE_API_KEY`.
+            base_url (str, optional): The base URL for accessing the Upstage API.
+            model (str): The model to be used for the document parse.
+                         Defaults to "document-parse".
+            split (SplitType, optional): The type of splitting to be applied.
+                                         Defaults to "none" (no splitting).
+            ocr (OCRMode, optional): Extract text from images in the document using OCR.
+                                     If the value is "force", OCR is used to extract
+                                     text from an image. If the value is "auto", text is
+                                     extracted from a PDF. (An error will occur if the
+                                     value is "auto" and the input is NOT in PDF format)
+            output_format (OutputFormat, optional): Format of the inference results.
+            coordinates (bool, optional): Whether to include the coordinates of the
+                                          OCR in the output.
+            base64_encoding (List[Category], optional): The category of the elements to
+                                                        be encoded in base64.
+            chatterer (Chatterer, optional): The Chatterer instance to use for image
+                                             description.
+            image_description_instruction (str, optional): The instruction to use for
+                                                           image description.
+
+
+        """
+        self.api_key = get_from_param_or_env(
+            "UPSTAGE_API_KEY",
+            api_key,
+            "UPSTAGE_API_KEY",
+            os.environ.get("UPSTAGE_API_KEY"),
+        )
+        self.base_url = base_url
+        self.model = model
+        self.split: SplitType = split
+        self.ocr: OCR = ocr
+        self.output_format: OutputFormat = output_format
+        self.coordinates = coordinates
+        self.base64_encoding: list[Category] = base64_encoding
+        self.chatterer = chatterer
+        self.image_description_instruction = image_description_instruction
+
+    def _get_response(self, files: dict[str, BytesReadable]) -> list[Element]:
+        """
+        Sends a POST request to the API endpoint with the provided files and
+        returns the response.
+
+        Args:
+            files (dict): A dictionary containing the files to be sent in the request.
+
+        Returns:
+            dict: The JSON response from the API.
+
+        Raises:
+            ValueError: If there is an error in the API call.
+        """
+        try:
+            headers = {
+                "Authorization": f"Bearer {self.api_key}",
+            }
+            response = requests.post(
+                self.base_url,
+                headers=headers,
+                files=files,
+                data={
+                    "ocr": self.ocr,
+                    "model": self.model,
+                    "output_formats": f"['{self.output_format}']",
+                    "coordinates": self.coordinates,
+                    "base64_encoding": f"{self.base64_encoding}",
+                },
+            )
+            response.raise_for_status()
+            result: object = response.json().get("elements", [])
+            if not isinstance(result, list):
+                raise ValueError(f"Failed to parse JSON data: {result}")
+            result = cast(list[object], result)
+            return [Element.model_validate(element) for element in result]
+        except requests.HTTPError as e:
+            raise ValueError(f"HTTP error: {e.response.text}")
+        except requests.RequestException as e:
+            # Handle any request-related exceptions
+            raise ValueError(f"Failed to send request: {e}")
+        except json.JSONDecodeError as e:
+            # Handle JSON decode errors
+            raise ValueError(f"Failed to decode JSON response: {e}")
+        except Exception as e:
+            # Handle any other exceptions
+            raise ValueError(f"An error occurred: {e}")
+
+    def _split_and_request(
+        self, full_docs: PdfReader, start_page: int, num_pages: int = DEFAULT_NUM_PAGES
+    ) -> list[Element]:
+        """
+        Splits the full pdf document into partial pages and sends a request to the
+        server.
+
+        Args:
+            full_docs (PdfReader): The full document to be split and requested.
+            start_page (int): The starting page number for splitting the document.
+            num_pages (int, optional): The number of pages to split the document
+                                       into.
+                                       Defaults to DEFAULT_NUMBER_OF_PAGE.
+
+        Returns:
+            response: The response from the server.
+        """
+        merger = PdfWriter()
+        merger.append(
+            full_docs,
+            pages=(start_page, min(start_page + num_pages, full_docs.get_num_pages())),
+        )
+
+        with io.BytesIO() as buffer:
+            merger.write(buffer)
+            buffer.seek(0)
+            return self._get_response({"document": buffer})
+
+    def _element_document(self, element: Element, start_page: int = 0) -> Document:
+        """
+        Converts an elements into a Document object.
+
+        Args:
+            elements (Dict) : The elements to convert.
+            start_page (int): The starting page number for splitting the document.
+                              This number starts from zero.
+
+        Returns:
+            A list containing a single Document object.
+
+        """
+        metadata: dict[str, object] = element.model_dump(exclude_none=True)
+        metadata["page"] = element.page + start_page
+        return Document(
+            page_content=element.parse_text(self),
+            metadata=metadata,
+        )
+
+    def _page_document(self, elements: list[Element], start_page: int = 0) -> list[Document]:
+        """
+        Combines elements with the same page number into a single Document object.
+
+        Args:
+            elements (List): A list of elements containing page numbers.
+            start_page (int): The starting page number for splitting the document.
+                              This number starts from zero.
+
+        Returns:
+            List[Document]: A list of Document objects, each representing a page
+                            with its content and metadata.
+        """
+        documents: list[Document] = []
+        pages: list[int] = sorted(set(map(lambda x: x.page, elements)))
+        page_group: list[list[Element]] = [[element for element in elements if element.page == x] for x in pages]
+        for group in page_group:
+            metadata: dict[str, object] = {
+                "page": group[0].page + start_page,
+            }
+            if self.base64_encoding:
+                metadata["base64_encodings"] = [element.base64_encoding for element in group if element.base64_encoding]
+            if self.coordinates:
+                metadata["coordinates"] = [element.coordinates for element in group if element.coordinates]
+            documents.append(
+                Document(
+                    page_content=" ".join(element.parse_text(self) for element in group),
+                    metadata=metadata,
+                )
+            )
+
+        return documents
+
+    def lazy_parse(self, blob: Blob, is_batch: bool = False) -> Iterator[Document]:
+        """
+        Lazily parses a document and yields Document objects based on the specified
+        split type.
+
+        Args:
+            blob (Blob): The input document blob to parse.
+            is_batch (bool, optional): Whether to parse the document in batches.
+                                       Defaults to False (single page parsing)
+
+        Yields:
+            Document: The parsed document object.
+
+        Raises:
+            ValueError: If an invalid split type is provided.
+
+        """
+
+        if is_batch:
+            num_pages = DEFAULT_NUM_PAGES
+        else:
+            num_pages = 1
+
+        full_docs: Optional[PdfReader] = None
+        try:
+            full_docs = PdfReader(str(blob.path))
+            number_of_pages = full_docs.get_num_pages()
+        except PdfReadError:
+            number_of_pages = 1
+        except Exception as e:
+            raise ValueError(f"Failed to read PDF file: {e}")
+
+        if self.split == "none":
+            result = ""
+            base64_encodings: list[str] = []
+            coordinates: list[list[Coordinate]] = []
+
+            if full_docs is not None:
+                start_page = 0
+                num_pages = DEFAULT_NUM_PAGES
+                for _ in range(number_of_pages):
+                    if start_page >= number_of_pages:
+                        break
+
+                    elements = self._split_and_request(full_docs, start_page, num_pages)
+                    for element in elements:
+                        result += element.parse_text(self)
+                        if self.base64_encoding and (base64_encoding := element.base64_encoding):
+                            base64_encodings.append(base64_encoding)
+                        if self.coordinates and (coords := element.coordinates):
+                            coordinates.append(coords)
+
+                    start_page += num_pages
+
+            else:
+                if not blob.path:
+                    raise ValueError("Blob path is required for non-PDF files.")
+
+                with open(blob.path, "rb") as f:
+                    elements = self._get_response({"document": f})
+
+                for element in elements:
+                    result += element.parse_text(self)
+
+                    if self.base64_encoding and (base64_encoding := element.base64_encoding):
+                        base64_encodings.append(base64_encoding)
+                    if self.coordinates and (coords := element.coordinates):
+                        coordinates.append(coords)
+            metadata: dict[str, object] = {"total_pages": number_of_pages}
+            if self.coordinates:
+                metadata["coordinates"] = coordinates
+            if self.base64_encoding:
+                metadata["base64_encodings"] = base64_encodings
+
+            yield Document(
+                page_content=result,
+                metadata=metadata,
+            )
+
+        elif self.split == "element":
+            if full_docs is not None:
+                start_page = 0
+                for _ in range(number_of_pages):
+                    if start_page >= number_of_pages:
+                        break
+
+                    elements = self._split_and_request(full_docs, start_page, num_pages)
+                    for element in elements:
+                        yield self._element_document(element, start_page)
+
+                    start_page += num_pages
+
+            else:
+                if not blob.path:
+                    raise ValueError("Blob path is required for non-PDF files.")
+                with open(blob.path, "rb") as f:
+                    elements = self._get_response({"document": f})
+
+                for element in elements:
+                    yield self._element_document(element)
+
+        elif self.split == "page":
+            if full_docs is not None:
+                start_page = 0
+                for _ in range(number_of_pages):
+                    if start_page >= number_of_pages:
+                        break
+
+                    elements = self._split_and_request(full_docs, start_page, num_pages)
+                    yield from self._page_document(elements, start_page)
+
+                    start_page += num_pages
+            else:
+                if not blob.path:
+                    raise ValueError("Blob path is required for non-PDF files.")
+                with open(blob.path, "rb") as f:
+                    elements = self._get_response({"document": f})
+
+                yield from self._page_document(elements)
+
+        else:
+            raise ValueError(f"Invalid split type: {self.split}")

chatterer/tools/webpage_to_markdown/utils.py

@@ -277,6 +277,7 @@ def get_image_url_and_markdown_links(
 
         image_data = Base64Image.from_url_or_path(markdown_link.url, headers=headers, config=config)
         if not image_data:
+            image_matches.setdefault(None, []).append(markdown_link)
             continue
         image_matches.setdefault(image_data, []).append(markdown_link)
     return image_matches
@@ -294,6 +295,7 @@ async def aget_image_url_and_markdown_links(
             markdown_link.url, headers=headers, config=config, return_coro=True
         )
         if not image_data:
+            image_matches.setdefault(None, []).append(markdown_link)
             continue
         image_matches.setdefault(image_data, []).append(markdown_link)
     return image_matches
@@ -306,7 +308,10 @@ def replace_images(
     for image_description, markdown_links in image_description_and_references.items():
         for markdown_link in markdown_links:
             if image_description is None:
-                replacements.append((markdown_link, markdown_link.link_markdown))
+                if markdown_link.type == "link":
+                    replacements.append((markdown_link, markdown_link.link_markdown))
+                elif markdown_link.type == "image":
+                    replacements.append((markdown_link, f"![{markdown_link.inline_text}](...)"))
             else:
                 replacements.append((
                     markdown_link,

chatterer/utils/bytesio.py

@@ -0,0 +1,59 @@
+import os
+from contextlib import contextmanager, suppress
+from io import BytesIO
+from typing import Iterator, Optional
+
+from ..common_types.io import BytesReadable, PathOrReadable, StringReadable
+
+
+@contextmanager
+def read_bytes_stream(
+    path_or_file: PathOrReadable,
+    assume_pathlike_bytes_as_path: bool = False,
+    assume_pathlike_string_as_path: bool = True,
+) -> Iterator[Optional[BytesReadable]]:
+    """
+    Context manager for opening a file or using an existing stream.
+
+    Handles different types of input (file paths, byte streams, string streams)
+    and yields a BytesReadable object that can be used to read binary data.
+
+    Args:
+        path_or_file: File path or readable object.
+        assume_pathlike_bytes_as_path: If True, assume bytes-like objects are file paths. Else, treat as data itself.
+        assume_pathlike_string_as_path: If True, assume string-like objects are file paths. Else, treat as data itself.
+
+    Yields:
+        Optional[BytesReadable]: A readable binary stream or None if opening fails.
+    """
+    stream: Optional[BytesReadable] = None
+    should_close: bool = True  # Whether the stream should be closed after use
+    try:
+        with suppress(BaseException):
+            if isinstance(path_or_file, BytesReadable):
+                # Assume the input is already a bytes stream
+                # NOTE: Delivers itself, so shouldn't be closed.
+                stream = path_or_file
+                should_close = False
+            elif isinstance(path_or_file, StringReadable):
+                # Convert the string stream to bytes stream
+                stream = BytesIO(path_or_file.read().encode("utf-8"))
+            elif isinstance(path_or_file, bytes):
+                # Convert the bytes-like object to bytes stream
+                if assume_pathlike_bytes_as_path and os.path.exists(path_or_file):
+                    stream = open(path_or_file, "rb")
+                else:
+                    stream = BytesIO(path_or_file)
+            elif isinstance(path_or_file, str):
+                # Convert the file path to bytes stream
+                if assume_pathlike_string_as_path and os.path.exists(path_or_file):
+                    stream = open(path_or_file, "rb")
+                else:
+                    stream = BytesIO(path_or_file.encode("utf-8"))
+            else:
+                # Assume the input is a file descriptor or path
+                stream = open(path_or_file, "rb")
+        yield stream
+    finally:
+        if stream is not None and should_close:
+            stream.close()

chatterer/utils/image.py

@@ -3,8 +3,8 @@ from __future__ import annotations
 import re
 from base64 import b64encode
 from io import BytesIO
+from logging import getLogger
 from pathlib import Path
-from traceback import print_exc
 from typing import (
     Awaitable,
     ClassVar,
@@ -28,6 +28,7 @@ from PIL.Image import Resampling
 from PIL.Image import open as image_open
 from pydantic import BaseModel
 
+logger = getLogger(__name__)
 ImageType: TypeAlias = Literal["jpeg", "jpg", "png", "gif", "webp", "bmp"]
 
 
@@ -123,7 +124,10 @@ class Base64Image(BaseModel):
             if return_coro:
                 return cls._afetch_remote_image(url_or_path, headers, config)
             return cls._fetch_remote_image(url_or_path, headers, config)
-        return cls._process_local_image(Path(url_or_path), config)
+        try:
+            return cls._process_local_image(Path(url_or_path), config)
+        except Exception:
+            return None
 
     @property
     def data_uri(self) -> str:
@@ -167,7 +171,7 @@ class Base64Image(BaseModel):
         max_size_mb = config.get("max_size_mb", float("inf"))
         image_size_mb = len(image_data) / (1024 * 1024)
         if image_size_mb > max_size_mb:
-            print(f"Image too large: {image_size_mb:.2f} MB > {max_size_mb} MB")
+            logger.error(f"Image too large: {image_size_mb:.2f} MB > {max_size_mb} MB")
             return None
 
         # 2) Pillow로 이미지 열기
@@ -182,7 +186,7 @@ class Base64Image(BaseModel):
                 # min_largest_side 기준
                 min_largest_side = config.get("min_largest_side", 1)
                 if largest_side < min_largest_side:
-                    print(f"Image too small: {largest_side} < {min_largest_side}")
+                    logger.error(f"Image too small: {largest_side} < {min_largest_side}")
                     return None
 
                 # resize 로직
@@ -200,7 +204,7 @@ class Base64Image(BaseModel):
                 pil_format: str = (im.format or "").lower()
                 allowed_formats: Sequence[ImageType] = config.get("formats", [])
                 if not cls._verify_ext(pil_format, allowed_formats):
-                    print(f"Invalid format: {pil_format} not in {allowed_formats}")
+                    logger.error(f"Invalid format: {pil_format} not in {allowed_formats}")
                     return None
 
                 # 다시 bytes 로 저장
@@ -210,7 +214,6 @@ class Base64Image(BaseModel):
                 final_bytes = output_buffer.read()
 
         except Exception:
-            print_exc()
             return None
 
         # 최종 base64 인코딩

{chatterer-0.1.11.dist-info → chatterer-0.1.13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: chatterer
-Version: 0.1.11
+Version: 0.1.13
 Summary: The highest-level interface for various LLM APIs.
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
@@ -15,18 +15,19 @@ Requires-Dist: markdownify>=1.1.0; extra == "conversion"
 Requires-Dist: commonmark>=0.9.1; extra == "conversion"
 Requires-Dist: playwright>=1.50.0; extra == "conversion"
 Requires-Dist: pillow>=11.1.0; extra == "conversion"
-Requires-Dist: mistune>=3.1.2; extra == "conversion"
-Requires-Dist: markitdown>=0.0.2; extra == "conversion"
+Requires-Dist: mistune>=3.1.3; extra == "conversion"
+Requires-Dist: markitdown>=0.1.1; extra == "conversion"
 Requires-Dist: pymupdf>=1.25.4; extra == "conversion"
-Requires-Dist: youtube-transcript-api>=1.0.2; extra == "conversion"
+Requires-Dist: youtube-transcript-api>=1.0.3; extra == "conversion"
+Requires-Dist: pypdf>=5.4.0; extra == "conversion"
 Provides-Extra: langchain
 Requires-Dist: chatterer[langchain-providers]; extra == "langchain"
 Requires-Dist: langchain-experimental>=0.3.4; extra == "langchain"
 Provides-Extra: langchain-providers
-Requires-Dist: langchain-openai>=0.3.7; extra == "langchain-providers"
-Requires-Dist: langchain-anthropic>=0.3.8; extra == "langchain-providers"
-Requires-Dist: langchain-google-genai>=2.0.10; extra == "langchain-providers"
-Requires-Dist: langchain-ollama>=0.2.3; extra == "langchain-providers"
+Requires-Dist: langchain-openai>=0.3.11; extra == "langchain-providers"
+Requires-Dist: langchain-anthropic>=0.3.10; extra == "langchain-providers"
+Requires-Dist: langchain-google-genai>=2.1.1; extra == "langchain-providers"
+Requires-Dist: langchain-ollama>=0.3.0; extra == "langchain-providers"
 Provides-Extra: all
 Requires-Dist: chatterer[langchain]; extra == "all"
 Requires-Dist: chatterer[conversion]; extra == "all"

{chatterer-0.1.11.dist-info → chatterer-0.1.13.dist-info}/RECORD

@@ -1,12 +1,15 @@
-chatterer/__init__.py,sha256=BPgCQ6VWGBXSh8xJr_0bpM0hcOOUz0KoxcKxOd9GYyI,1388
-chatterer/language_model.py,sha256=DX_mU855JHHqE0gdnieWZNOwX1BjIO4VK4EightRL3w,24353
+chatterer/__init__.py,sha256=444C_hySiaJNBsG40l6d_xYY_KT5rBiQLR1mgzOc19A,1460
+chatterer/language_model.py,sha256=gjZC8SyTNZ0rQke_SongcfQid26coLtE7lguhdoFuX8,24078
 chatterer/messages.py,sha256=OtbZ3two0LUQ4PXES97FDIBUSO3IcMHdFV1VFkDL2mI,229
 chatterer/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+chatterer/common_types/__init__.py,sha256=jfS6m5UANSvGjzQ_nzYDpryn5uZqNb06-4xCsQ2C_lw,376
+chatterer/common_types/io.py,sha256=fetiyi1suZ3NF2mj5k5KDLJLGKS1n4J-5UmH7JN36g8,817
 chatterer/strategies/__init__.py,sha256=SdOggbmHpw4f7Njwy-T8q64e91OLOUp1k0a0ozZd4qI,221
 chatterer/strategies/atom_of_thoughts.py,sha256=CygOCLu5vLk-fzY9O-iE3qLShfjD7iY40ks9jH4ULBM,40872
 chatterer/strategies/base.py,sha256=b2gMPqodp97OP1dkHfj0UqixjdjVhmTw_V5qJ7i2S6g,427
-chatterer/tools/__init__.py,sha256=hmWIuLJWotGQodL__i4LLbHdXe7Nl5uKHqNke9tHMro,705
-chatterer/tools/convert_to_text.py,sha256=kBqxCJ0IoiAw2eiPYqep_SPZm-TtYKF7mdACLsWQUuI,15915
+chatterer/tools/__init__.py,sha256=CK6hHDmgqHg70k6hHcMdHv5qutKBfReaNy2c4EaKOns,864
+chatterer/tools/convert_to_text.py,sha256=gfeMDogvDg8G4ZcRC3m4yU24_0-r_cl5gXHwg2Ym9p4,14222
+chatterer/tools/upstage_document_parser.py,sha256=s0mtukC93y7zwS94gjyvgcvCsr2fAUzt1LZPWYxdF1Q,17165
 chatterer/tools/youtube.py,sha256=GhyE05JBF_eos01A_N-X5tZv4wQJ--IjErBbEBeNBpQ,6037
 chatterer/tools/citation_chunking/__init__.py,sha256=gG7Fnkkp28UpcWMbfMY_4gqzZSZ8QzlhalHBoeoq7K0,82
 chatterer/tools/citation_chunking/chunks.py,sha256=50Dpa43RaYftlNox8tM1qI8htZ3_AJ9Uyyn02WsmxYk,2173
@@ -17,11 +20,12 @@ chatterer/tools/citation_chunking/reference.py,sha256=uRKufkU41Zedz6MQUCy-aCk4Rw
 chatterer/tools/citation_chunking/utils.py,sha256=M4pH2-UIE1VLzQLXDqjEe4L3Xcy0e0KhAP3I2U2BNms,6348
 chatterer/tools/webpage_to_markdown/__init__.py,sha256=bHH4qfnXyw8Zz-yBPLaTezF1sh9njvNBJmhBVtcpjsA,123
 chatterer/tools/webpage_to_markdown/playwright_bot.py,sha256=yP0KixYZNQ4Kn_ZCFDI3mVyBD_DpUGfqgklpaGJUTCU,27496
-chatterer/tools/webpage_to_markdown/utils.py,sha256=ZLUU94imYciEdynD2K7Dmcsbt8BVQTaOP56Ba6DAFvk,12593
+chatterer/tools/webpage_to_markdown/utils.py,sha256=TK88-ReOUTs8njIGDY-nCNNVCPwHCVb6nV5wNuDxx2Q,12938
 chatterer/utils/__init__.py,sha256=8nzpFJKU_wSRPH6LBP6HRBotPMrSl_VO9UlmFprTrK0,334
+chatterer/utils/bytesio.py,sha256=3MC2atOOFKo5YxuReo_y_t8Wem9p2Y1ahC5M2lGclwI,2618
 chatterer/utils/code_agent.py,sha256=UaWdeGzJMPzRSFy9yrxuveBJsvOPSa0te6OuE18bees,5143
-chatterer/utils/image.py,sha256=1imiyq6TB9NIIGx3zAA2OwMWuXlifYIAjwfWRWa4WIM,10858
-chatterer-0.1.11.dist-info/METADATA,sha256=S3hRkxG1DlFc_NGrra1xhniiCDDVoVrow2N96OJy8i0,4458
-chatterer-0.1.11.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-chatterer-0.1.11.dist-info/top_level.txt,sha256=7nSQKP0bHxPRc7HyzdbKsJdkvPgYD0214o6slRizv9s,10
-chatterer-0.1.11.dist-info/RECORD,,
+chatterer/utils/image.py,sha256=mBqVBAhIpe1PovxKMPJ77GHcUsePQlgIWW2FZgh-6Z4,10952
+chatterer-0.1.13.dist-info/METADATA,sha256=AAhg295_57oJYop5P0cBHQiiP1ArzP_Zo-8Lq7APzYY,4511
+chatterer-0.1.13.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+chatterer-0.1.13.dist-info/top_level.txt,sha256=7nSQKP0bHxPRc7HyzdbKsJdkvPgYD0214o6slRizv9s,10
+chatterer-0.1.13.dist-info/RECORD,,