chatterer 0.1.5__py3-none-any.whl → 0.1.7__py3-none-any.whl

chatterer/tools/citation_chunking/reference.py

@@ -0,0 +1,26 @@
+from typing import Literal, TypeAlias
+
+from pydantic import BaseModel, Field
+
+
+class MultiMatchRegex(BaseModel):
+    type: Literal["multi_match_regex"] = Field(
+        description="A regex pattern that should match multiple instances of the subject in the document."
+    )
+    regular_expression: str = Field(
+        description="The regex pattern that should match multiple instances of the subject in the document."
+    )
+
+    def __hash__(self) -> int:
+        return hash((self.type, self.regular_expression))
+
+
+class SingleMatchCitation(BaseModel):
+    start_from: str = Field(description="A snippet of text at the beginning of the cited section.")
+    end_at: str = Field(description="A snippet of text at the end of the cited section.")
+
+    def __hash__(self) -> int:
+        return hash((self.start_from, self.end_at))
+
+
+Reference: TypeAlias = SingleMatchCitation | MultiMatchRegex

chatterer/tools/citation_chunking/utils.py

@@ -0,0 +1,138 @@
+from typing import Callable, NamedTuple, Self, TypeVar
+
+from pydantic import BaseModel
+
+T = TypeVar("T", bound=BaseModel)
+
+
+class MatchedText(NamedTuple):
+    text: str
+    start_idx: int
+    end_idx: int
+
+    @classmethod
+    def from_text(
+        cls,
+        full_text: str,
+        len_func: Callable[[str], int],
+        chunk_size: int = 2048,
+        token_overlap: int = 0,
+        separator: str = "\n",
+    ) -> list[Self]:
+        """
+        토큰 수 제한과 선택적 오버랩을 기준으로 텍스트를 청크로 분할합니다.
+        각 청크는 원본 텍스트 내의 위치 정보 (start_idx, end_idx)와 함께 반환됩니다.
+        텍스트는 separator 문자열로 분할하며, 토큰 수는 len_func 함수를 통해 계산합니다.
+
+        Args:
+            full_text: 분할할 전체 텍스트.
+            len_func: 주어진 텍스트의 토큰 수를 반환하는 함수.
+            chunk_size: 각 청크의 최대 토큰 수. 기본값은 2048.
+            token_overlap: 청크 간 중첩할 토큰 수. 기본값은 0.
+            separator: 텍스트를 분할할 구분자 문자열. 기본값은 "\n".
+
+        Returns:
+            각 요소가 (chunk_text, start_idx, end_idx)인 튜플의 리스트.
+            chunk_text는 whole_text 내에서 whole_text[start_idx:end_idx]와 동일한 부분 문자열입니다.
+        """
+        text_chunks: list[Self] = []
+        sep_token_count: int = len_func(separator)
+        sep_len = len(separator)
+
+        # 먼저, separator를 기준으로 원본 텍스트를 분할하되 각 조각의 시작/종료 인덱스를 기록합니다.
+        piece_infos: list[Self] = []  # 각 튜플: (piece_text, start_index, end_index)
+        start_idx = 0
+        while True:
+            idx = full_text.find(separator, start_idx)
+            if idx == -1:
+                # 마지막 조각: separator가 더 이상 없으므로 전체 남은 부분을 추가합니다.
+                piece_infos.append(
+                    cls(
+                        text=full_text[start_idx:],
+                        start_idx=start_idx,
+                        end_idx=len(full_text),
+                    )
+                )
+                break
+            else:
+                piece_infos.append(
+                    cls(
+                        text=full_text[start_idx:idx],
+                        start_idx=start_idx,
+                        end_idx=idx,
+                    )
+                )
+                start_idx = idx + sep_len
+
+        current_chunk: list[Self] = []
+        current_token_count: int = 0
+        i = 0
+        while i < len(piece_infos):
+            piece_info = piece_infos[i]
+            piece = piece_info.text
+            piece_start = piece_info.start_idx
+            piece_end = piece_info.end_idx
+            # 원래 코드는 각 조각에 separator의 토큰 수도 포함합니다.
+            piece_token_count: int = len_func(piece) + sep_token_count
+
+            # 현재 청크에 추가하면 chunk_size를 초과하는 경우
+            if current_token_count + piece_token_count > chunk_size:
+                # 단일 조각이 chunk_size보다 큰 경우엔 어쩔 수 없이 추가합니다.
+                if not current_chunk:
+                    current_chunk.append(
+                        cls(
+                            text=piece,
+                            start_idx=piece_start,
+                            end_idx=piece_end,
+                        )
+                    )
+                    current_token_count += piece_token_count
+                    i += 1
+                # 현재 청크 완성 → 청크에 추가
+                chunk_start = current_chunk[0].start_idx
+                # current_chunk에 담긴 조각들은 원본 텍스트상 연속되어 있으므로,
+                # 청크의 종료 인덱스는 마지막 조각의 end_index가 됩니다.
+                chunk_end = current_chunk[-1].end_idx
+                # 원본 텍스트의 해당 구간을 그대로 추출하면 separator가 포함됩니다.
+                chunk_text = full_text[chunk_start:chunk_end]
+                text_chunks.append(
+                    cls(
+                        text=chunk_text,
+                        start_idx=chunk_start,
+                        end_idx=chunk_end,
+                    )
+                )
+
+                # token_overlap이 적용되는 경우: 청크 끝부분 일부를 다음 청크에 오버랩합니다.
+                if token_overlap > 0:
+                    overlap_chunk: list[Self] = []
+                    overlap_count: int = 0
+                    # 뒤에서부터 역순으로 오버랩할 조각들을 선택합니다.
+                    for j in range(len(current_chunk) - 1, -1, -1):
+                        p_text = current_chunk[j].text
+                        p_token_count = len_func(p_text) + sep_token_count
+                        # 최소 한 조각은 포함하고, 오버랩 토큰 수가 token_overlap 이하라면 계속 추가
+                        if overlap_count + p_token_count <= token_overlap or not overlap_chunk:
+                            overlap_chunk.insert(0, current_chunk[j])
+                            overlap_count += p_token_count
+                        else:
+                            break
+                    current_chunk = overlap_chunk.copy()
+                    current_token_count = overlap_count
+                else:
+                    current_chunk.clear()
+                    current_token_count = 0
+            else:
+                # 청크에 추가 후 다음 조각 진행
+                current_chunk.append(cls(text=piece, start_idx=piece_start, end_idx=piece_end))
+                current_token_count += piece_token_count
+                i += 1
+
+        # 남은 조각이 있다면 마지막 청크로 추가합니다.
+        if current_chunk:
+            chunk_start = current_chunk[0].start_idx
+            chunk_end = current_chunk[-1].end_idx
+            chunk_text = full_text[chunk_start:chunk_end]
+            text_chunks.append(cls(text=chunk_text, start_idx=chunk_start, end_idx=chunk_end))
+
+        return text_chunks

chatterer/tools/convert_to_text.py

@@ -0,0 +1,466 @@
+import ast
+import importlib
+import os
+import re
+import site
+from contextlib import contextmanager, suppress
+from fnmatch import fnmatch
+from io import BufferedReader, BufferedWriter, BytesIO, StringIO, TextIOWrapper
+from pathlib import Path
+from typing import (
+    TYPE_CHECKING,
+    Callable,
+    Iterator,
+    NamedTuple,
+    NotRequired,
+    Optional,
+    Self,
+    Sequence,
+    TypeAlias,
+    TypedDict,
+)
+
+if TYPE_CHECKING:
+    from bs4 import Tag
+    from openai import OpenAI
+    from requests import Response, Session
+
+try:
+    from tiktoken import get_encoding, list_encoding_names
+
+    enc = get_encoding(list_encoding_names()[-1])
+except ImportError:
+    enc = None
+
+
+# Type definition for representing a file tree structure
+type FileTree = dict[str, Optional[FileTree]]
+
+# Type aliases for callback functions and file descriptors
+CodeLanguageCallback: TypeAlias = Callable[["Tag"], Optional[str]]
+FileDescriptorOrPath: TypeAlias = int | str | bytes | os.PathLike[str] | os.PathLike[bytes]
+
+# Type aliases for different types of IO objects
+BytesReadable: TypeAlias = BytesIO | BufferedReader
+BytesWritable: TypeAlias = BytesIO | BufferedWriter
+StringReadable: TypeAlias = StringIO | TextIOWrapper
+StringWritable: TypeAlias = StringIO | TextIOWrapper
+
+# Combined type aliases for readable and writable objects
+Readable: TypeAlias = BytesReadable | StringReadable
+Writable: TypeAlias = BytesWritable | StringWritable
+
+# Type alias for path or readable object
+PathOrReadable: TypeAlias = FileDescriptorOrPath | Readable
+
+
+class HtmlToMarkdownOptions(TypedDict):
+    """
+    TypedDict for options used in HTML to Markdown conversion.
+
+    Contains various configuration options for controlling how HTML is converted to Markdown,
+    including formatting preferences, escape behaviors, and styling options.
+    """
+
+    autolinks: NotRequired[bool]
+    bullets: NotRequired[str]
+    code_language: NotRequired[str]
+    code_language_callback: NotRequired[CodeLanguageCallback]
+    convert: NotRequired[Sequence[str]]
+    default_title: NotRequired[bool]
+    escape_asterisks: NotRequired[bool]
+    escape_underscores: NotRequired[bool]
+    escape_misc: NotRequired[bool]
+    heading_style: NotRequired[str]
+    keep_inline_images_in: NotRequired[Sequence[str]]
+    newline_style: NotRequired[str]
+    strip: NotRequired[Sequence[str]]
+    strip_document: NotRequired[str]
+    strong_em_symbol: NotRequired[str]
+    sub_symbol: NotRequired[str]
+    sup_symbol: NotRequired[str]
+    table_infer_header: NotRequired[bool]
+    wrap: NotRequired[bool]
+    wrap_width: NotRequired[int]
+
+
+def get_default_html_to_markdown_options() -> HtmlToMarkdownOptions:
+    """
+    Returns the default options for HTML to Markdown conversion.
+
+    This function provides a set of sensible defaults for the markdownify library,
+    including settings for bullets, escaping, heading styles, and other formatting options.
+
+    Returns:
+        HtmlToMarkdownOptions: A dictionary of default conversion options.
+    """
+    from markdownify import (  # pyright: ignore[reportUnknownVariableType, reportMissingTypeStubs]
+        ASTERISK,
+        SPACES,
+        STRIP,
+        UNDERLINED,
+    )
+
+    return {
+        "autolinks": True,
+        "bullets": "*+-",  # An iterable of bullet types.
+        "code_language": "",
+        "default_title": False,
+        "escape_asterisks": True,
+        "escape_underscores": True,
+        "escape_misc": False,
+        "heading_style": UNDERLINED,
+        "keep_inline_images_in": [],
+        "newline_style": SPACES,
+        "strip_document": STRIP,
+        "strong_em_symbol": ASTERISK,
+        "sub_symbol": "",
+        "sup_symbol": "",
+        "table_infer_header": False,
+        "wrap": False,
+        "wrap_width": 80,
+    }
+
+
+class CodeSnippets(NamedTuple):
+    """
+    A named tuple that represents code snippets extracted from Python files.
+
+    Contains the paths to the files, the concatenated text of all snippets,
+    and the base directory of the files.
+    """
+
+    paths: list[Path]
+    snippets_text: str
+    base_dir: Path
+
+    @classmethod
+    def from_path_or_pkgname(cls, path_or_pkgname: str, ban_file_patterns: Optional[list[str]] = None) -> Self:
+        """
+        Creates a CodeSnippets instance from a file path or package name.
+
+        Args:
+            path_or_pkgname: Path to a file/directory or a Python package name.
+            ban_file_patterns: Optional list of patterns to exclude files.
+
+        Returns:
+            A new CodeSnippets instance with extracted code snippets.
+        """
+        paths: list[Path] = _get_pyscript_paths(path_or_pkgname=path_or_pkgname, ban_fn_patterns=ban_file_patterns)
+        snippets_text: str = "".join(_get_a_snippet(p) for p in paths)
+        return cls(
+            paths=paths,
+            snippets_text=snippets_text,
+            base_dir=_get_base_dir(paths),
+        )
+
+    @property
+    def metadata(self) -> str:
+        """
+        Generates metadata about the code snippets.
+
+        Returns a string containing information about the file tree structure,
+        total number of files, tokens (if tiktoken is available), and lines.
+
+        Returns:
+            str: Formatted metadata string.
+        """
+        file_paths: list[Path] = self.paths
+        text: str = self.snippets_text
+
+        base_dir: Path = _get_base_dir(file_paths)
+        results: list[str] = [base_dir.as_posix()]
+
+        file_tree: FileTree = {}
+        for file_path in sorted(file_paths):
+            rel_path = file_path.relative_to(base_dir)
+            subtree: Optional[FileTree] = file_tree
+            for part in rel_path.parts[:-1]:
+                if subtree is not None:
+                    subtree = subtree.setdefault(part, {})
+            if subtree is not None:
+                subtree[rel_path.parts[-1]] = None
+
+        def _display_tree(tree: FileTree, prefix: str = "") -> None:
+            """
+            Helper function to recursively display a file tree structure.
+
+            Args:
+                tree: The file tree dictionary to display.
+                prefix: Current line prefix for proper indentation.
+            """
+            items: list[tuple[str, Optional[FileTree]]] = sorted(tree.items())
+            count: int = len(items)
+            for idx, (name, subtree) in enumerate(items):
+                branch: str = "└── " if idx == count - 1 else "├── "
+                results.append(f"{prefix}{branch}{name}")
+                if subtree is not None:
+                    extension: str = "    " if idx == count - 1 else "│   "
+                    _display_tree(tree=subtree, prefix=prefix + extension)
+
+        _display_tree(file_tree)
+        results.append(f"- Total files: {len(file_paths)}")
+        if enc is not None:
+            num_tokens: int = len(enc.encode(text, disallowed_special=()))
+            results.append(f"- Total tokens: {num_tokens}")
+        results.append(f"- Total lines: {text.count('\n') + 1}")
+        return "\n".join(results)
+
+
+def html_to_markdown(html: str, options: Optional[HtmlToMarkdownOptions]) -> str:
+    """
+    Convert HTML content to Markdown using the provided options.
+
+    Args:
+        html (str): HTML content to convert.
+        options (HtmlToMarkdownOptions): Options for the conversion.
+
+    Returns:
+        str: The Markdown content.
+    """
+    from markdownify import markdownify  # pyright: ignore[reportUnknownVariableType, reportMissingTypeStubs]
+
+    return str(markdownify(html, **(options or {})))  # pyright: ignore[reportUnknownArgumentType]
+
+
+def pdf_to_text(path_or_file: PathOrReadable) -> str:
+    """
+    Convert a PDF file to plain text.
+
+    Extracts text from each page of a PDF file and formats it with page markers.
+
+    Args:
+        path_or_file: Path to a PDF file or a readable object containing PDF data.
+
+    Returns:
+        str: Extracted text with page markers.
+
+    Raises:
+        FileNotFoundError: If the file cannot be found or opened.
+    """
+    from pymupdf import Document  # pyright: ignore[reportMissingTypeStubs]
+
+    with _open_stream(path_or_file) as stream:
+        if stream is None:
+            raise FileNotFoundError(path_or_file)
+        return "\n".join(
+            f"<!-- Page {page_no} -->\n{text.strip()}\n"
+            for page_no, text in enumerate(
+                (
+                    page.get_textpage().extractText()  # pyright: ignore[reportUnknownMemberType]
+                    for page in Document(stream=stream.read())
+                ),
+                1,
+            )
+        )
+
+
+def anything_to_markdown(
+    source: "str | Response | Path",
+    requests_session: Optional["Session"] = None,
+    llm_client: Optional["OpenAI"] = None,
+    llm_model: Optional[str] = None,
+    style_map: Optional[str] = None,
+    exiftool_path: Optional[str] = None,
+    docintel_endpoint: Optional[str] = None,
+) -> str:
+    """
+    Convert various types of content to Markdown format.
+
+    Uses the MarkItDown library to convert different types of content (URLs, files, API responses)
+    to Markdown format.
+
+    Args:
+        source: The source content to convert (URL string, Response object, or Path).
+        requests_session: Optional requests Session for HTTP requests.
+        llm_client: Optional OpenAI client for LLM-based conversions.
+        llm_model: Optional model name for the LLM.
+        style_map: Optional style mapping configuration.
+        exiftool_path: Optional path to exiftool for metadata extraction.
+        docintel_endpoint: Optional Document Intelligence API endpoint.
+
+    Returns:
+        str: The converted Markdown content.
+    """
+    from markitdown import MarkItDown
+
+    result = MarkItDown(
+        requests_session=requests_session,
+        llm_client=llm_client,
+        llm_model=llm_model,
+        style_map=style_map,
+        exiftool_path=exiftool_path,
+        docintel_endpoint=docintel_endpoint,
+    ).convert(source)
+    return result.text_content
+
+
+# Alias for CodeSnippets.from_path_or_pkgname for backward compatibility
+pyscripts_to_snippets = CodeSnippets.from_path_or_pkgname
+
+
+def _pattern_to_regex(pattern: str) -> re.Pattern[str]:
+    """
+    Converts an fnmatch pattern to a regular expression.
+
+    In this function, '**' is converted to match any character including directory separators.
+    The remaining '*' matches any character except directory separators, and '?' matches a single character.
+
+    Args:
+        pattern: The fnmatch pattern to convert.
+
+    Returns:
+        A compiled regular expression pattern.
+    """
+    # First escape the pattern
+    pattern = re.escape(pattern)
+    # Convert '**' to match any character including directory separators ('.*')
+    pattern = pattern.replace(r"\*\*", ".*")
+    # Then convert single '*' to match any character except directory separators
+    pattern = pattern.replace(r"\*", "[^/]*")
+    # Convert '?' to match a single character
+    pattern = pattern.replace(r"\?", ".")
+    # Anchor the pattern to start and end
+    pattern = "^" + pattern + "$"
+    return re.compile(pattern)
+
+
+def _is_banned(p: Path, ban_patterns: list[str]) -> bool:
+    """
+    Checks if a given path matches any of the ban patterns.
+
+    Determines if the path p matches any pattern in ban_patterns using either
+    fnmatch-based or recursive patterns (i.e., containing '**').
+
+    Note: Patterns should use POSIX-style paths (i.e., '/' separators).
+
+    Args:
+        p: The path to check.
+        ban_patterns: List of patterns to match against.
+
+    Returns:
+        bool: True if the path matches any ban pattern, False otherwise.
+    """
+    p_str = p.as_posix()
+    for pattern in ban_patterns:
+        if "**" in pattern:
+            regex = _pattern_to_regex(pattern)
+            if regex.match(p_str):
+                return True
+        else:
+            # Simple fnmatch: '*' by default doesn't match '/'
+            if fnmatch(p_str, pattern):
+                return True
+    return False
+
+
+def _get_a_snippet(fpath: Path) -> str:
+    """
+    Extracts a code snippet from a Python file.
+
+    Reads the file, parses it as Python code, and returns a formatted code snippet
+    with the relative path as a header in markdown code block format.
+
+    Args:
+        fpath: Path to the Python file.
+
+    Returns:
+        str: Formatted code snippet or empty string if the file doesn't exist.
+    """
+    if not fpath.is_file():
+        return ""
+
+    cleaned_code: str = "\n".join(
+        line for line in ast.unparse(ast.parse(fpath.read_text(encoding="utf-8"))).splitlines()
+    )
+    if site_dir := next(
+        (d for d in reversed(site.getsitepackages()) if fpath.is_relative_to(d)),
+        None,
+    ):
+        display_path = fpath.relative_to(site_dir)
+    elif fpath.is_relative_to(cwd := Path.cwd()):
+        display_path = fpath.relative_to(cwd)
+    else:
+        display_path = fpath.absolute()
+    return f"```{display_path}\n{cleaned_code}\n```\n\n"
+
+
+def _get_base_dir(target_files: Sequence[Path]) -> Path:
+    """
+    Determines the common base directory for a sequence of file paths.
+
+    Finds the directory with the shortest path that is a parent to at least one file.
+
+    Args:
+        target_files: Sequence of file paths.
+
+    Returns:
+        Path: The common base directory.
+    """
+    return sorted(
+        {file_path.parent for file_path in target_files},
+        key=lambda p: len(p.parts),
+    )[0]
+
+
+def _get_pyscript_paths(path_or_pkgname: str, ban_fn_patterns: Optional[list[str]] = None) -> list[Path]:
+    """
+    Gets paths to Python script files from a directory, file, or package name.
+
+    If path_or_pkgname is a directory, finds all .py files recursively.
+    If it's a file, returns just that file.
+    If it's a package name, imports the package and finds all .py files in its directory.
+
+    Args:
+        path_or_pkgname: Path to directory/file or package name.
+        ban_fn_patterns: Optional list of patterns to exclude files.
+
+    Returns:
+        list[Path]: List of paths to Python files.
+    """
+    path = Path(path_or_pkgname)
+    pypaths: list[Path]
+    if path.is_dir():
+        pypaths = list(path.rglob("*.py", case_sensitive=False))
+    elif path.is_file():
+        pypaths = [path]
+    else:
+        pypaths = [
+            p
+            for p in Path(next(iter(importlib.import_module(path_or_pkgname).__path__))).rglob(
+                "*.py", case_sensitive=False
+            )
+            if p.is_file()
+        ]
+    return [p for p in pypaths if ban_fn_patterns and not _is_banned(p, ban_fn_patterns)]
+
+
+@contextmanager
+def _open_stream(
+    path_or_file: PathOrReadable,
+) -> Iterator[Optional[BytesReadable]]:
+    """
+    Context manager for opening a file or using an existing stream.
+
+    Handles different types of input (file paths, byte streams, string streams)
+    and yields a BytesReadable object that can be used to read binary data.
+
+    Args:
+        path_or_file: File path or readable object.
+
+    Yields:
+        Optional[BytesReadable]: A readable binary stream or None if opening fails.
+    """
+    stream: Optional[BytesReadable] = None
+    try:
+        with suppress(BaseException):
+            if isinstance(path_or_file, BytesReadable):
+                stream = path_or_file
+            elif isinstance(path_or_file, StringReadable):
+                stream = BytesIO(path_or_file.read().encode("utf-8"))
+            else:
+                stream = open(path_or_file, "rb")
+        yield stream
+    finally:
+        if stream is not None:
+            stream.close()

chatterer/tools/webpage_to_markdown/__init__.py

@@ -0,0 +1,4 @@
+from .playwright_bot import PlayWrightBot
+from .utils import MarkdownLink
+
+__all__ = ["PlayWrightBot", "MarkdownLink"]