chatterer 0.1.12__py3-none-any.whl → 0.1.13__py3-none-any.whl

chatterer/tools/youtube.py

@@ -1,146 +1,146 @@
-import json
-import unicodedata
-import urllib.parse
-from dataclasses import dataclass
-from typing import Any, Optional, Self, cast
-
-import requests
-
-
-def get_youtube_video_details(
-    query: str,
-) -> list[dict[str, Optional[str]]]:
-    """Search for video metadata on YouTube using the given query. Returns a list of dictionaries containing `video_id`, `title`, `channel`, `duration`, `views`, `publish_time`, and `long_desc`."""
-    return [
-        {
-            "video_id": video_id,
-            "title": video.title,
-            "channel": video.channel,
-            "duration": video.duration,
-            "views": video.views,
-            "publish_time": video.publish_time,
-            "long_desc": video.long_desc,
-        }
-        for video in YoutubeSearchResult.from_query(base_url="https://youtube.com", query=query, max_results=10)
-        if (video_id := _get_video_id(video.url_suffix))
-    ]
-
-
-def get_youtube_video_subtitle(video_id: str) -> str:
-    """Get the transcript of a YouTube video using the given video ID."""
-
-    from youtube_transcript_api._api import YouTubeTranscriptApi
-
-    get_transcript = YouTubeTranscriptApi.get_transcript  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]
-    list_transcripts = YouTubeTranscriptApi.list_transcripts  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]
-
-    result: str = ""
-    buffer_timestamp: str = "0s"
-    buffer_texts: list[str] = []
-    for entry in get_transcript(video_id, languages=(next(iter(list_transcripts(video_id))).language_code,)):  # pyright: ignore[reportUnknownVariableType]
-        entry = cast(dict[object, object], entry)
-        text: str = str(entry.get("text", "")).strip().replace("\n", " ")
-        if not text:
-            continue
-        if len(buffer_texts) >= 10 or _is_special_char(text) or (buffer_texts and _is_special_char(buffer_texts[-1])):
-            result += f"[{buffer_timestamp}] {'. '.join(buffer_texts)}\n"
-            start = entry.get("start", 0)
-            if start:
-                buffer_timestamp = f"{start:.0f}s"
-            buffer_texts = [text]
-        else:
-            buffer_texts.append(text)
-
-    if buffer_texts:
-        result += f"[{buffer_timestamp}] {' '.join(buffer_texts)}"
-    return result
-
-
-def _get_video_id(suffix: str) -> str:
-    urllib_parse_result = urllib.parse.urlparse(suffix)
-    if urllib_parse_result.path.startswith("/shorts/"):
-        # Fore shorts (/shorts/...) the video ID is in the path
-        parts = urllib_parse_result.path.split("/")
-        if len(parts) < 3:
-            print(f"Failed to get video ID from {suffix}")
-            return ""
-        return parts[2]
-
-    query: str = urllib.parse.urlparse(suffix).query
-    query_strings = urllib.parse.parse_qs(query)
-    if "v" not in query_strings:
-        print(f"Failed to get video ID from {suffix}")
-        return ""
-    return next(iter(query_strings["v"]), "")
-
-
-def _is_special_char(text: str) -> bool:
-    if not text:
-        return False
-    return not unicodedata.category(text[0]).startswith("L")
-
-
-@dataclass
-class YoutubeSearchResult:
-    url_suffix: str
-    id: Optional[str]
-    thumbnails: list[str]
-    title: Optional[str]
-    long_desc: Optional[str]
-    channel: Optional[str]
-    duration: Optional[str]
-    views: Optional[str]
-    publish_time: Optional[str]
-
-    @classmethod
-    def from_query(cls, base_url: str, query: str, max_results: int) -> list[Self]:
-        url: str = f"{base_url}/results?search_query={urllib.parse.quote_plus(query)}"
-        response: str = requests.get(url).text
-        while "ytInitialData" not in response:
-            response = requests.get(url).text
-        results: list[Self] = cls.parse_html(response)
-        return results[:max_results]
-
-    @classmethod
-    def parse_html(cls, html: str) -> list[Self]:
-        results: list[Self] = []
-        start: int = html.index("ytInitialData") + len("ytInitialData") + 3
-        end: int = html.index("};", start) + 1
-        data: Any = json.loads(html[start:end])
-        for contents in data["contents"]["twoColumnSearchResultsRenderer"]["primaryContents"]["sectionListRenderer"][
-            "contents"
-        ]:
-            for video in contents["itemSectionRenderer"]["contents"]:
-                if "videoRenderer" in video.keys():
-                    video_data = video.get("videoRenderer", {})
-                    suffix = (
-                        video_data.get("navigationEndpoint", {})
-                        .get("commandMetadata", {})
-                        .get("webCommandMetadata", {})
-                        .get("url", None)
-                    )
-                    if not suffix:
-                        continue
-                    res = cls(
-                        id=video_data.get("videoId", None),
-                        thumbnails=[
-                            thumb.get("url", None) for thumb in video_data.get("thumbnail", {}).get("thumbnails", [{}])
-                        ],
-                        title=video_data.get("title", {}).get("runs", [[{}]])[0].get("text", None),
-                        long_desc=video_data.get("descriptionSnippet", {}).get("runs", [{}])[0].get("text", None),
-                        channel=video_data.get("longBylineText", {}).get("runs", [[{}]])[0].get("text", None),
-                        duration=video_data.get("lengthText", {}).get("simpleText", 0),
-                        views=video_data.get("viewCountText", {}).get("simpleText", 0),
-                        publish_time=video_data.get("publishedTimeText", {}).get("simpleText", 0),
-                        url_suffix=suffix,
-                    )
-                    results.append(res)
-
-            if results:
-                break
-        return results
-
-
-if __name__ == "__main__":
-    print(get_youtube_video_details("BTS"))
-    # print(get_youtube_transcript("y7jrpS8GHxs"))
+import json
+import unicodedata
+import urllib.parse
+from dataclasses import dataclass
+from typing import Any, Optional, Self, cast
+
+import requests
+
+
+def get_youtube_video_details(
+    query: str,
+) -> list[dict[str, Optional[str]]]:
+    """Search for video metadata on YouTube using the given query. Returns a list of dictionaries containing `video_id`, `title`, `channel`, `duration`, `views`, `publish_time`, and `long_desc`."""
+    return [
+        {
+            "video_id": video_id,
+            "title": video.title,
+            "channel": video.channel,
+            "duration": video.duration,
+            "views": video.views,
+            "publish_time": video.publish_time,
+            "long_desc": video.long_desc,
+        }
+        for video in YoutubeSearchResult.from_query(base_url="https://youtube.com", query=query, max_results=10)
+        if (video_id := _get_video_id(video.url_suffix))
+    ]
+
+
+def get_youtube_video_subtitle(video_id: str) -> str:
+    """Get the transcript of a YouTube video using the given video ID."""
+
+    from youtube_transcript_api._api import YouTubeTranscriptApi
+
+    get_transcript = YouTubeTranscriptApi.get_transcript  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]
+    list_transcripts = YouTubeTranscriptApi.list_transcripts  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]
+
+    result: str = ""
+    buffer_timestamp: str = "0s"
+    buffer_texts: list[str] = []
+    for entry in get_transcript(video_id, languages=(next(iter(list_transcripts(video_id))).language_code,)):  # pyright: ignore[reportUnknownVariableType]
+        entry = cast(dict[object, object], entry)
+        text: str = str(entry.get("text", "")).strip().replace("\n", " ")
+        if not text:
+            continue
+        if len(buffer_texts) >= 10 or _is_special_char(text) or (buffer_texts and _is_special_char(buffer_texts[-1])):
+            result += f"[{buffer_timestamp}] {'. '.join(buffer_texts)}\n"
+            start = entry.get("start", 0)
+            if start:
+                buffer_timestamp = f"{start:.0f}s"
+            buffer_texts = [text]
+        else:
+            buffer_texts.append(text)
+
+    if buffer_texts:
+        result += f"[{buffer_timestamp}] {' '.join(buffer_texts)}"
+    return result
+
+
+def _get_video_id(suffix: str) -> str:
+    urllib_parse_result = urllib.parse.urlparse(suffix)
+    if urllib_parse_result.path.startswith("/shorts/"):
+        # Fore shorts (/shorts/...) the video ID is in the path
+        parts = urllib_parse_result.path.split("/")
+        if len(parts) < 3:
+            print(f"Failed to get video ID from {suffix}")
+            return ""
+        return parts[2]
+
+    query: str = urllib.parse.urlparse(suffix).query
+    query_strings = urllib.parse.parse_qs(query)
+    if "v" not in query_strings:
+        print(f"Failed to get video ID from {suffix}")
+        return ""
+    return next(iter(query_strings["v"]), "")
+
+
+def _is_special_char(text: str) -> bool:
+    if not text:
+        return False
+    return not unicodedata.category(text[0]).startswith("L")
+
+
+@dataclass
+class YoutubeSearchResult:
+    url_suffix: str
+    id: Optional[str]
+    thumbnails: list[str]
+    title: Optional[str]
+    long_desc: Optional[str]
+    channel: Optional[str]
+    duration: Optional[str]
+    views: Optional[str]
+    publish_time: Optional[str]
+
+    @classmethod
+    def from_query(cls, base_url: str, query: str, max_results: int) -> list[Self]:
+        url: str = f"{base_url}/results?search_query={urllib.parse.quote_plus(query)}"
+        response: str = requests.get(url).text
+        while "ytInitialData" not in response:
+            response = requests.get(url).text
+        results: list[Self] = cls.parse_html(response)
+        return results[:max_results]
+
+    @classmethod
+    def parse_html(cls, html: str) -> list[Self]:
+        results: list[Self] = []
+        start: int = html.index("ytInitialData") + len("ytInitialData") + 3
+        end: int = html.index("};", start) + 1
+        data: Any = json.loads(html[start:end])
+        for contents in data["contents"]["twoColumnSearchResultsRenderer"]["primaryContents"]["sectionListRenderer"][
+            "contents"
+        ]:
+            for video in contents["itemSectionRenderer"]["contents"]:
+                if "videoRenderer" in video.keys():
+                    video_data = video.get("videoRenderer", {})
+                    suffix = (
+                        video_data.get("navigationEndpoint", {})
+                        .get("commandMetadata", {})
+                        .get("webCommandMetadata", {})
+                        .get("url", None)
+                    )
+                    if not suffix:
+                        continue
+                    res = cls(
+                        id=video_data.get("videoId", None),
+                        thumbnails=[
+                            thumb.get("url", None) for thumb in video_data.get("thumbnail", {}).get("thumbnails", [{}])
+                        ],
+                        title=video_data.get("title", {}).get("runs", [[{}]])[0].get("text", None),
+                        long_desc=video_data.get("descriptionSnippet", {}).get("runs", [{}])[0].get("text", None),
+                        channel=video_data.get("longBylineText", {}).get("runs", [[{}]])[0].get("text", None),
+                        duration=video_data.get("lengthText", {}).get("simpleText", 0),
+                        views=video_data.get("viewCountText", {}).get("simpleText", 0),
+                        publish_time=video_data.get("publishedTimeText", {}).get("simpleText", 0),
+                        url_suffix=suffix,
+                    )
+                    results.append(res)
+
+            if results:
+                break
+        return results
+
+
+if __name__ == "__main__":
+    print(get_youtube_video_details("BTS"))
+    # print(get_youtube_transcript("y7jrpS8GHxs"))

chatterer/utils/__init__.py

@@ -1,15 +1,15 @@
-from .code_agent import (
-    CodeExecutionResult,
-    FunctionSignature,
-    get_default_repl_tool,
-    insert_callables_into_global,
-)
-from .image import Base64Image
-
-__all__ = [
-    "Base64Image",
-    "FunctionSignature",
-    "CodeExecutionResult",
-    "get_default_repl_tool",
-    "insert_callables_into_global",
-]
+from .code_agent import (
+    CodeExecutionResult,
+    FunctionSignature,
+    get_default_repl_tool,
+    insert_callables_into_global,
+)
+from .image import Base64Image
+
+__all__ = [
+    "Base64Image",
+    "FunctionSignature",
+    "CodeExecutionResult",
+    "get_default_repl_tool",
+    "insert_callables_into_global",
+]

chatterer/utils/bytesio.py

@@ -0,0 +1,59 @@
+import os
+from contextlib import contextmanager, suppress
+from io import BytesIO
+from typing import Iterator, Optional
+
+from ..common_types.io import BytesReadable, PathOrReadable, StringReadable
+
+
+@contextmanager
+def read_bytes_stream(
+    path_or_file: PathOrReadable,
+    assume_pathlike_bytes_as_path: bool = False,
+    assume_pathlike_string_as_path: bool = True,
+) -> Iterator[Optional[BytesReadable]]:
+    """
+    Context manager for opening a file or using an existing stream.
+
+    Handles different types of input (file paths, byte streams, string streams)
+    and yields a BytesReadable object that can be used to read binary data.
+
+    Args:
+        path_or_file: File path or readable object.
+        assume_pathlike_bytes_as_path: If True, assume bytes-like objects are file paths. Else, treat as data itself.
+        assume_pathlike_string_as_path: If True, assume string-like objects are file paths. Else, treat as data itself.
+
+    Yields:
+        Optional[BytesReadable]: A readable binary stream or None if opening fails.
+    """
+    stream: Optional[BytesReadable] = None
+    should_close: bool = True  # Whether the stream should be closed after use
+    try:
+        with suppress(BaseException):
+            if isinstance(path_or_file, BytesReadable):
+                # Assume the input is already a bytes stream
+                # NOTE: Delivers itself, so shouldn't be closed.
+                stream = path_or_file
+                should_close = False
+            elif isinstance(path_or_file, StringReadable):
+                # Convert the string stream to bytes stream
+                stream = BytesIO(path_or_file.read().encode("utf-8"))
+            elif isinstance(path_or_file, bytes):
+                # Convert the bytes-like object to bytes stream
+                if assume_pathlike_bytes_as_path and os.path.exists(path_or_file):
+                    stream = open(path_or_file, "rb")
+                else:
+                    stream = BytesIO(path_or_file)
+            elif isinstance(path_or_file, str):
+                # Convert the file path to bytes stream
+                if assume_pathlike_string_as_path and os.path.exists(path_or_file):
+                    stream = open(path_or_file, "rb")
+                else:
+                    stream = BytesIO(path_or_file.encode("utf-8"))
+            else:
+                # Assume the input is a file descriptor or path
+                stream = open(path_or_file, "rb")
+        yield stream
+    finally:
+        if stream is not None and should_close:
+            stream.close()

chatterer/utils/code_agent.py

@@ -1,138 +1,138 @@
-import inspect
-import textwrap
-from typing import (
-    TYPE_CHECKING,
-    Callable,
-    Iterable,
-    NamedTuple,
-    Optional,
-    Self,
-)
-
-from langchain_core.runnables.config import RunnableConfig
-
-if TYPE_CHECKING:
-    from langchain_experimental.tools import PythonAstREPLTool
-
-
-class FunctionSignature(NamedTuple):
-    name: str
-    callable: Callable[..., object]
-    signature: str
-
-    @classmethod
-    def from_callable(cls, callables: Optional[Callable[..., object] | Iterable[Callable[..., object]]]) -> list[Self]:
-        if callables is None:
-            return []
-        if callable(callables):
-            return [cls._from_callable(callables)]
-        return [cls._from_callable(callable) for callable in callables]
-
-    @classmethod
-    def _from_callable(cls, callable: Callable[..., object]) -> Self:
-        """
-        Get the name and signature of a function as a string.
-        """
-        # Determine if the function is async
-        is_async_func = inspect.iscoroutinefunction(callable)
-        function_def = "async def" if is_async_func else "def"
-
-        # Determine the function name based on the type of callable
-        if inspect.isfunction(callable):
-            # For regular Python functions, use __code__.co_name
-            function_name = callable.__code__.co_name
-        elif hasattr(callable, "name"):
-            # For StructuredTool or similar objects with a 'name' attribute
-            function_name = callable.name  # type: ignore
-        elif hasattr(callable, "__name__"):
-            # For other callables with a __name__ attribute
-            function_name = callable.__name__
-        else:
-            # Fallback to the class name if no name is found
-            function_name = type(callable).__name__
-
-        # Build the signature string
-        signature = f"{function_def} {function_name}{inspect.signature(callable)}:"
-        docstring = inspect.getdoc(callable)
-        if docstring:
-            docstring = f'"""{docstring.strip()}"""'
-            return cls(
-                name=function_name, callable=callable, signature=f"{signature}\n{textwrap.indent(docstring, '    ')}"
-            )
-        else:
-            return cls(name=function_name, callable=callable, signature=signature)
-
-    @classmethod
-    def as_prompt(
-        cls,
-        function_signatures: Iterable[Self],
-        prefix: Optional[str] = "You can use the pre-made functions below without defining them:\n",
-        sep: str = "\n---\n",
-    ) -> str:
-        """
-        Generate a prompt string from a list of callables.
-        """
-        body: str = sep.join(fsig.signature for fsig in function_signatures)
-        if prefix:
-            return f"{prefix}{body}"
-        return body
-
-
-class CodeExecutionResult(NamedTuple):
-    code: str
-    output: str
-
-    @classmethod
-    def from_code(
-        cls,
-        code: str,
-        repl_tool: Optional["PythonAstREPLTool"] = None,
-        config: Optional[RunnableConfig] = None,
-        function_signatures: Optional[Iterable[FunctionSignature]] = None,
-        **kwargs: object,
-    ) -> Self:
-        """
-        Execute code using the Python Code Execution Language Model.
-        """
-        if repl_tool is None:
-            repl_tool = get_default_repl_tool()
-        if function_signatures:
-            insert_callables_into_global(function_signatures=function_signatures, repl_tool=repl_tool)
-        output = str(repl_tool.invoke(code, config=config, **kwargs))  # pyright: ignore[reportUnknownMemberType]
-        return cls(code=code, output=output)
-
-    @classmethod
-    async def afrom_code(
-        cls,
-        code: str,
-        repl_tool: Optional["PythonAstREPLTool"] = None,
-        config: Optional[RunnableConfig] = None,
-        function_signatures: Optional[Iterable[FunctionSignature]] = None,
-        **kwargs: object,
-    ) -> Self:
-        """
-        Execute code using the Python Code Execution Language Model asynchronously.
-        """
-        if repl_tool is None:
-            repl_tool = get_default_repl_tool()
-        if function_signatures:
-            insert_callables_into_global(function_signatures=function_signatures, repl_tool=repl_tool)
-        output = str(await repl_tool.ainvoke(code, config=config, **kwargs))  # pyright: ignore[reportUnknownMemberType]
-        return cls(code=code, output=output)
-
-
-def get_default_repl_tool() -> "PythonAstREPLTool":
-    from langchain_experimental.tools import PythonAstREPLTool
-
-    return PythonAstREPLTool()
-
-
-def insert_callables_into_global(
-    function_signatures: Iterable[FunctionSignature], repl_tool: "PythonAstREPLTool"
-) -> None:
-    """Insert callables into the REPL tool's globals."""
-    repl_globals: Optional[dict[str, object]] = repl_tool.globals  # pyright: ignore[reportUnknownMemberType]
-    if repl_globals is None:
-        repl_tool.globals = {fsig.name: fsig.callable for fsig in function_signatures}
-    else:
-        repl_globals.update({fsig.name: fsig.callable for fsig in function_signatures})
+import inspect
+import textwrap
+from typing import (
+    TYPE_CHECKING,
+    Callable,
+    Iterable,
+    NamedTuple,
+    Optional,
+    Self,
+)
+
+from langchain_core.runnables.config import RunnableConfig
+
+if TYPE_CHECKING:
+    from langchain_experimental.tools import PythonAstREPLTool
+
+
+class FunctionSignature(NamedTuple):
+    name: str
+    callable: Callable[..., object]
+    signature: str
+
+    @classmethod
+    def from_callable(cls, callables: Optional[Callable[..., object] | Iterable[Callable[..., object]]]) -> list[Self]:
+        if callables is None:
+            return []
+        if callable(callables):
+            return [cls._from_callable(callables)]
+        return [cls._from_callable(callable) for callable in callables]
+
+    @classmethod
+    def _from_callable(cls, callable: Callable[..., object]) -> Self:
+        """
+        Get the name and signature of a function as a string.
+        """
+        # Determine if the function is async
+        is_async_func = inspect.iscoroutinefunction(callable)
+        function_def = "async def" if is_async_func else "def"
+
+        # Determine the function name based on the type of callable
+        if inspect.isfunction(callable):
+            # For regular Python functions, use __code__.co_name
+            function_name = callable.__code__.co_name
+        elif hasattr(callable, "name"):
+            # For StructuredTool or similar objects with a 'name' attribute
+            function_name = callable.name  # type: ignore
+        elif hasattr(callable, "__name__"):
+            # For other callables with a __name__ attribute
+            function_name = callable.__name__
+        else:
+            # Fallback to the class name if no name is found
+            function_name = type(callable).__name__
+
+        # Build the signature string
+        signature = f"{function_def} {function_name}{inspect.signature(callable)}:"
+        docstring = inspect.getdoc(callable)
+        if docstring:
+            docstring = f'"""{docstring.strip()}"""'
+            return cls(
+                name=function_name, callable=callable, signature=f"{signature}\n{textwrap.indent(docstring, '    ')}"
+            )
+        else:
+            return cls(name=function_name, callable=callable, signature=signature)
+
+    @classmethod
+    def as_prompt(
+        cls,
+        function_signatures: Iterable[Self],
+        prefix: Optional[str] = "You can use the pre-made functions below without defining them:\n",
+        sep: str = "\n---\n",
+    ) -> str:
+        """
+        Generate a prompt string from a list of callables.
+        """
+        body: str = sep.join(fsig.signature for fsig in function_signatures)
+        if prefix:
+            return f"{prefix}{body}"
+        return body
+
+
+class CodeExecutionResult(NamedTuple):
+    code: str
+    output: str
+
+    @classmethod
+    def from_code(
+        cls,
+        code: str,
+        repl_tool: Optional["PythonAstREPLTool"] = None,
+        config: Optional[RunnableConfig] = None,
+        function_signatures: Optional[Iterable[FunctionSignature]] = None,
+        **kwargs: object,
+    ) -> Self:
+        """
+        Execute code using the Python Code Execution Language Model.
+        """
+        if repl_tool is None:
+            repl_tool = get_default_repl_tool()
+        if function_signatures:
+            insert_callables_into_global(function_signatures=function_signatures, repl_tool=repl_tool)
+        output = str(repl_tool.invoke(code, config=config, **kwargs))  # pyright: ignore[reportUnknownMemberType]
+        return cls(code=code, output=output)
+
+    @classmethod
+    async def afrom_code(
+        cls,
+        code: str,
+        repl_tool: Optional["PythonAstREPLTool"] = None,
+        config: Optional[RunnableConfig] = None,
+        function_signatures: Optional[Iterable[FunctionSignature]] = None,
+        **kwargs: object,
+    ) -> Self:
+        """
+        Execute code using the Python Code Execution Language Model asynchronously.
+        """
+        if repl_tool is None:
+            repl_tool = get_default_repl_tool()
+        if function_signatures:
+            insert_callables_into_global(function_signatures=function_signatures, repl_tool=repl_tool)
+        output = str(await repl_tool.ainvoke(code, config=config, **kwargs))  # pyright: ignore[reportUnknownMemberType]
+        return cls(code=code, output=output)
+
+
+def get_default_repl_tool() -> "PythonAstREPLTool":
+    from langchain_experimental.tools import PythonAstREPLTool
+
+    return PythonAstREPLTool()
+
+
+def insert_callables_into_global(
+    function_signatures: Iterable[FunctionSignature], repl_tool: "PythonAstREPLTool"
+) -> None:
+    """Insert callables into the REPL tool's globals."""
+    repl_globals: Optional[dict[str, object]] = repl_tool.globals  # pyright: ignore[reportUnknownMemberType]
+    if repl_globals is None:
+        repl_tool.globals = {fsig.name: fsig.callable for fsig in function_signatures}
+    else:
+        repl_globals.update({fsig.name: fsig.callable for fsig in function_signatures})