chatterer 0.1.8__py3-none-any.whl → 0.1.10__py3-none-any.whl

chatterer/__init__.py

@@ -1,7 +1,8 @@
-from .language_model import Chatterer
+from .language_model import Chatterer, interactive_shell
 from .messages import (
     AIMessage,
     BaseMessage,
+    FunctionMessage,
     HumanMessage,
     SystemMessage,
 )
@@ -15,9 +16,19 @@ from .tools import (
     anything_to_markdown,
     citation_chunker,
     get_default_html_to_markdown_options,
+    get_youtube_video_subtitle,
     html_to_markdown,
+    init_webpage_to_markdown,
     pdf_to_text,
     pyscripts_to_snippets,
+    get_youtube_video_details,
+)
+from .utils import (
+    Base64Image,
+    CodeExecutionResult,
+    FunctionSignature,
+    get_default_repl_tool,
+    insert_callables_into_global,
 )
 
 __all__ = [
@@ -36,4 +47,14 @@ __all__ = [
     "HumanMessage",
     "SystemMessage",
     "AIMessage",
+    "FunctionMessage",
+    "Base64Image",
+    "init_webpage_to_markdown",
+    "FunctionSignature",
+    "CodeExecutionResult",
+    "get_default_repl_tool",
+    "insert_callables_into_global",
+    "get_youtube_video_subtitle",
+    "get_youtube_video_details",
+    "interactive_shell",
 ]

chatterer/language_model.py

@@ -2,9 +2,12 @@ from typing import (
     TYPE_CHECKING,
     Any,
     AsyncIterator,
+    Callable,
+    Iterable,
     Iterator,
     Optional,
     Self,
+    Sequence,
     Type,
     TypeAlias,
     TypeVar,
@@ -18,15 +21,39 @@ from langchain_core.runnables.base import Runnable
 from langchain_core.runnables.config import RunnableConfig
 from pydantic import BaseModel, Field
 
-from .messages import AIMessage, BaseMessage, HumanMessage
+from .messages import AIMessage, BaseMessage, HumanMessage, SystemMessage
+from .utils.code_agent import CodeExecutionResult, FunctionSignature, get_default_repl_tool
 
 if TYPE_CHECKING:
     from instructor import Partial
+    from langchain_experimental.tools.python.tool import PythonAstREPLTool
 
 PydanticModelT = TypeVar("PydanticModelT", bound=BaseModel)
 StructuredOutputType: TypeAlias = dict[object, object] | BaseModel
 
-DEFAULT_IMAGE_DESCRIPTION_INSTRUCTION = "Just describe all the details you see in the image in few sentences."
+DEFAULT_IMAGE_DESCRIPTION_INSTRUCTION = "Provide a detailed description of all visible elements in the image, summarizing key details in a few clear sentences."
+DEFAULT_CODE_GENERATION_PROMPT = (
+    "You are utilizing a Python code execution tool now.\n"
+    "Your goal is to generate Python code that solves the task efficiently and appends both the code and its output to your context memory.\n"
+    "Since your context window is highly limited, type `pass` if no code execution is needed.\n"
+    "\n"
+    "To optimize tool efficiency, follow these guidelines:\n"
+    "- Write concise, efficient code that directly serves the intended purpose.\n"
+    "- Avoid unnecessary operations (e.g., excessive loops, recursion, or heavy computations).\n"
+    "- Handle potential errors gracefully (e.g., using try-except blocks).\n"
+    "- Prevent excessive output by limiting print statements to essential information only (e.g., avoid printing large datasets).\n"
+    "\n"
+    "Return your response strictly in the following JSON format:\n"
+    '{\n  "code": "<your_python_code_here>"\n}\n\n'
+)
+
+
+DEFAULT_FUNCTION_REFERENCE_PREFIX_PROMPT = (
+    "Below functions are included in global scope and can be used in your code.\n"
+    "Do not try to redefine the function(s).\n"
+    "You don't have to force yourself to use these tools - use them only when you need to.\n"
+)
+DEFAULT_FUNCTION_REFERENCE_SEPARATOR = "\n---\n"  # Separator to distinguish different function references
 
 
 class Chatterer(BaseModel):
@@ -162,7 +189,7 @@ class Chatterer(BaseModel):
         stop: Optional[list[str]] = None,
         **kwargs: Any,
     ) -> PydanticModelT:
-        result: StructuredOutputType = with_structured_output(
+        result: StructuredOutputType = _with_structured_output(
             client=self.client,
             response_model=response_model,
             structured_output_kwargs=self.structured_output_kwargs,
@@ -180,7 +207,7 @@ class Chatterer(BaseModel):
         stop: Optional[list[str]] = None,
         **kwargs: Any,
     ) -> PydanticModelT:
-        result: StructuredOutputType = await with_structured_output(
+        result: StructuredOutputType = await _with_structured_output(
             client=self.client,
             response_model=response_model,
             structured_output_kwargs=self.structured_output_kwargs,
@@ -204,7 +231,7 @@ class Chatterer(BaseModel):
             raise ImportError("Please install `instructor` with `pip install instructor` to use this feature.")
 
         partial_response_model = instructor.Partial[response_model]
-        for chunk in with_structured_output(
+        for chunk in _with_structured_output(
             client=self.client,
             response_model=partial_response_model,
             structured_output_kwargs=self.structured_output_kwargs,
@@ -225,7 +252,7 @@ class Chatterer(BaseModel):
             raise ImportError("Please install `instructor` with `pip install instructor` to use this feature.")
 
         partial_response_model = instructor.Partial[response_model]
-        async for chunk in with_structured_output(
+        async for chunk in _with_structured_output(
             client=self.client,
             response_model=partial_response_model,
             structured_output_kwargs=self.structured_output_kwargs,
@@ -288,8 +315,77 @@ class Chatterer(BaseModel):
         except Exception:
             return None
 
+    def invoke_code_execution(
+        self,
+        messages: LanguageModelInput,
+        repl_tool: Optional["PythonAstREPLTool"] = None,
+        prompt_for_code_invoke: Optional[str] = DEFAULT_CODE_GENERATION_PROMPT,
+        function_signatures: Optional[FunctionSignature | Iterable[FunctionSignature]] = None,
+        function_reference_prefix: Optional[str] = DEFAULT_FUNCTION_REFERENCE_PREFIX_PROMPT,
+        function_reference_seperator: str = DEFAULT_FUNCTION_REFERENCE_SEPARATOR,
+        config: Optional[RunnableConfig] = None,
+        stop: Optional[list[str]] = None,
+        **kwargs: Any,
+    ) -> CodeExecutionResult:
+        if not function_signatures:
+            function_signatures = []
+        elif isinstance(function_signatures, FunctionSignature):
+            function_signatures = [function_signatures]
+        messages = augment_prompt_for_toolcall(
+            function_signatures=function_signatures,
+            messages=messages,
+            prompt_for_code_invoke=prompt_for_code_invoke,
+            function_reference_prefix=function_reference_prefix,
+            function_reference_seperator=function_reference_seperator,
+        )
+        code_obj: PythonCodeToExecute = self.generate_pydantic(
+            response_model=PythonCodeToExecute, messages=messages, config=config, stop=stop, **kwargs
+        )
+        return CodeExecutionResult.from_code(
+            code=code_obj.code,
+            config=config,
+            repl_tool=repl_tool,
+            function_signatures=function_signatures,
+            **kwargs,
+        )
+
+    async def ainvoke_code_execution(
+        self,
+        messages: LanguageModelInput,
+        repl_tool: Optional["PythonAstREPLTool"] = None,
+        prompt_for_code_invoke: Optional[str] = DEFAULT_CODE_GENERATION_PROMPT,
+        additional_callables: Optional[Callable[..., object] | Sequence[Callable[..., object]]] = None,
+        function_reference_prefix: Optional[str] = DEFAULT_FUNCTION_REFERENCE_PREFIX_PROMPT,
+        function_reference_seperator: str = DEFAULT_FUNCTION_REFERENCE_SEPARATOR,
+        config: Optional[RunnableConfig] = None,
+        stop: Optional[list[str]] = None,
+        **kwargs: Any,
+    ) -> CodeExecutionResult:
+        function_signatures: list[FunctionSignature] = FunctionSignature.from_callable(additional_callables)
+        messages = augment_prompt_for_toolcall(
+            function_signatures=function_signatures,
+            messages=messages,
+            prompt_for_code_invoke=prompt_for_code_invoke,
+            function_reference_prefix=function_reference_prefix,
+            function_reference_seperator=function_reference_seperator,
+        )
+        code_obj: PythonCodeToExecute = await self.agenerate_pydantic(
+            response_model=PythonCodeToExecute, messages=messages, config=config, stop=stop, **kwargs
+        )
+        return await CodeExecutionResult.afrom_code(
+            code=code_obj.code,
+            config=config,
+            repl_tool=repl_tool,
+            function_signatures=function_signatures,
+            **kwargs,
+        )
+
+
+class PythonCodeToExecute(BaseModel):
+    code: str = Field(description="Python code to execute")
 
-def with_structured_output(
+
+def _with_structured_output(
     client: BaseChatModel,
     response_model: Type["PydanticModelT | Partial[PydanticModelT]"],
     structured_output_kwargs: dict[str, Any],
@@ -297,75 +393,188 @@ def with_structured_output(
     return client.with_structured_output(schema=response_model, **structured_output_kwargs)  # pyright: ignore[reportUnknownVariableType, reportUnknownMemberType]
 
 
-if __name__ == "__main__":
-    import asyncio
-
-    # 테스트용 Pydantic 모델 정의
-    class Propositions(BaseModel):
-        proposition_topic: str
-        proposition_content: str
-
-    chatterer = Chatterer.openai()
-    prompt = "What is the meaning of life?"
-
-    # === Synchronous Tests ===
-
-    # generate
-    print("=== Synchronous generate ===")
-    result_sync = chatterer(prompt)
-    print("Result (generate):", result_sync)
-
-    # generate_stream
-    print("\n=== Synchronous generate_stream ===")
-    for i, chunk in enumerate(chatterer.generate_stream(prompt)):
-        print(f"Chunk {i}:", chunk)
-
-    # generate_pydantic
-    print("\n=== Synchronous generate_pydantic ===")
-    result_pydantic = chatterer(prompt, Propositions)
-    print("Result (generate_pydantic):", result_pydantic)
-
-    # generate_pydantic_stream
-    print("\n=== Synchronous generate_pydantic_stream ===")
-    for i, chunk in enumerate(chatterer.generate_pydantic_stream(Propositions, prompt)):
-        print(f"Pydantic Chunk {i}:", chunk)
-
-    # === Asynchronous Tests ===
-
-    # Async helper function to enumerate async iterator
-    async def async_enumerate(aiter: AsyncIterator[Any], start: int = 0) -> AsyncIterator[tuple[int, Any]]:
-        i = start
-        async for item in aiter:
-            yield i, item
-            i += 1
-
-    async def run_async_tests():
-        # 6. agenerate
-        print("\n=== Asynchronous agenerate ===")
-        result_async = await chatterer.agenerate(prompt)
-        print("Result (agenerate):", result_async)
-
-        # 7. agenerate_stream
-        print("\n=== Asynchronous agenerate_stream ===")
-        async for i, chunk in async_enumerate(chatterer.agenerate_stream(prompt)):
-            print(f"Async Chunk {i}:", chunk)
-
-        # 8. agenerate_pydantic
-        print("\n=== Asynchronous agenerate_pydantic ===")
-        try:
-            result_async_pydantic = await chatterer.agenerate_pydantic(Propositions, prompt)
-            print("Result (agenerate_pydantic):", result_async_pydantic)
-        except Exception as e:
-            print("Error in agenerate_pydantic:", e)
+def _add_message_last(messages: LanguageModelInput, prompt_to_add: str) -> LanguageModelInput:
+    if isinstance(messages, str):
+        messages += f"\n{prompt_to_add}"
+    elif isinstance(messages, Sequence):
+        messages = list(messages)
+        messages.append(SystemMessage(content=prompt_to_add))
+    else:
+        messages = messages.to_messages()
+        messages.append(SystemMessage(content=prompt_to_add))
+    return messages
+
+
+# def _add_message_first(messages: LanguageModelInput, prompt_to_add: str) -> LanguageModelInput:
+#     if isinstance(messages, str):
+#         messages = f"{prompt_to_add}\n{messages}"
+#     elif isinstance(messages, Sequence):
+#         messages = list(messages)
+#         messages.insert(0, SystemMessage(content=prompt_to_add))
+#     else:
+#         messages = messages.to_messages()
+#         messages.insert(0, SystemMessage(content=prompt_to_add))
+#     return messages
+
+
+def augment_prompt_for_toolcall(
+    function_signatures: Iterable[FunctionSignature],
+    messages: LanguageModelInput,
+    prompt_for_code_invoke: Optional[str] = DEFAULT_CODE_GENERATION_PROMPT,
+    function_reference_prefix: Optional[str] = DEFAULT_FUNCTION_REFERENCE_PREFIX_PROMPT,
+    function_reference_seperator: str = DEFAULT_FUNCTION_REFERENCE_SEPARATOR,
+) -> LanguageModelInput:
+    if function_signatures:
+        messages = _add_message_last(
+            messages=messages,
+            prompt_to_add=FunctionSignature.as_prompt(
+                function_signatures, function_reference_prefix, function_reference_seperator
+            ),
+        )
+    if prompt_for_code_invoke:
+        messages = _add_message_last(messages=messages, prompt_to_add=prompt_for_code_invoke)
+    return messages
+
+
+def interactive_shell(
+    chatterer: Chatterer = Chatterer.openai(),
+    system_instruction: BaseMessage | Iterable[BaseMessage] = ([
+        SystemMessage("You are an AI that can answer questions and execute Python code."),
+    ]),
+    repl_tool: Optional["PythonAstREPLTool"] = None,
+    prompt_for_code_invoke: Optional[str] = DEFAULT_CODE_GENERATION_PROMPT,
+    additional_callables: Optional[Callable[..., object] | Sequence[Callable[..., object]]] = None,
+    function_reference_prefix: Optional[str] = DEFAULT_FUNCTION_REFERENCE_PREFIX_PROMPT,
+    function_reference_seperator: str = DEFAULT_FUNCTION_REFERENCE_SEPARATOR,
+    config: Optional[RunnableConfig] = None,
+    stop: Optional[list[str]] = None,
+    **kwargs: Any,
+) -> None:
+    # Define the CodeExecutionDecision class using Pydantic
+
+    from rich.console import Console
+    from rich.prompt import Prompt
+
+    class IsCodeExecutionNeeded(BaseModel):
+        is_code_execution_needed: bool = Field(
+            description="Whether Python tool calling is needed to answer user query."
+        )
 
-        # 9. agenerate_pydantic_stream
-        print("\n=== Asynchronous agenerate_pydantic_stream ===")
-        try:
-            i = 0
-            async for chunk in chatterer.agenerate_pydantic_stream(Propositions, prompt):
-                print(f"Async Pydantic Chunk {i}:", chunk)
-                i += 1
-        except Exception as e:
-            print("Error in agenerate_pydantic_stream:", e)
-
-    asyncio.run(run_async_tests())
+    class IsFurtherCodeExecutionNeeded(BaseModel):
+        review_on_code_execution: str = Field(description="Review on the code execution.")
+        next_action: str = Field(description="Next action to take.")
+        is_further_code_execution_needed: bool = Field(
+            description="Whether further Python tool calling is needed to answer user query."
+        )
+
+    # Get default REPL tool if not provided.
+    # This tool namespace is persistent across multiple code executions.
+    if repl_tool is None:
+        repl_tool = get_default_repl_tool()
+
+    function_signatures: list[FunctionSignature] = FunctionSignature.from_callable(additional_callables)
+
+    # Initialize Rich console
+    console = Console()
+
+    # Initialize conversation context
+    context: list[BaseMessage] = []
+    if system_instruction:
+        if isinstance(system_instruction, BaseMessage):
+            context.append(system_instruction)
+        else:
+            context.extend(system_instruction)
+
+    # Display welcome message
+    console.print("[bold blue]Welcome to the Interactive Chatterer Shell![/bold blue]")
+    console.print("Type 'quit' or 'exit' to end the conversation.")
+
+    while True:
+        # Get user input
+        user_input = Prompt.ask("[bold green]You[/bold green]")
+        if user_input.lower() in ["quit", "exit"]:
+            console.print("[bold blue]Goodbye![/bold blue]")
+            break
+
+        # Add user message to context
+        context.append(HumanMessage(content=user_input))
+
+        # Determine if code execution is needed
+        decision = chatterer.generate_pydantic(
+            response_model=IsCodeExecutionNeeded,  # Use response_model instead of pydantic_model
+            messages=augment_prompt_for_toolcall(
+                function_signatures=function_signatures,
+                messages=context,
+                prompt_for_code_invoke=prompt_for_code_invoke,
+                function_reference_prefix=function_reference_prefix,
+                function_reference_seperator=function_reference_seperator,
+            ),
+        )
+
+        if decision.is_code_execution_needed:
+            # Execute code if needed
+            code_result = chatterer.invoke_code_execution(
+                messages=context,
+                repl_tool=repl_tool,
+                prompt_for_code_invoke=prompt_for_code_invoke,
+                function_signatures=function_signatures,
+                function_reference_prefix=function_reference_prefix,
+                function_reference_seperator=function_reference_seperator,
+                config=config,
+                stop=stop,
+                **kwargs,
+            )
+
+            if code_result.code.strip() == "pass":
+                tool_use_message = None
+            else:
+                code_session_messages: list[BaseMessage] = []
+                while True:
+                    code_execution_message = SystemMessage(
+                        content=f"Executed code:\n```python\n{code_result.code}\n```\nOutput:\n{code_result.output}"
+                    )
+                    code_session_messages.append(code_execution_message)
+                    console.print("[bold yellow]Executed code:[/bold yellow]")
+                    console.print(f"[code]{code_result.code}[/code]")
+                    console.print("[bold yellow]Output:[/bold yellow]")
+                    console.print(code_result.output)
+
+                    decision = chatterer.generate_pydantic(
+                        response_model=IsFurtherCodeExecutionNeeded,  # Use response_model instead of pydantic_model
+                        messages=augment_prompt_for_toolcall(
+                            function_signatures=function_signatures,
+                            messages=context + code_session_messages,
+                            prompt_for_code_invoke=prompt_for_code_invoke,
+                            function_reference_prefix=function_reference_prefix,
+                            function_reference_seperator=function_reference_seperator,
+                        ),
+                    )
+                    review_on_code_execution = decision.review_on_code_execution
+                    next_action = decision.next_action
+                    console.print("[bold blue]AI:[/bold blue]")
+                    console.print(f"-[bold yellow]Review on code execution:[/bold yellow] {review_on_code_execution}")
+                    console.print(f"-[bold yellow]Next Action:[/bold yellow] {next_action}")
+                    code_session_messages.append(
+                        AIMessage(
+                            content=f"- Review upon code execution: {decision.review_on_code_execution}\n- Next Action: {decision.next_action}"
+                        )
+                    )
+                    if not decision.is_further_code_execution_needed:
+                        tool_use_message = code_execution_message
+                        break
+        else:
+            # No code execution required
+            tool_use_message = None
+
+        # Add system message to context
+        if tool_use_message:
+            context.append(tool_use_message)
+
+        # Generate and display chatbot response
+        response = chatterer.generate(messages=context)  # Use generate instead of generate_response
+        context.append(AIMessage(content=response))
+        console.print(f"[bold blue]AI:[/bold blue] {response}")
+
+
+if __name__ == "__main__":
+    interactive_shell()

chatterer/messages.py

@@ -1,8 +1,9 @@
-from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, SystemMessage
+from langchain_core.messages import AIMessage, BaseMessage, FunctionMessage, HumanMessage, SystemMessage
 
 __all__ = [
     "AIMessage",
     "BaseMessage",
     "HumanMessage",
     "SystemMessage",
+    "FunctionMessage",
 ]

chatterer/tools/__init__.py

@@ -6,6 +6,14 @@ from .convert_to_text import (
     pdf_to_text,
     pyscripts_to_snippets,
 )
+from .youtube import get_youtube_video_subtitle, get_youtube_video_details
+
+
+def init_webpage_to_markdown():
+    from . import webpage_to_markdown
+
+    return webpage_to_markdown
+
 
 __all__ = [
     "html_to_markdown",
@@ -14,4 +22,7 @@ __all__ = [
     "get_default_html_to_markdown_options",
     "pyscripts_to_snippets",
     "citation_chunker",
+    "init_webpage_to_markdown",
+    "get_youtube_video_subtitle",
+    "get_youtube_video_details",
 ]

chatterer/tools/youtube.py

@@ -0,0 +1,132 @@
+import json
+import unicodedata
+import urllib.parse
+from dataclasses import dataclass
+from typing import Any, Optional, Self, cast
+
+import requests
+
+
+def get_youtube_video_details(
+    query: str,
+) -> list[dict[str, Optional[str]]]:
+    """Search for video metadata on YouTube using the given query. Returns a list of dictionaries containing `video_id`, `title`, `channel`, `duration`, `views`, `publish_time`, and `long_desc`."""
+    return [
+        {
+            "video_id": video_id,
+            "title": video.title,
+            "channel": video.channel,
+            "duration": video.duration,
+            "views": video.views,
+            "publish_time": video.publish_time,
+            "long_desc": video.long_desc,
+        }
+        for video in YoutubeSearchResult.from_query(base_url="https://youtube.com", query=query, max_results=10)
+        if (video_id := _get_video_id(video.url_suffix))
+    ]
+
+
+def get_youtube_video_subtitle(video_id: str) -> str:
+    """Get the transcript of a YouTube video using the given video ID."""
+
+    from youtube_transcript_api._api import YouTubeTranscriptApi
+
+    get_transcript = YouTubeTranscriptApi.get_transcript  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]
+    list_transcripts = YouTubeTranscriptApi.list_transcripts  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]
+
+    result: str = ""
+    buffer_timestamp: str = "0s"
+    buffer_texts: list[str] = []
+    for entry in get_transcript(video_id, languages=(next(iter(list_transcripts(video_id))).language_code,)):  # pyright: ignore[reportUnknownVariableType]
+        entry = cast(dict[object, object], entry)
+        text: str = str(entry.get("text", "")).strip().replace("\n", " ")
+        if not text:
+            continue
+        if len(buffer_texts) >= 10 or _is_special_char(text) or (buffer_texts and _is_special_char(buffer_texts[-1])):
+            result += f"[{buffer_timestamp}] {'. '.join(buffer_texts)}\n"
+            start = entry.get("start", 0)
+            if start:
+                buffer_timestamp = f"{start:.0f}s"
+            buffer_texts = [text]
+        else:
+            buffer_texts.append(text)
+
+    if buffer_texts:
+        result += f"[{buffer_timestamp}] {' '.join(buffer_texts)}"
+    return result
+
+
+def _get_video_id(suffix: str) -> str:
+    return next(iter(urllib.parse.parse_qs(urllib.parse.urlparse(suffix).query)["v"]), "")
+
+
+def _is_special_char(text: str) -> bool:
+    if not text:
+        return False
+    return not unicodedata.category(text[0]).startswith("L")
+
+
+@dataclass
+class YoutubeSearchResult:
+    url_suffix: str
+    id: Optional[str]
+    thumbnails: list[str]
+    title: Optional[str]
+    long_desc: Optional[str]
+    channel: Optional[str]
+    duration: Optional[str]
+    views: Optional[str]
+    publish_time: Optional[str]
+
+    @classmethod
+    def from_query(cls, base_url: str, query: str, max_results: int) -> list[Self]:
+        url: str = f"{base_url}/results?search_query={urllib.parse.quote_plus(query)}"
+        response: str = requests.get(url).text
+        while "ytInitialData" not in response:
+            response = requests.get(url).text
+        results: list[Self] = cls.parse_html(response)
+        return results[:max_results]
+
+    @classmethod
+    def parse_html(cls, html: str) -> list[Self]:
+        results: list[Self] = []
+        start: int = html.index("ytInitialData") + len("ytInitialData") + 3
+        end: int = html.index("};", start) + 1
+        data: Any = json.loads(html[start:end])
+        for contents in data["contents"]["twoColumnSearchResultsRenderer"]["primaryContents"]["sectionListRenderer"][
+            "contents"
+        ]:
+            for video in contents["itemSectionRenderer"]["contents"]:
+                if "videoRenderer" in video.keys():
+                    video_data = video.get("videoRenderer", {})
+                    suffix = (
+                        video_data.get("navigationEndpoint", {})
+                        .get("commandMetadata", {})
+                        .get("webCommandMetadata", {})
+                        .get("url", None)
+                    )
+                    if not suffix:
+                        continue
+                    res = cls(
+                        id=video_data.get("videoId", None),
+                        thumbnails=[
+                            thumb.get("url", None) for thumb in video_data.get("thumbnail", {}).get("thumbnails", [{}])
+                        ],
+                        title=video_data.get("title", {}).get("runs", [[{}]])[0].get("text", None),
+                        long_desc=video_data.get("descriptionSnippet", {}).get("runs", [{}])[0].get("text", None),
+                        channel=video_data.get("longBylineText", {}).get("runs", [[{}]])[0].get("text", None),
+                        duration=video_data.get("lengthText", {}).get("simpleText", 0),
+                        views=video_data.get("viewCountText", {}).get("simpleText", 0),
+                        publish_time=video_data.get("publishedTimeText", {}).get("simpleText", 0),
+                        url_suffix=suffix,
+                    )
+                    results.append(res)
+
+            if results:
+                break
+        return results
+
+
+if __name__ == "__main__":
+    print(get_youtube_video_details("BTS"))
+    # print(get_youtube_transcript("y7jrpS8GHxs"))

chatterer/utils/__init__.py

@@ -0,0 +1,15 @@
+from .code_agent import (
+    CodeExecutionResult,
+    FunctionSignature,
+    get_default_repl_tool,
+    insert_callables_into_global,
+)
+from .image import Base64Image
+
+__all__ = [
+    "Base64Image",
+    "FunctionSignature",
+    "CodeExecutionResult",
+    "get_default_repl_tool",
+    "insert_callables_into_global",
+]

chatterer/utils/code_agent.py

@@ -0,0 +1,138 @@
+import inspect
+import textwrap
+from typing import (
+    TYPE_CHECKING,
+    Callable,
+    Iterable,
+    NamedTuple,
+    Optional,
+    Self,
+)
+
+from langchain_core.runnables.config import RunnableConfig
+
+if TYPE_CHECKING:
+    from langchain_experimental.tools import PythonAstREPLTool
+
+
+class FunctionSignature(NamedTuple):
+    name: str
+    callable: Callable[..., object]
+    signature: str
+
+    @classmethod
+    def from_callable(cls, callables: Optional[Callable[..., object] | Iterable[Callable[..., object]]]) -> list[Self]:
+        if callables is None:
+            return []
+        if callable(callables):
+            return [cls._from_callable(callables)]
+        return [cls._from_callable(callable) for callable in callables]
+
+    @classmethod
+    def _from_callable(cls, callable: Callable[..., object]) -> Self:
+        """
+        Get the name and signature of a function as a string.
+        """
+        # Determine if the function is async
+        is_async_func = inspect.iscoroutinefunction(callable)
+        function_def = "async def" if is_async_func else "def"
+
+        # Determine the function name based on the type of callable
+        if inspect.isfunction(callable):
+            # For regular Python functions, use __code__.co_name
+            function_name = callable.__code__.co_name
+        elif hasattr(callable, "name"):
+            # For StructuredTool or similar objects with a 'name' attribute
+            function_name = callable.name  # type: ignore
+        elif hasattr(callable, "__name__"):
+            # For other callables with a __name__ attribute
+            function_name = callable.__name__
+        else:
+            # Fallback to the class name if no name is found
+            function_name = type(callable).__name__
+
+        # Build the signature string
+        signature = f"{function_def} {function_name}{inspect.signature(callable)}:"
+        docstring = inspect.getdoc(callable)
+        if docstring:
+            docstring = f'"""{docstring.strip()}"""'
+            return cls(
+                name=function_name, callable=callable, signature=f"{signature}\n{textwrap.indent(docstring, '    ')}"
+            )
+        else:
+            return cls(name=function_name, callable=callable, signature=signature)
+
+    @classmethod
+    def as_prompt(
+        cls,
+        function_signatures: Iterable[Self],
+        prefix: Optional[str] = "You can use the pre-made functions below without defining them:\n",
+        sep: str = "\n---\n",
+    ) -> str:
+        """
+        Generate a prompt string from a list of callables.
+        """
+        body: str = sep.join(fsig.signature for fsig in function_signatures)
+        if prefix:
+            return f"{prefix}{body}"
+        return body
+
+
+class CodeExecutionResult(NamedTuple):
+    code: str
+    output: str
+
+    @classmethod
+    def from_code(
+        cls,
+        code: str,
+        repl_tool: Optional["PythonAstREPLTool"] = None,
+        config: Optional[RunnableConfig] = None,
+        function_signatures: Optional[Iterable[FunctionSignature]] = None,
+        **kwargs: object,
+    ) -> Self:
+        """
+        Execute code using the Python Code Execution Language Model.
+        """
+        if repl_tool is None:
+            repl_tool = get_default_repl_tool()
+        if function_signatures:
+            insert_callables_into_global(function_signatures=function_signatures, repl_tool=repl_tool)
+        output = str(repl_tool.invoke(code, config=config, **kwargs))  # pyright: ignore[reportUnknownMemberType]
+        return cls(code=code, output=output)
+
+    @classmethod
+    async def afrom_code(
+        cls,
+        code: str,
+        repl_tool: Optional["PythonAstREPLTool"] = None,
+        config: Optional[RunnableConfig] = None,
+        function_signatures: Optional[Iterable[FunctionSignature]] = None,
+        **kwargs: object,
+    ) -> Self:
+        """
+        Execute code using the Python Code Execution Language Model asynchronously.
+        """
+        if repl_tool is None:
+            repl_tool = get_default_repl_tool()
+        if function_signatures:
+            insert_callables_into_global(function_signatures=function_signatures, repl_tool=repl_tool)
+        output = str(await repl_tool.ainvoke(code, config=config, **kwargs))  # pyright: ignore[reportUnknownMemberType]
+        return cls(code=code, output=output)
+
+
+def get_default_repl_tool() -> "PythonAstREPLTool":
+    from langchain_experimental.tools import PythonAstREPLTool
+
+    return PythonAstREPLTool()
+
+
+def insert_callables_into_global(
+    function_signatures: Iterable[FunctionSignature], repl_tool: "PythonAstREPLTool"
+) -> None:
+    """Insert callables into the REPL tool's globals."""
+    repl_globals: Optional[dict[str, object]] = repl_tool.globals  # pyright: ignore[reportUnknownMemberType]
+    if repl_globals is None:
+        repl_tool.globals = {fsig.name: fsig.callable for fsig in function_signatures}
+    else:
+        repl_globals.update({fsig.name: fsig.callable for fsig in function_signatures})

{chatterer-0.1.8.dist-info → chatterer-0.1.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: chatterer
-Version: 0.1.8
+Version: 0.1.10
 Summary: The highest-level interface for various LLM APIs.
 Requires-Python: >=3.12
 Description-Content-Type: text/markdown
@@ -18,13 +18,17 @@ Requires-Dist: pillow>=11.1.0; extra == "conversion"
 Requires-Dist: mistune>=3.1.2; extra == "conversion"
 Requires-Dist: markitdown>=0.0.2; extra == "conversion"
 Requires-Dist: pymupdf>=1.25.4; extra == "conversion"
+Requires-Dist: youtube-transcript-api>=1.0.2; extra == "conversion"
+Provides-Extra: langchain
+Requires-Dist: chatterer[langchain-providers]; extra == "langchain"
+Requires-Dist: langchain-experimental>=0.3.4; extra == "langchain"
 Provides-Extra: langchain-providers
 Requires-Dist: langchain-openai>=0.3.7; extra == "langchain-providers"
 Requires-Dist: langchain-anthropic>=0.3.8; extra == "langchain-providers"
 Requires-Dist: langchain-google-genai>=2.0.10; extra == "langchain-providers"
 Requires-Dist: langchain-ollama>=0.2.3; extra == "langchain-providers"
 Provides-Extra: all
-Requires-Dist: chatterer[langchain-providers]; extra == "all"
+Requires-Dist: chatterer[langchain]; extra == "all"
 Requires-Dist: chatterer[conversion]; extra == "all"
 Requires-Dist: chatterer[dev]; extra == "all"
 

{chatterer-0.1.8.dist-info → chatterer-0.1.10.dist-info}/RECORD

@@ -1,12 +1,13 @@
-chatterer/__init__.py,sha256=kl8VWiDJIt5IQjaBpQu13n0GrzP3qzaNXyA68B1xHTE,802
-chatterer/language_model.py,sha256=S8x2IbzZBi1mAKSKrGuoB4-gfKBz73RCNXt_H-fiDzc,13826
-chatterer/messages.py,sha256=-NyOIK7wJI1uVD8qaJPeLA0LqirFEsZ1mOYoO1F2wLc,188
+chatterer/__init__.py,sha256=BPgCQ6VWGBXSh8xJr_0bpM0hcOOUz0KoxcKxOd9GYyI,1388
+chatterer/language_model.py,sha256=qnVC5_W4IYM0y0o1PTYMGXUlblRv5fsRk0zIiL_vT3Q,24491
+chatterer/messages.py,sha256=OtbZ3two0LUQ4PXES97FDIBUSO3IcMHdFV1VFkDL2mI,229
 chatterer/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 chatterer/strategies/__init__.py,sha256=SdOggbmHpw4f7Njwy-T8q64e91OLOUp1k0a0ozZd4qI,221
 chatterer/strategies/atom_of_thoughts.py,sha256=CygOCLu5vLk-fzY9O-iE3qLShfjD7iY40ks9jH4ULBM,40872
 chatterer/strategies/base.py,sha256=b2gMPqodp97OP1dkHfj0UqixjdjVhmTw_V5qJ7i2S6g,427
-chatterer/tools/__init__.py,sha256=yA4RcHIAO33xsmWXQTmtSm9bk1p80yJKSadtMa3X-aY,415
+chatterer/tools/__init__.py,sha256=hmWIuLJWotGQodL__i4LLbHdXe7Nl5uKHqNke9tHMro,705
 chatterer/tools/convert_to_text.py,sha256=kBqxCJ0IoiAw2eiPYqep_SPZm-TtYKF7mdACLsWQUuI,15915
+chatterer/tools/youtube.py,sha256=5hwASZWA92d_7Y5RqlCK2tULRBQx8bGnE_0NCnvaKi0,5499
 chatterer/tools/citation_chunking/__init__.py,sha256=gG7Fnkkp28UpcWMbfMY_4gqzZSZ8QzlhalHBoeoq7K0,82
 chatterer/tools/citation_chunking/chunks.py,sha256=50Dpa43RaYftlNox8tM1qI8htZ3_AJ9Uyyn02WsmxYk,2173
 chatterer/tools/citation_chunking/citation_chunker.py,sha256=yx5O9pUkowlNcFyyNf7f3sbq7-CV8AXOzFnviDldPR8,4894
@@ -17,8 +18,10 @@ chatterer/tools/citation_chunking/utils.py,sha256=M4pH2-UIE1VLzQLXDqjEe4L3Xcy0e0
 chatterer/tools/webpage_to_markdown/__init__.py,sha256=bHH4qfnXyw8Zz-yBPLaTezF1sh9njvNBJmhBVtcpjsA,123
 chatterer/tools/webpage_to_markdown/playwright_bot.py,sha256=yP0KixYZNQ4Kn_ZCFDI3mVyBD_DpUGfqgklpaGJUTCU,27496
 chatterer/tools/webpage_to_markdown/utils.py,sha256=ZLUU94imYciEdynD2K7Dmcsbt8BVQTaOP56Ba6DAFvk,12593
+chatterer/utils/__init__.py,sha256=8nzpFJKU_wSRPH6LBP6HRBotPMrSl_VO9UlmFprTrK0,334
+chatterer/utils/code_agent.py,sha256=UaWdeGzJMPzRSFy9yrxuveBJsvOPSa0te6OuE18bees,5143
 chatterer/utils/image.py,sha256=F3_D1677UDFlgp-UQBS_ChkNODzf_VOfjYNSUi02MaI,10852
-chatterer-0.1.8.dist-info/METADATA,sha256=01CGNp0oae5VdHM5gzqPKYFtlSqufE0h5XFMdn2E_6c,4234
-chatterer-0.1.8.dist-info/WHEEL,sha256=1tXe9gY0PYatrMPMDd6jXqjfpz_B-Wqm32CPfRC58XU,91
-chatterer-0.1.8.dist-info/top_level.txt,sha256=7nSQKP0bHxPRc7HyzdbKsJdkvPgYD0214o6slRizv9s,10
-chatterer-0.1.8.dist-info/RECORD,,
+chatterer-0.1.10.dist-info/METADATA,sha256=qPx7b41yUvBG0XFH4ra89LIyOSUZUo_8gZ-adVkTKME,4458
+chatterer-0.1.10.dist-info/WHEEL,sha256=1tXe9gY0PYatrMPMDd6jXqjfpz_B-Wqm32CPfRC58XU,91
+chatterer-0.1.10.dist-info/top_level.txt,sha256=7nSQKP0bHxPRc7HyzdbKsJdkvPgYD0214o6slRizv9s,10
+chatterer-0.1.10.dist-info/RECORD,,