langroid 0.16.7__py3-none-any.whl → 0.17.0__py3-none-any.whl

langroid/agent/xml_tool_message.py

@@ -1,13 +1,24 @@
-from typing import get_type_hints
+from typing import Any, Dict, List, Optional
 
 from lxml import etree
 
 from langroid.agent.tool_message import ToolMessage
 
 
-class XmlToolMessage(ToolMessage):
+class XMLToolMessage(ToolMessage):
     """
     Abstract class for tools formatted using XML instead of JSON.
+    Mainly tested for non-nested tool structures.
+
+    When a subclass defines a field with the attribute `verbatim=True`,
+    its value will be:
+        - preserved as is, including whitespace, indents, quotes, newlines, etc
+            with no escaping, and
+        - enclosed in a CDATA section in the XML output.
+    This is useful for LLMs sending code as part of a tool;
+    results can be far superior compared to sending code in JSON-formatted tools,
+    where code needs to confirm to JSON's strict rules and escaping requirements.
+    (see test_xml_tool_message.py for an example).
     """
 
     request: str
@@ -19,72 +30,191 @@ class XmlToolMessage(ToolMessage):
         root_element = "tool"
 
     @classmethod
-    def parse(cls, formatted_string: str) -> "XmlToolMessage":
+    def extract_field_values(cls, formatted_string: str) -> Optional[Dict[str, Any]]:
+        """
+        Extracts field values from an XML-formatted string.
+
+        Args:
+            formatted_string (str): The XML-formatted string to parse.
+
+        Returns:
+            Optional[Dict[str, Any]]: A dictionary containing the extracted field
+                values, where keys are the XML element names and values are their
+                corresponding contents.
+            Returns None if parsing fails or the root element is not a dictionary.
+
+        Raises:
+            etree.XMLSyntaxError: If the input string is not valid XML.
+        """
         parser = etree.XMLParser(strip_cdata=False)
         root = etree.fromstring(formatted_string.encode("utf-8"), parser=parser)
-        if root.tag != cls.Config.root_element:
-            raise ValueError(
-                f"Invalid root element: expected {cls.Config.root_element}, got {root.tag}"
+
+        def parse_element(element: etree._Element) -> Any | Dict[str, Any] | str:
+            # Skip elements starting with underscore
+            field_info = cls.__fields__.get(element.tag)
+            is_verbatim = field_info and field_info.field_info.extra.get(
+                "verbatim", False
             )
 
-        data = {}
-        type_hints = get_type_hints(cls)
-        exclude_fields = cls.Config.schema_extra.get("exclude", set())
-        for elem in root:
-            if elem.tag not in exclude_fields:
-                field_type = type_hints.get(elem.tag, str)
-                if elem.tag == "code":
-                    data[elem.tag] = elem.text if elem.text else ""
-                else:
-                    value = elem.text.strip() if elem.text else ""
-                    if field_type == int:
-                        data[elem.tag] = int(value)
-                    elif field_type == float:
-                        data[elem.tag] = float(value)
-                    elif field_type == bool:
-                        data[elem.tag] = value.lower() in ("true", "1", "yes")
-                    else:
-                        data[elem.tag] = value
-
-        return cls(**data)
+            if is_verbatim:
+                # For code elements, preserve the content as is, including whitespace
+                content = element.text if element.text else ""
+                # Strip leading and trailing triple backticks if present,
+                # accounting for whitespace
+                return (
+                    content.strip().removeprefix("```").removesuffix("```").strip()
+                    if content.strip().startswith("```")
+                    and content.strip().endswith("```")
+                    else content
+                )
+            elif len(element) == 0:
+                # For non-code leaf elements, strip whitespace
+                return element.text.strip() if element.text else ""
+            else:
+                # For branch elements, recurse
+                return {child.tag: parse_element(child) for child in element}
+
+        result = parse_element(root)
+        if not isinstance(result, dict):
+            return None
+        # Filter out empty dictionaries from skipped underscore fields
+        return {k: v for k, v in result.items() if v != {}}
+
+    @classmethod
+    def parse(cls, formatted_string: str) -> Optional["XMLToolMessage"]:
+        """
+        Parses the XML-formatted string and returns an instance of the class.
+
+        Args:
+            formatted_string (str): The XML-formatted string to parse.
+
+        Returns:
+            Optional["XMLToolMessage"]: An instance of the class if parsing succeeds,
+                None otherwise.
+        """
+        parsed_data = cls.extract_field_values(formatted_string)
+        if parsed_data is None:
+            return None
+
+        # Use Pydantic's parse_obj to create and validate the instance
+        return cls.parse_obj(parsed_data)
 
     @classmethod
-    def instructions(cls) -> str:
+    def format_instructions(cls, tool: bool = False) -> str:
+        """
+        Instructions to the LLM showing how to use the XML tool.
+
+        Args:
+            tool: Not used in this implementation, kept for compatibility.
+
+        Returns:
+            str: instructions on how to use the XML message
+        """
         fields = [
             f
             for f in cls.__fields__.keys()
             if f not in cls.Config.schema_extra.get("exclude", set())
         ]
 
+        instructions = """
+        To use this tool, please provide the required information in an XML-like 
+        format. Here's how to structure your input:\n\n
+        """
+
         preamble = "Placeholders:\n"
         for field in fields:
             preamble += f"{field.upper()} = [value for {field}]\n"
 
-        example = f"Formatting example:\n\n<{cls.Config.root_element}>\n"
+        verbatim_fields = [
+            field
+            for field, field_info in cls.__fields__.items()
+            if field_info.field_info.extra.get("verbatim", False)
+        ]
+
+        verbatim_alert = ""
+        if len(verbatim_fields) > 0:
+            verbatim_alert = f"""
+            EXTREMELY IMPORTANT: For these fields:
+            {', '.join(verbatim_fields)},
+            the contents MUST be wrapped in a CDATA section, and the content
+            must be written verbatim WITHOUT any modifications or escaping,
+            such as spaces, tabs, indents, newlines, quotes, etc.
+            """
+        xml_format = f"Formatting example:\n\n<{cls.Config.root_element}>\n"
         for field in fields:
             if field == "code":
-                example += f"  <{field}><![CDATA[{{{field.upper()}}}]]></{field}>\n"
+                xml_format += f"  <{field}><![CDATA[{{{field.upper()}}}]]></{field}>\n"
             else:
-                example += f"  <{field}>{{{field.upper()}}}</{field}>\n"
-        example += f"</{cls.Config.root_element}>"
-
-        return f"{preamble}\n{example}"
-
-    @classmethod
-    def format(cls, instance: "XmlToolMessage") -> str:
-        root = etree.Element(cls.Config.root_element)
-        exclude_fields = cls.Config.schema_extra.get("exclude", set())
-        for name, value in instance.dict().items():
+                xml_format += f"  <{field}>{{{field.upper()}}}</{field}>\n"
+        xml_format += f"</{cls.Config.root_element}>"
+
+        examples_str = ""
+        if cls.examples():
+            examples_str = "EXAMPLES:\n" + cls.usage_examples()
+
+        return f"""
+            TOOL: {cls.default_value("request")}
+            PURPOSE: {cls.default_value("purpose")} 
+
+            {instructions}
+            {preamble}
+            {xml_format}
+
+            Make sure to replace the placeholders with actual values 
+            when using the tool.                
+            {verbatim_alert}            
+            {examples_str}
+            """.lstrip()
+
+    def format_example(self) -> str:
+        """
+        Format the current instance as an XML example.
+
+        Returns:
+            str: A string representation of the current instance in XML format.
+
+        Raises:
+            ValueError: If the result from etree.tostring is not a string.
+        """
+        root = etree.Element(self.Config.root_element)
+        exclude_fields = self.Config.schema_extra.get("exclude", set())
+        for name, value in self.dict().items():
             if name not in exclude_fields:
                 elem = etree.SubElement(root, name)
-                if name == "code":
+                field_info = self.__class__.__fields__[name]
+                is_verbatim = field_info.field_info.extra.get("verbatim", False)
+                if is_verbatim:
                     elem.text = etree.CDATA(str(value))
                 else:
                     elem.text = str(value)
-        return etree.tostring(root, encoding="unicode", pretty_print=True)
+        result = etree.tostring(root, encoding="unicode", pretty_print=True)
+        if not isinstance(result, str):
+            raise ValueError("Unexpected non-string result from etree.tostring")
+        return result
 
     @classmethod
-    def find_candidates(cls, text: str) -> str:
+    def find_candidates(cls, text: str) -> List[str]:
+        """
+        Find and extract all potential XML tool messages from the given text.
+
+        This method searches for XML-like structures in the input text that match
+        the expected format of the tool message. It looks for opening and closing
+        tags that correspond to the root element defined in the XMLToolMessage class,
+        which is by default <tool>.
+
+        Args:
+            text (str): The input text to search for XML tool messages.
+
+        Returns:
+            List[str]: A list of strings, each representing a potential XML tool
+                       message.
+                       These candidates include both the opening and
+                       closing tags, so that they are individually parseable.
+
+        Note:
+            This method ensures that all candidates are valid and parseable by
+            inserting a closing tag if it's missing for the last candidate.
+        """
         root_tag = cls.Config.root_element
         opening_tag = f"<{root_tag}>"
         closing_tag = f"</{root_tag}>"
@@ -97,10 +227,13 @@ class XmlToolMessage(ToolMessage):
                 break
             end = text.find(closing_tag, start)
             if end == -1:
-                # For the last candidate, allow missing closing tag
-                candidates.append(text[start:])
+                # For the last candidate, insert the closing tag if it's missing
+                candidate = text[start:]
+                if not candidate.strip().endswith(closing_tag):
+                    candidate += closing_tag
+                candidates.append(candidate)
                 break
             candidates.append(text[start : end + len(closing_tag)])
             start = end + len(closing_tag)
 
-        return "\n".join(candidates)
+        return candidates

langroid/utils/constants.py

@@ -26,3 +26,5 @@ TOOL = "TOOL"
 # it no letters, digits or underscores.
 # See tests/main/test_msg_routing for example usage
 AT = "|@|"
+TOOL_BEGIN = "TOOL_BEGIN"
+TOOL_END = "TOOL_END"

langroid/utils/git_utils.py

@@ -0,0 +1,251 @@
+import fnmatch
+import logging
+import textwrap
+from pathlib import Path
+from typing import List
+
+import git
+from github import Github, GithubException
+
+from langroid.utils.system import create_file
+
+logger = logging.getLogger(__name__)
+
+
+def git_read_file(repo: str, filepath: str) -> str:
+    """
+    Read the contents of a file from a GitHub repository.
+
+    Args:
+        repo (str): The GitHub repository in the format "owner/repo"
+        filepath (str): The file path relative to the repository root
+
+    Returns:
+        str: The contents of the file as a string
+    """
+    try:
+        g = Github()
+        github_repo = g.get_repo(repo)
+        file_content = github_repo.get_contents(filepath)
+        if isinstance(file_content, list) and len(file_content) > 0:
+            return file_content[0].decoded_content.decode("utf-8")
+        elif hasattr(file_content, "decoded_content"):
+            return file_content.decoded_content.decode("utf-8")
+        else:
+            logger.error(f"Unexpected file_content type: {type(file_content)}")
+            return ""
+    except GithubException as e:
+        logger.error(f"An error occurred while reading file {filepath}: {e}")
+        return ""
+
+
+def get_file_list(repo: str, dir: str, pat: str = "") -> List[str]:
+    """
+    Get a list of files in a specified directory of a GitHub repository.
+
+    Args:
+        repo (str): The GitHub repository in the format "owner/repo"
+        dir (str): The directory path relative to the repository root
+        pat (str): Optional wildcard pattern to filter file names (default: "")
+
+    Returns:
+        List[str]: A list of file paths in the specified directory
+    """
+    try:
+        g = Github()
+        github_repo = g.get_repo(repo)
+        contents = github_repo.get_contents(dir)
+
+        file_list = []
+        if isinstance(contents, list):
+            file_list = [content.path for content in contents if content.type == "file"]
+        elif hasattr(contents, "path") and hasattr(contents, "type"):
+            if contents.type == "file":
+                file_list = [contents.path]
+
+        if pat:
+            file_list = [file for file in file_list if fnmatch.fnmatch(file, pat)]
+        return sorted(file_list)
+
+    except GithubException as e:
+        logger.error(f"An error occurred while fetching file list: {e}")
+        return []
+
+
+def git_init_repo(dir: str) -> git.Repo | None:
+    """
+    Set up a Git repository in the specified directory.
+
+    Args:
+        dir (str): Path to the directory where the Git repository should be initialized
+
+    Returns:
+        git.Repo: The initialized Git repository object
+    """
+    repo_path = Path(dir).expanduser()
+    try:
+        repo = git.Repo.init(repo_path)
+        logger.info(f"Git repository initialized in {repo_path}")
+
+        gitignore_content = textwrap.dedent(
+            """
+        /target/
+        **/*.rs.bk
+        Cargo.lock
+        """
+        ).strip()
+
+        gitignore_path = repo_path / ".gitignore"
+        create_file(gitignore_path, gitignore_content)
+        logger.info(f"Created .gitignore file in {repo_path}")
+
+        # Ensure the default branch is 'main'
+        # Check if we're on the master branch
+        if repo.active_branch.name == "master":
+            # Rename the branch
+            repo.git.branch("-m", "master", "main")
+            print("Branch renamed from 'master' to 'main'")
+        else:
+            print("Current branch is not 'master'. No changes made.")
+        return repo
+    except git.GitCommandError as e:
+        logger.error(f"An error occurred while initializing the repository: {e}")
+        return None
+
+
+def git_commit_file(repo: git.Repo, filepath: str, msg: str) -> None:
+    """
+    Commit a file to a Git repository.
+
+    Args:
+        repo (git.Repo): The Git repository object
+        filepath (str): Path to the file to be committed
+        msg (str): The commit message
+
+    Returns:
+        None
+    """
+    try:
+        repo.index.add([filepath])
+        repo.index.commit(f"{msg}; Updated {filepath}")
+        logger.info(f"Successfully committed {filepath}")
+    except git.GitCommandError as e:
+        logger.error(f"An error occurred while committing: {e}")
+
+
+def git_commit_mods(repo: git.Repo, msg: str = "commit all changes") -> None:
+    """
+    Commit all modifications in the Git repository.
+    Does not raise an error if there's nothing to commit.
+
+    Args:
+        repo (git.Repo): The Git repository object
+
+    Returns:
+        None
+    """
+    try:
+        if repo.is_dirty():
+            repo.git.add(update=True)
+            repo.index.commit(msg)
+            logger.info("Successfully committed all modifications")
+        else:
+            logger.info("No changes to commit")
+    except git.GitCommandError as e:
+        logger.error(f"An error occurred while committing modifications: {e}")
+
+
+def git_restore_repo(repo: git.Repo) -> None:
+    """
+    Restore all unstaged, uncommitted changes in the Git repository.
+    This function undoes any dirty files to the last commit.
+
+    Args:
+        repo (git.Repo): The Git repository object
+
+    Returns:
+        None
+    """
+    try:
+        if repo.is_dirty():
+            repo.git.restore(".")
+            logger.info("Successfully restored all unstaged changes")
+        else:
+            logger.info("No unstaged changes to restore")
+    except git.GitCommandError as e:
+        logger.error(f"An error occurred while restoring changes: {e}")
+
+
+def git_restore_file(repo: git.Repo, file_path: str) -> None:
+    """
+    Restore a specific file in the Git repository to its state in the last commit.
+    This function undoes changes to the specified file.
+
+    Args:
+        repo (git.Repo): The Git repository object
+        file_path (str): Path to the file to be restored
+
+    Returns:
+        None
+    """
+    try:
+        repo.git.restore(file_path)
+        logger.info(f"Successfully restored file: {file_path}")
+    except git.GitCommandError as e:
+        logger.error(f"An error occurred while restoring file {file_path}: {e}")
+
+
+def git_create_checkout_branch(repo: git.Repo, branch: str) -> None:
+    """
+    Create and checkout a new branch in the given Git repository.
+    If the branch already exists, it will be checked out.
+    If we're already on the specified branch, no action is taken.
+
+    Args:
+        repo (git.Repo): The Git repository object
+        branch (str): The name of the branch to create or checkout
+
+    Returns:
+        None
+    """
+    try:
+        if repo.active_branch.name == branch:
+            logger.info(f"Already on branch: {branch}")
+            return
+
+        if branch in repo.heads:
+            repo.heads[branch].checkout()
+            logger.info(f"Checked out existing branch: {branch}")
+        else:
+            new_branch = repo.create_head(branch)
+            new_branch.checkout()
+            logger.info(f"Created and checked out new branch: {branch}")
+    except git.GitCommandError as e:
+        logger.error(f"An error occurred while creating/checking out branch: {e}")
+
+
+def git_diff_file(repo: git.Repo, filepath: str) -> str:
+    """
+    Show diffs of file between the latest commit and the previous one if any.
+
+    Args:
+        repo (git.Repo): The Git repository object
+        filepath (str): Path to the file to be diffed
+
+    Returns:
+        str: The diff output as a string
+    """
+    try:
+        # Get the two most recent commits
+        commits = list(repo.iter_commits(paths=filepath, max_count=2))
+
+        if len(commits) < 2:
+            return "No previous commit found for comparison."
+
+        # Get the diff between the two commits for the specific file
+        diff = repo.git.diff(commits[1].hexsha, commits[0].hexsha, filepath)
+
+        return str(diff)
+    except git.GitCommandError as e:
+        logger.error(f"An error occurred while getting diff: {e}")
+        return f"Error: {str(e)}"

langroid/utils/system.py

@@ -1,3 +1,4 @@
+import difflib
 import getpass
 import hashlib
 import importlib
@@ -8,6 +9,7 @@ import shutil
 import socket
 import traceback
 import uuid
+from pathlib import Path
 from typing import Any
 
 logger = logging.getLogger(__name__)
@@ -182,3 +184,79 @@ def hash(s: str) -> str:
 def generate_unique_id() -> str:
     """Generate a unique ID using UUID4."""
     return str(uuid.uuid4())
+
+
+def create_file(filepath: str | Path, content: str = "") -> None:
+    """
+    Create a file with the given content in the specified directory.
+    If content is empty, it will simply touch to create an empty file.
+
+    Args:
+        filepath (str|Path): The relative path of the file to be created
+        content (str): The content to be written to the file
+    """
+    Path(filepath).parent.mkdir(parents=True, exist_ok=True)
+    if content == "":
+        Path(filepath).touch()
+    else:
+        # the newline = '\n` argument is used to ensure that
+        # newlines in the content are written as actual line breaks
+        with open(filepath, "w", newline="\n") as f:
+            f.write(content)
+    logger.warning(f"File created/updated: {filepath}")
+
+
+def read_file(path: str, line_numbers: bool = False) -> str:
+    """
+    Read the contents of a file.
+
+    Args:
+        path (str): The path to the file to be read.
+        line_numbers (bool, optional): If True, prepend line numbers to each line.
+            Defaults to False.
+
+    Returns:
+        str: The contents of the file, optionally with line numbers.
+
+    Raises:
+        FileNotFoundError: If the specified file does not exist.
+    """
+    # raise an error if the file does not exist
+    if not Path(path).exists():
+        raise FileNotFoundError(f"File not found: {path}")
+    file = Path(path).expanduser()
+    content = file.read_text()
+    if line_numbers:
+        lines = content.splitlines()
+        numbered_lines = [f"{i+1}: {line}" for i, line in enumerate(lines)]
+        return "\n".join(numbered_lines)
+    return content
+
+
+def diff_files(file1: str, file2: str) -> str:
+    """
+    Find the diffs between two files, in unified diff format.
+    """
+    with open(file1, "r") as f1, open(file2, "r") as f2:
+        lines1 = f1.readlines()
+        lines2 = f2.readlines()
+
+    differ = difflib.unified_diff(lines1, lines2, fromfile=file1, tofile=file2)
+    diff_result = "".join(differ)
+    return diff_result
+
+
+def list_dir(path: str | Path) -> list[str]:
+    """
+    List the contents of a directory.
+
+    Args:
+        path (str): The path to the directory.
+
+    Returns:
+        list[str]: A list of the files and directories in the specified directory.
+    """
+    dir_path = Path(path)
+    if not dir_path.is_dir():
+        raise NotADirectoryError(f"Path is not a directory: {dir_path}")
+    return [str(p) for p in dir_path.iterdir()]

{langroid-0.16.7.dist-info → langroid-0.17.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: langroid
-Version: 0.16.7
+Version: 0.17.0
 Summary: Harness LLMs with Multi-Agent Programming
 License: MIT
 Author: Prasad Chalasani
@@ -48,6 +48,7 @@ Requires-Dist: faker (>=18.9.0,<19.0.0)
 Requires-Dist: fakeredis (>=2.12.1,<3.0.0)
 Requires-Dist: fastembed (>=0.3.1,<0.4.0) ; extra == "all" or extra == "fastembed"
 Requires-Dist: fire (>=0.5.0,<0.6.0)
+Requires-Dist: gitpython (>=3.1.43,<4.0.0)
 Requires-Dist: google-api-python-client (>=2.95.0,<3.0.0)
 Requires-Dist: google-generativeai (>=0.5.2,<0.6.0)
 Requires-Dist: groq (>=0.5.0,<0.6.0)
@@ -55,7 +56,7 @@ Requires-Dist: grpcio (>=1.62.1,<2.0.0)
 Requires-Dist: halo (>=0.0.31,<0.0.32)
 Requires-Dist: huggingface-hub (>=0.21.2,<0.22.0) ; extra == "hf-transformers" or extra == "all" or extra == "transformers"
 Requires-Dist: jinja2 (>=3.1.2,<4.0.0)
-Requires-Dist: json-repair (>=0.29.0,<0.30.0)
+Requires-Dist: json-repair (>=0.29.9,<0.30.0)
 Requires-Dist: lancedb (>=0.8.2,<0.9.0) ; extra == "vecdbs" or extra == "lancedb"
 Requires-Dist: litellm (>=1.30.1,<2.0.0) ; extra == "all" or extra == "litellm"
 Requires-Dist: lxml (>=4.9.3,<5.0.0)
@@ -143,7 +144,7 @@ Description-Content-Type: text/markdown
 </h3>
 
 `Langroid` is an intuitive, lightweight, extensible and principled
-Python framework to easily build LLM-powered applications, from ex-CMU and UW-Madison researchers. 
+Python framework to easily build LLM-powered applications, from CMU and UW-Madison researchers. 
 You set up Agents, equip them with optional components (LLM, 
 vector-store and tools/functions), assign them tasks, and have them 
 collaboratively solve a problem by exchanging messages. 
@@ -242,6 +243,8 @@ teacher_task.run()
 <details>
 <summary> <b>Click to expand</b></summary>
 
+- **Oct 2024:**
+  - **[0.17.0]** XML-based tools, see [docs](https://langroid.github.io/langroid/tutorials/xml-tools/).
 - **Sep 2024:**
   - **[0.16.0](https://github.com/langroid/langroid/releases/tag/0.16.0)**  Support for OpenAI `o1-mini` and `o1-preview` models.
   - **[0.15.0](https://github.com/langroid/langroid/releases/tag/0.15.0)** Cerebras API support -- run llama-3.1 models hosted on Cerebras Cloud (very fast inference).