aider-ce 0.88.20__py3-none-any.whl

aider/run_cmd.py

@@ -0,0 +1,133 @@
+import os
+import platform
+import subprocess
+import sys
+from io import BytesIO
+
+import pexpect
+import psutil
+
+
+def run_cmd(command, verbose=False, error_print=None, cwd=None):
+    try:
+        if sys.stdin.isatty() and hasattr(pexpect, "spawn") and platform.system() != "Windows":
+            return run_cmd_pexpect(command, verbose, cwd)
+
+        return run_cmd_subprocess(command, verbose, cwd)
+    except OSError as e:
+        error_message = f"Error occurred while running command '{command}': {str(e)}"
+        if error_print is None:
+            print(error_message)
+        else:
+            error_print(error_message)
+        return 1, error_message
+
+
+def get_windows_parent_process_name():
+    try:
+        current_process = psutil.Process()
+        while True:
+            parent = current_process.parent()
+            if parent is None:
+                break
+            parent_name = parent.name().lower()
+            if parent_name in ["powershell.exe", "cmd.exe"]:
+                return parent_name
+            current_process = parent
+        return None
+    except Exception:
+        return None
+
+
+def run_cmd_subprocess(command, verbose=False, cwd=None, encoding=sys.stdout.encoding):
+    if verbose:
+        print("Using run_cmd_subprocess:", command)
+
+    try:
+        shell = os.environ.get("SHELL", "/bin/sh")
+        parent_process = None
+
+        # Determine the appropriate shell
+        if platform.system() == "Windows":
+            parent_process = get_windows_parent_process_name()
+            if parent_process == "powershell.exe":
+                command = f"powershell -Command {command}"
+
+        if verbose:
+            print("Running command:", command)
+            print("SHELL:", shell)
+            if platform.system() == "Windows":
+                print("Parent process:", parent_process)
+
+        process = subprocess.Popen(
+            command,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True,
+            shell=True,
+            executable=shell if platform.system() != "Windows" else None,
+            encoding=encoding,
+            errors="replace",
+            bufsize=0,  # Set bufsize to 0 for unbuffered output
+            universal_newlines=True,
+            cwd=cwd,
+        )
+
+        output = []
+        while True:
+            chunk = process.stdout.read(1)
+            if not chunk:
+                break
+            print(chunk, end="", flush=True)  # Print the chunk in real-time
+            output.append(chunk)  # Store the chunk for later use
+
+        process.wait()
+        return process.returncode, "".join(output)
+    except Exception as e:
+        return 1, str(e)
+
+
+def run_cmd_pexpect(command, verbose=False, cwd=None):
+    """
+    Run a shell command interactively using pexpect, capturing all output.
+
+    :param command: The command to run as a string.
+    :param verbose: If True, print output in real-time.
+    :return: A tuple containing (exit_status, output)
+    """
+    if verbose:
+        print("Using run_cmd_pexpect:", command)
+
+    output = BytesIO()
+
+    def output_callback(b):
+        output.write(b)
+        return b
+
+    try:
+        # Use the SHELL environment variable, falling back to /bin/sh if not set
+        shell = os.environ.get("SHELL", "/bin/sh")
+        if verbose:
+            print("With shell:", shell)
+
+        if os.path.exists(shell):
+            # Use the shell from SHELL environment variable
+            if verbose:
+                print("Running pexpect.spawn with shell:", shell)
+            child = pexpect.spawn(shell, args=["-i", "-c", command], encoding="utf-8", cwd=cwd)
+        else:
+            # Fall back to spawning the command directly
+            if verbose:
+                print("Running pexpect.spawn without shell.")
+            child = pexpect.spawn(command, encoding="utf-8", cwd=cwd)
+
+        # Transfer control to the user, capturing output
+        child.interact(output_filter=output_callback)
+
+        # Wait for the command to finish and get the exit status
+        child.close()
+        return child.exitstatus, output.getvalue().decode("utf-8", errors="replace")
+
+    except (pexpect.ExceptionPexpect, TypeError, ValueError) as e:
+        error_msg = f"Error running command {command}: {e}"
+        return 1, error_msg

aider/scrape.py

@@ -0,0 +1,293 @@
+#!/usr/bin/env python
+
+import re
+import sys
+
+import pypandoc
+
+from aider import __version__, urls, utils
+from aider.dump import dump  # noqa: F401
+
+aider_user_agent = f"Aider/{__version__} +{urls.website}"
+
+# Playwright is nice because it has a simple way to install dependencies on most
+# platforms.
+
+
+def check_playwright():
+    try:
+        from playwright.async_api import async_playwright  # noqa: F401
+        from playwright.sync_api import sync_playwright  # noqa: F401
+
+        has_playwright = True
+    except ImportError:
+        has_playwright = False
+
+    return has_playwright
+
+
+async def check_chromium():
+    has_chromium = False
+
+    if check_playwright():
+        from playwright.async_api import async_playwright
+
+        try:
+            async with async_playwright() as p:
+                browser = await p.chromium.launch()
+                await browser.close()
+                has_chromium = True
+        except Exception as e:
+            has_chromium = False
+            print(f"chromium errors {e}")
+
+    return has_chromium
+
+
+async def install_playwright(io):
+    has_playwright = check_playwright()
+    has_chromium = await check_chromium()
+
+    if has_playwright and has_chromium:
+        return True
+
+    pip_cmd = utils.get_pip_install(["aider-ce[playwright]"])
+    chromium_cmd = "-m playwright install --with-deps chromium"
+    chromium_cmd = [sys.executable] + chromium_cmd.split()
+
+    cmds = ""
+    if not has_playwright:
+        cmds += " ".join(pip_cmd) + "\n"
+    if not has_chromium:
+        cmds += " ".join(chromium_cmd) + "\n"
+
+    text = f"""For the best web scraping, install Playwright:
+
+{cmds}
+See {urls.enable_playwright} for more info.
+"""
+
+    io.tool_output(text)
+    if not await io.confirm_ask("Install playwright?", default="y"):
+        return
+
+    if not has_playwright:
+        success, output = utils.run_install(pip_cmd)
+        if not success:
+            io.tool_error(output)
+            return
+
+    success, output = utils.run_install(chromium_cmd)
+    if not success:
+        io.tool_error(output)
+        return
+
+    return True
+
+
+class Scraper:
+    pandoc_available = None
+    playwright_available = None
+    playwright_instructions_shown = False
+
+    # Public API...
+    def __init__(self, print_error=None, playwright_available=None, verify_ssl=True):
+        """
+        `print_error` - a function to call to print error/debug info.
+        `verify_ssl` - if False, disable SSL certificate verification when scraping.
+        """
+        if print_error:
+            self.print_error = print_error
+        else:
+            self.print_error = print
+
+        self.playwright_available = playwright_available
+        self.verify_ssl = verify_ssl
+
+    async def scrape(self, url):
+        """
+        Scrape a url and turn it into readable markdown if it's HTML.
+        If it's plain text or non-HTML, return it as-is.
+
+        `url` - the URL to scrape.
+        """
+
+        if self.playwright_available:
+            content, mime_type = await self.scrape_with_playwright(url)
+        else:
+            content, mime_type = self.scrape_with_httpx(url)
+
+        if not content:
+            self.print_error(f"Failed to retrieve content from {url}")
+            return None
+
+        # Check if the content is HTML based on MIME type or content
+        if (mime_type and mime_type.startswith("text/html")) or (
+            mime_type is None and self.looks_like_html(content)
+        ):
+            self.try_pandoc()
+            content = self.html_to_markdown(content)
+
+        return content
+
+    def looks_like_html(self, content):
+        """
+        Check if the content looks like HTML.
+        """
+        if isinstance(content, str):
+            # Check for common HTML tags
+            html_patterns = [
+                r"<!DOCTYPE\s+html",
+                r"<html",
+                r"<head",
+                r"<body",
+                r"<div",
+                r"<p>",
+                r"<a\s+href=",
+            ]
+            return any(re.search(pattern, content, re.IGNORECASE) for pattern in html_patterns)
+        return False
+
+    # Internals...
+    async def scrape_with_playwright(self, url):
+        import playwright  # noqa: F401
+        from playwright.async_api import Error as PlaywrightError
+        from playwright.async_api import TimeoutError as PlaywrightTimeoutError
+        from playwright.async_api import async_playwright
+
+        async with async_playwright() as p:
+            try:
+                browser = await p.chromium.launch()
+            except Exception as e:
+                self.playwright_available = False
+                self.print_error(str(e))
+                return None, None
+
+            try:
+                context = await browser.new_context(ignore_https_errors=not self.verify_ssl)
+                page = await context.new_page()
+
+                user_agent = await page.evaluate("navigator.userAgent")
+                user_agent = user_agent.replace("Headless", "")
+                user_agent = user_agent.replace("headless", "")
+                user_agent += " " + aider_user_agent
+
+                await page.set_extra_http_headers({"User-Agent": user_agent})
+
+                response = None
+                try:
+                    response = await page.goto(url, wait_until="networkidle", timeout=5000)
+                except PlaywrightTimeoutError:
+                    self.print_error(f"Page didn't quiesce, scraping content anyway: {url}")
+                    response = None
+                except PlaywrightError as e:
+                    self.print_error(f"Error navigating to {url}: {str(e)}")
+                    return None, None
+
+                try:
+                    content = await page.content()
+                    mime_type = None
+                    if response:
+                        content_type = await response.header_value("content-type")
+                        if content_type:
+                            mime_type = content_type.split(";")[0]
+                except PlaywrightError as e:
+                    self.print_error(f"Error retrieving page content: {str(e)}")
+                    content = None
+                    mime_type = None
+            finally:
+                await browser.close()
+
+        return content, mime_type
+
+    def scrape_with_httpx(self, url):
+        import httpx
+
+        headers = {"User-Agent": f"Mozilla./5.0 ({aider_user_agent})"}
+        try:
+            with httpx.Client(
+                headers=headers, verify=self.verify_ssl, follow_redirects=True
+            ) as client:
+                response = client.get(url)
+                response.raise_for_status()
+                return response.text, response.headers.get("content-type", "").split(";")[0]
+        except httpx.HTTPError as http_err:
+            self.print_error(f"HTTP error occurred: {http_err}")
+        except Exception as err:
+            self.print_error(f"An error occurred: {err}")
+        return None, None
+
+    def try_pandoc(self):
+        if self.pandoc_available:
+            return
+
+        try:
+            pypandoc.get_pandoc_version()
+            self.pandoc_available = True
+            return
+        except OSError:
+            pass
+
+        try:
+            pypandoc.download_pandoc(delete_installer=True)
+        except Exception as err:
+            self.print_error(f"Unable to install pandoc: {err}")
+            return
+
+        self.pandoc_available = True
+
+    def html_to_markdown(self, page_source):
+        from bs4 import BeautifulSoup
+
+        soup = BeautifulSoup(page_source, "html.parser")
+        soup = slimdown_html(soup)
+        page_source = str(soup)
+
+        if not self.pandoc_available:
+            return page_source
+
+        try:
+            md = pypandoc.convert_text(page_source, "markdown", format="html")
+        except OSError:
+            return page_source
+
+        md = re.sub(r"</div>", "      ", md)
+        md = re.sub(r"<div>", "     ", md)
+
+        md = re.sub(r"\n\s*\n", "\n\n", md)
+
+        return md
+
+
+def slimdown_html(soup):
+    for svg in soup.find_all("svg"):
+        svg.decompose()
+
+    if soup.img:
+        soup.img.decompose()
+
+    for tag in soup.find_all(href=lambda x: x and x.startswith("data:")):
+        tag.decompose()
+
+    for tag in soup.find_all(src=lambda x: x and x.startswith("data:")):
+        tag.decompose()
+
+    for tag in soup.find_all(True):
+        for attr in list(tag.attrs):
+            if attr != "href":
+                tag.attrs.pop(attr, None)
+
+    return soup
+
+
+async def main(url):
+    scraper = Scraper(playwright_available=check_playwright())
+    content = await scraper.scrape(url)
+    print(content)
+
+
+if __name__ == "__main__":
+    if len(sys.argv) < 2:
+        print("Usage: python playw.py <URL>")
+        sys.exit(1)
+    main(sys.argv[1])

aider/sendchat.py

@@ -0,0 +1,242 @@
+from aider.dump import dump  # noqa: F401
+from aider.utils import format_messages
+
+
+def sanity_check_messages(messages):
+    """Check if messages alternate between user and assistant roles.
+    System messages can be interspersed anywhere.
+    Also verifies the last non-system message is from the user.
+    Validates tool message sequences.
+    Returns True if valid, False otherwise."""
+    last_role = None
+    last_non_system_role = None
+    i = 0
+    n = len(messages)
+
+    while i < n:
+        msg = messages[i]
+        role = msg.get("role")
+
+        # Handle tool sequences atomically
+        if role == "assistant" and "tool_calls" in msg and msg["tool_calls"]:
+            # Validate tool sequence
+            expected_ids = {call["id"] for call in msg["tool_calls"]}
+            i += 1
+
+            # Check for tool responses
+            while i < n and expected_ids:
+                next_msg = messages[i]
+                if next_msg.get("role") == "tool" and next_msg.get("tool_call_id") in expected_ids:
+                    expected_ids.discard(next_msg.get("tool_call_id"))
+                    i += 1
+                else:
+                    break
+
+            # If we still have expected IDs, the tool sequence is incomplete
+            if expected_ids:
+                turns = format_messages(messages)
+                raise ValueError(
+                    "Incomplete tool sequence - missing responses for tool calls:\n\n" + turns
+                )
+
+            # Continue to next message after tool sequence
+            continue
+
+        elif role == "tool":
+            # Orphaned tool message without preceding assistant tool_calls
+            turns = format_messages(messages)
+            raise ValueError(
+                "Orphaned tool message without preceding assistant tool_calls:\n\n" + turns
+            )
+
+        # Handle normal role alternation
+        if role == "system":
+            i += 1
+            continue
+
+        if last_role and role == last_role:
+            turns = format_messages(messages)
+            raise ValueError("Messages don't properly alternate user/assistant:\n\n" + turns)
+
+        last_role = role
+        last_non_system_role = role
+        i += 1
+
+    # Ensure last non-system message is from user
+    return last_non_system_role == "user"
+
+
+def clean_orphaned_tool_messages(messages):
+    """Remove orphaned tool messages and incomplete tool sequences.
+
+    This function removes:
+    - Tool messages without a preceding assistant message containing tool_calls
+    - Assistant messages with tool_calls that don't have complete tool responses
+
+    Args:
+        messages: List of message dictionaries
+
+    Returns:
+        Cleaned list of messages with orphaned tool sequences removed
+    """
+    if not messages:
+        return messages
+
+    cleaned = []
+    i = 0
+    n = len(messages)
+
+    while i < n:
+        msg = messages[i]
+        role = msg.get("role")
+
+        # If it's an assistant message with tool_calls, check if we have complete responses
+        if role == "assistant" and "tool_calls" in msg and msg["tool_calls"]:
+            # Start of potential tool sequence
+            tool_sequence = [msg]
+            expected_ids = {call["id"] for call in msg["tool_calls"]}
+            j = i + 1
+
+            # Collect tool responses
+            while j < n and expected_ids:
+                next_msg = messages[j]
+                if next_msg.get("role") == "tool" and next_msg.get("tool_call_id") in expected_ids:
+                    tool_sequence.append(next_msg)
+                    expected_ids.discard(next_msg.get("tool_call_id"))
+                    j += 1
+                else:
+                    break
+
+            # If we have all tool responses, keep the sequence
+            if not expected_ids:
+                cleaned.extend(tool_sequence)
+                i = j
+            else:
+                # Incomplete sequence - skip the entire tool sequence
+                i = j
+                # Don't add anything to cleaned
+                continue
+
+        elif role == "tool":
+            # Orphaned tool message without preceding assistant tool_calls - skip it
+            i += 1
+            continue
+        else:
+            # Regular message - add it
+            cleaned.append(msg)
+            i += 1
+
+    return cleaned
+
+
+def ensure_alternating_roles(messages):
+    """Ensure messages alternate between 'assistant' and 'user' roles.
+
+    Inserts empty messages of the opposite role when consecutive messages
+    of the same 'user' or 'assistant' role are found. Messages with other
+    roles (e.g. 'system', 'tool') are ignored by the alternation logic.
+
+    Also handles tool call sequences properly - when an assistant message
+    contains tool_calls, processes the complete tool sequence atomically.
+
+    Args:
+        messages: List of message dictionaries with 'role' and 'content' keys.
+
+    Returns:
+        List of messages with alternating roles.
+    """
+    if not messages:
+        return messages
+
+    # First clean orphaned tool messages
+    messages = clean_orphaned_tool_messages(messages)
+
+    result = []
+    i = 0
+    n = len(messages)
+    prev_role = None
+
+    while i < n:
+        msg = messages[i]
+        role = msg.get("role")
+
+        # Handle tool call sequences atomically
+        if role == "assistant" and "tool_calls" in msg and msg["tool_calls"]:
+            # Start of tool sequence - collect all related messages
+            tool_sequence = [msg]
+            expected_ids = {call["id"] for call in msg["tool_calls"]}
+            i += 1
+
+            # Collect tool responses
+            while i < n and expected_ids:
+                next_msg = messages[i]
+                if next_msg.get("role") == "tool" and next_msg.get("tool_call_id") in expected_ids:
+                    tool_sequence.append(next_msg)
+                    expected_ids.discard(next_msg.get("tool_call_id"))
+                    i += 1
+                else:
+                    break
+
+            # Add missing tool responses as empty
+            for tool_id in expected_ids:
+                tool_sequence.append(
+                    {"role": "tool", "tool_call_id": tool_id, "content": "(empty response)"}
+                )
+
+            # Add the complete tool sequence to result
+            for tool_msg in tool_sequence:
+                result.append(tool_msg)
+
+            # Update prev_role to assistant after processing tool sequence
+            prev_role = "assistant"
+            continue
+
+        # Handle normal message alternation
+        if role in ("user", "assistant"):
+            if role == prev_role:
+                # Insert empty message of opposite role
+                opposite_role = "user" if role == "assistant" else "assistant"
+                result.append(
+                    {
+                        "role": opposite_role,
+                        "content": (
+                            "(empty response)"
+                            if opposite_role == "assistant"
+                            else "(empty request)"
+                        ),
+                    }
+                )
+                prev_role = opposite_role
+
+            result.append(msg)
+            prev_role = role
+        else:
+            # For non-user/assistant roles, just add them directly
+            result.append(msg)
+
+        i += 1
+
+    # Consolidate consecutive empty messages in a single pass
+    consolidated = []
+    for msg in result:
+        if not consolidated:
+            consolidated.append(msg)
+            continue
+
+        last_msg = consolidated[-1]
+        current_role = msg.get("role")
+        last_role = last_msg.get("role")
+
+        # Skip consecutive empty messages with the same role
+        if (
+            current_role in ("user", "assistant")
+            and last_role in ("user", "assistant")
+            and current_role == last_role
+            and msg.get("content") in ["", "(empty response)", "(empty request)"]
+            and last_msg.get("content") in ["", "(empty response)", "(empty request)"]
+        ):
+            continue
+
+        consolidated.append(msg)
+
+    return consolidated