patch-chat 0.1.0__py3-none-any.whl

patch/run_cmd.py

@@ -0,0 +1,146 @@
+import os
+import platform
+import subprocess
+import sys
+from io import BytesIO
+
+import pexpect
+import psutil
+
+
+def run_cmd(command, verbose=False, error_print=None, cwd=None):
+    try:
+        if sys.stdin.isatty() and hasattr(pexpect, "spawn") and platform.system() != "Windows":
+            return run_cmd_pexpect(command, verbose, cwd)
+
+        return run_cmd_subprocess(command, verbose, cwd)
+    except OSError as e:
+        error_message = f"Error occurred while running command '{command}': {str(e)}"
+        if error_print is None:
+            print(error_message)
+        else:
+            error_print(error_message)
+        return 1, error_message
+
+
+def get_windows_parent_process_name():
+    try:
+        current_process = psutil.Process()
+        while True:
+            parent = current_process.parent()
+            if parent is None:
+                break
+            parent_name = parent.name().lower()
+            if parent_name in ["powershell.exe", "cmd.exe"]:
+                return parent_name
+            current_process = parent
+        return None
+    except Exception:
+        return None
+
+
+def run_cmd_subprocess(command, verbose=False, cwd=None, encoding=sys.stdout.encoding):
+    if verbose:
+        print("Using run_cmd_subprocess:", command)
+
+    process = None
+    try:
+        shell = os.environ.get("SHELL", "/bin/sh")
+        parent_process = None
+
+        # Determine the appropriate shell
+        if platform.system() == "Windows":
+            parent_process = get_windows_parent_process_name()
+            if parent_process == "powershell.exe":
+                command = f"powershell -Command {command}"
+
+        if verbose:
+            print("Running command:", command)
+            print("SHELL:", shell)
+            if platform.system() == "Windows":
+                print("Parent process:", parent_process)
+
+        process = subprocess.Popen(
+            command,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True,
+            shell=True,
+            encoding=encoding,
+            errors="replace",
+            bufsize=0,  # Set bufsize to 0 for unbuffered output
+            universal_newlines=True,
+            cwd=cwd,
+        )
+
+        output = []
+        while True:
+            chunk = process.stdout.read(1)
+            if not chunk:
+                break
+            print(chunk, end="", flush=True)  # Print the chunk in real-time
+            output.append(chunk)  # Store the chunk for later use
+
+        process.wait()
+        return process.returncode, "".join(output)
+    except Exception as e:
+        return 1, str(e)
+    finally:
+        if process is not None:
+            if process.poll() is None:
+                try:
+                    process.terminate()
+                    process.wait(timeout=1)
+                except subprocess.TimeoutExpired:
+                    process.kill()
+                    process.wait()
+                except OSError:
+                    process.wait()
+            if process.stdout:
+                process.stdout.close()
+
+
+def run_cmd_pexpect(command, verbose=False, cwd=None):
+    """
+    Run a shell command interactively using pexpect, capturing all output.
+
+    :param command: The command to run as a string.
+    :param verbose: If True, print output in real-time.
+    :return: A tuple containing (exit_status, output)
+    """
+    if verbose:
+        print("Using run_cmd_pexpect:", command)
+
+    output = BytesIO()
+
+    def output_callback(b):
+        output.write(b)
+        return b
+
+    try:
+        # Use the SHELL environment variable, falling back to /bin/sh if not set
+        shell = os.environ.get("SHELL", "/bin/sh")
+        if verbose:
+            print("With shell:", shell)
+
+        if os.path.exists(shell):
+            # Use the shell from SHELL environment variable
+            if verbose:
+                print("Running pexpect.spawn with shell:", shell)
+            child = pexpect.spawn(shell, args=["-i", "-c", command], encoding="utf-8", cwd=cwd)
+        else:
+            # Fall back to spawning the command directly
+            if verbose:
+                print("Running pexpect.spawn without shell.")
+            child = pexpect.spawn(command, encoding="utf-8", cwd=cwd)
+
+        # Transfer control to the user, capturing output
+        child.interact(output_filter=output_callback)
+
+        # Wait for the command to finish and get the exit status
+        child.close()
+        return child.exitstatus, output.getvalue().decode("utf-8", errors="replace")
+
+    except (pexpect.ExceptionPexpect, TypeError, ValueError) as e:
+        error_msg = f"Error running command {command}: {e}"
+        return 1, error_msg

patch/scrape.py

@@ -0,0 +1,316 @@
+#!/usr/bin/env python
+
+import re
+import sys
+
+import pypandoc
+
+from patch import __version__, urls, utils
+from patch.dump import dump  # noqa: F401
+
+patch_user_agent = f"Patch/{__version__} +{urls.website}"
+MAX_SCRAPED_CONTENT_BYTES = 5 * 1024 * 1024
+
+# Playwright is nice because it has a simple way to install dependencies on most
+# platforms.
+
+
+def check_env():
+    try:
+        from playwright.sync_api import sync_playwright
+
+        has_pip = True
+    except ImportError:
+        has_pip = False
+
+    try:
+        with sync_playwright() as p:
+            p.chromium.launch()
+            has_chromium = True
+    except Exception:
+        has_chromium = False
+
+    return has_pip, has_chromium
+
+
+def has_playwright():
+    has_pip, has_chromium = check_env()
+    return has_pip and has_chromium
+
+
+def install_playwright(io):
+    has_pip, has_chromium = check_env()
+    if has_pip and has_chromium:
+        return True
+
+    pip_cmd = utils.get_pip_install(["patch-chat[playwright]"])
+    chromium_cmd = "-m playwright install --with-deps chromium"
+    chromium_cmd = [sys.executable] + chromium_cmd.split()
+
+    cmds = ""
+    if not has_pip:
+        cmds += " ".join(pip_cmd) + "\n"
+    if not has_chromium:
+        cmds += " ".join(chromium_cmd) + "\n"
+
+    text = f"""For the best web scraping, install Playwright:
+
+{cmds}
+See {urls.enable_playwright} for more info.
+"""
+
+    io.tool_output(text)
+    if not io.confirm_ask("Install playwright?", default="y"):
+        return
+
+    if not has_pip:
+        success, output = utils.run_install(pip_cmd)
+        if not success:
+            io.tool_error(output)
+            return
+
+    success, output = utils.run_install(chromium_cmd)
+    if not success:
+        io.tool_error(output)
+        return
+
+    return True
+
+
+class Scraper:
+    pandoc_available = None
+    playwright_available = None
+    playwright_instructions_shown = False
+
+    # Public API...
+    def __init__(self, print_error=None, playwright_available=None, verify_ssl=True):
+        """
+        `print_error` - a function to call to print error/debug info.
+        `verify_ssl` - if False, disable SSL certificate verification when scraping.
+        """
+        if print_error:
+            self.print_error = print_error
+        else:
+            self.print_error = print
+
+        self.playwright_available = playwright_available
+        self.verify_ssl = verify_ssl
+
+    def scrape(self, url):
+        """
+        Scrape a url and turn it into readable markdown if it's HTML.
+        If it's plain text or non-HTML, return it as-is.
+
+        `url` - the URL to scrape.
+        """
+
+        if self.playwright_available:
+            content, mime_type = self.scrape_with_playwright(url)
+        else:
+            content, mime_type = self.scrape_with_httpx(url)
+
+        if not content:
+            self.print_error(f"Failed to retrieve content from {url}")
+            return None
+
+        # Check if the content is HTML based on MIME type or content
+        if (mime_type and mime_type.startswith("text/html")) or (
+            mime_type is None and self.looks_like_html(content)
+        ):
+            self.try_pandoc()
+            content = self.html_to_markdown(content)
+
+        return content
+
+    def looks_like_html(self, content):
+        """
+        Check if the content looks like HTML.
+        """
+        if isinstance(content, str):
+            # Check for common HTML tags
+            html_patterns = [
+                r"<!DOCTYPE\s+html",
+                r"<html",
+                r"<head",
+                r"<body",
+                r"<div",
+                r"<p>",
+                r"<a\s+href=",
+            ]
+            return any(re.search(pattern, content, re.IGNORECASE) for pattern in html_patterns)
+        return False
+
+    def content_too_large(self, size, url):
+        if size <= MAX_SCRAPED_CONTENT_BYTES:
+            return False
+
+        if MAX_SCRAPED_CONTENT_BYTES >= 1024 * 1024:
+            limit = f"{MAX_SCRAPED_CONTENT_BYTES // (1024 * 1024)} MB"
+        else:
+            limit = f"{MAX_SCRAPED_CONTENT_BYTES} bytes"
+        self.print_error(f"Content from {url} exceeds the {limit} scraping limit")
+        return True
+
+    # Internals...
+    def scrape_with_playwright(self, url):
+        import playwright  # noqa: F401
+        from playwright.sync_api import Error as PlaywrightError
+        from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
+        from playwright.sync_api import sync_playwright
+
+        with sync_playwright() as p:
+            try:
+                browser = p.chromium.launch()
+            except Exception as e:
+                self.playwright_available = False
+                self.print_error(str(e))
+                return None, None
+
+            try:
+                context = browser.new_context(ignore_https_errors=not self.verify_ssl)
+                page = context.new_page()
+
+                user_agent = page.evaluate("navigator.userAgent")
+                user_agent = user_agent.replace("Headless", "")
+                user_agent = user_agent.replace("headless", "")
+                user_agent += " " + patch_user_agent
+
+                page.set_extra_http_headers({"User-Agent": user_agent})
+
+                response = None
+                try:
+                    response = page.goto(url, wait_until="networkidle", timeout=5000)
+                except PlaywrightTimeoutError:
+                    print(f"Page didn't quiesce, scraping content anyway: {url}")
+                    response = None
+                except PlaywrightError as e:
+                    self.print_error(f"Error navigating to {url}: {str(e)}")
+                    return None, None
+
+                try:
+                    content_size = page.evaluate(
+                        "new TextEncoder().encode(document.documentElement.outerHTML).length"
+                    )
+                    if self.content_too_large(content_size, url):
+                        return None, None
+                    content = page.content()
+                    mime_type = None
+                    if response:
+                        content_type = response.header_value("content-type")
+                        if content_type:
+                            mime_type = content_type.split(";")[0]
+                except PlaywrightError as e:
+                    self.print_error(f"Error retrieving page content: {str(e)}")
+                    content = None
+                    mime_type = None
+            finally:
+                browser.close()
+
+        return content, mime_type
+
+    def scrape_with_httpx(self, url):
+        import httpx
+
+        headers = {"User-Agent": f"Mozilla./5.0 ({patch_user_agent})"}
+        try:
+            with httpx.Client(
+                headers=headers, verify=self.verify_ssl, follow_redirects=True
+            ) as client:
+                with client.stream("GET", url) as response:
+                    response.raise_for_status()
+                    content_length = response.headers.get("content-length")
+                    if content_length and self.content_too_large(int(content_length), url):
+                        return None, None
+
+                    chunks = []
+                    size = 0
+                    for chunk in response.iter_bytes():
+                        size += len(chunk)
+                        if self.content_too_large(size, url):
+                            return None, None
+                        chunks.append(chunk)
+
+                    encoding = response.encoding or "utf-8"
+                    content = b"".join(chunks).decode(encoding, errors="replace")
+                    mime_type = response.headers.get("content-type", "").split(";")[0]
+                    return content, mime_type
+        except httpx.HTTPError as http_err:
+            self.print_error(f"HTTP error occurred: {http_err}")
+        except Exception as err:
+            self.print_error(f"An error occurred: {err}")
+        return None, None
+
+    def try_pandoc(self):
+        if self.pandoc_available:
+            return
+
+        try:
+            pypandoc.get_pandoc_version()
+            self.pandoc_available = True
+            return
+        except OSError:
+            pass
+
+        try:
+            pypandoc.download_pandoc(delete_installer=True)
+        except Exception as err:
+            self.print_error(f"Unable to install pandoc: {err}")
+            return
+
+        self.pandoc_available = True
+
+    def html_to_markdown(self, page_source):
+        from bs4 import BeautifulSoup
+
+        soup = BeautifulSoup(page_source, "html.parser")
+        soup = slimdown_html(soup)
+        page_source = str(soup)
+
+        if not self.pandoc_available:
+            return page_source
+
+        try:
+            md = pypandoc.convert_text(page_source, "markdown", format="html")
+        except OSError:
+            return page_source
+
+        md = re.sub(r"</div>", "      ", md)
+        md = re.sub(r"<div>", "     ", md)
+
+        md = re.sub(r"\n\s*\n", "\n\n", md)
+
+        return md
+
+
+def slimdown_html(soup):
+    for svg in soup.find_all("svg"):
+        svg.decompose()
+
+    if soup.img:
+        soup.img.decompose()
+
+    for tag in soup.find_all(href=lambda x: x and x.startswith("data:")):
+        tag.decompose()
+
+    for tag in soup.find_all(src=lambda x: x and x.startswith("data:")):
+        tag.decompose()
+
+    for tag in soup.find_all(True):
+        for attr in list(tag.attrs):
+            if attr != "href":
+                tag.attrs.pop(attr, None)
+
+    return soup
+
+
+def main(url):
+    scraper = Scraper(playwright_available=has_playwright())
+    content = scraper.scrape(url)
+    print(content)
+
+
+if __name__ == "__main__":
+    if len(sys.argv) < 2:
+        print("Usage: python playw.py <URL>")
+        sys.exit(1)
+    main(sys.argv[1])

patch/sendchat.py

@@ -0,0 +1,61 @@
+from patch.dump import dump  # noqa: F401
+from patch.utils import format_messages
+
+
+def sanity_check_messages(messages):
+    """Check if messages alternate between user and assistant roles.
+    System messages can be interspersed anywhere.
+    Also verifies the last non-system message is from the user.
+    Returns True if valid, False otherwise."""
+    last_role = None
+    last_non_system_role = None
+
+    for msg in messages:
+        role = msg.get("role")
+        if role == "system":
+            continue
+
+        if last_role and role == last_role:
+            turns = format_messages(messages)
+            raise ValueError("Messages don't properly alternate user/assistant:\n\n" + turns)
+
+        last_role = role
+        last_non_system_role = role
+
+    # Ensure last non-system message is from user
+    return last_non_system_role == "user"
+
+
+def ensure_alternating_roles(messages):
+    """Ensure messages alternate between 'assistant' and 'user' roles.
+
+    Inserts empty messages of the opposite role when consecutive messages
+    of the same role are found.
+
+    Args:
+        messages: List of message dictionaries with 'role' and 'content' keys.
+
+    Returns:
+        List of messages with alternating roles.
+    """
+    if not messages:
+        return messages
+
+    fixed_messages = []
+    prev_role = None
+
+    for msg in messages:
+        current_role = msg.get("role")  # Get 'role', None if missing
+
+        # If current role same as previous, insert empty message
+        # of the opposite role
+        if current_role == prev_role:
+            if current_role == "user":
+                fixed_messages.append({"role": "assistant", "content": ""})
+            else:
+                fixed_messages.append({"role": "user", "content": ""})
+
+        fixed_messages.append(msg)
+        prev_role = current_role
+
+    return fixed_messages