indent 0.1.26__py3-none-any.whl

exponent/core/remote_execution/git.py

@@ -0,0 +1,268 @@
+import os
+from collections.abc import Callable
+from pathlib import Path
+from typing import cast
+
+import pygit2
+from anyio import Path as AsyncPath
+from gitignore_parser import (
+    IgnoreRule,
+    handle_negation,
+    parse_gitignore,
+    rule_from_pattern,
+)
+from pygit2 import Tree
+from pygit2.enums import DiffOption
+from pygit2.repository import Repository
+
+from exponent.core.remote_execution.types import (
+    GitInfo,
+)
+from exponent.core.remote_execution.utils import safe_read_file
+
+GIT_OBJ_COMMIT = 1
+
+
+async def git_file_walk(
+    repo: Repository,
+    directory: str,
+) -> list[str]:
+    """
+    Walk through a directory and return all file paths, respecting .gitignore and additional ignore patterns.
+    """
+    tree = get_git_subtree_for_dir(repo, directory)
+
+    if not tree:
+        return []
+
+    # diff to the empty tree to see all files
+    tracked_diff = tree.diff_to_tree()
+
+    tracked_files = [delta.new_file.path for delta in tracked_diff.deltas]
+
+    # Find untracked files relative to the root
+    untracked_diff = repo.diff(flags=DiffOption.INCLUDE_UNTRACKED)
+    untracked_files_from_root = [
+        AsyncPath(delta.new_file.path) for delta in untracked_diff.deltas
+    ]
+
+    # Current working directory relative to the repo root
+    dir_path = await AsyncPath(directory).resolve()
+    repo_path = await AsyncPath(repo.workdir).resolve()
+
+    if repo_path == dir_path:
+        relative_directory = str(repo_path)
+    else:
+        relative_directory = str(dir_path.relative_to(repo_path))
+
+    # Resolve all untracked files that are within the current working directory
+    untracked_files = []
+    for untracked_file in untracked_files_from_root:
+        if not untracked_file.is_relative_to(relative_directory):
+            continue
+
+        untracked_files.append(str(untracked_file.relative_to(relative_directory)))
+
+    # Combine both as sets to remove duplicates
+    return list(set(tracked_files) | set(untracked_files))
+
+
+def get_repo(working_directory: str) -> Repository | None:
+    try:
+        return Repository(working_directory)
+    except pygit2.GitError:
+        return None
+
+
+async def get_git_info(working_directory: str) -> GitInfo | None:
+    try:
+        repo = Repository(working_directory)
+    except pygit2.GitError:
+        return None
+
+    return GitInfo(
+        branch=(await _get_git_branch(repo)) or "<unknown branch>",
+        remote=_get_git_remote(repo),
+    )
+
+
+def get_tracked_files_in_dir(
+    repo: Repository,
+    dir: str | Path,
+    filter_func: Callable[[str], bool] | None = None,
+) -> list[str]:
+    rel_path = get_path_relative_to_repo_root(repo, dir)
+    dir_tree = get_git_subtree_for_dir(repo, dir)
+    entries: list[str] = []
+    if not dir_tree:
+        return entries
+    for entry in dir_tree:
+        if not entry.name:
+            continue
+        entry_path = str(Path(f"{repo.workdir}/{rel_path}/{entry.name}"))
+        if entry.type_str == "tree":
+            entries.extend(get_tracked_files_in_dir(repo, entry_path, filter_func))
+        elif entry.type_str == "blob":
+            if not filter_func or filter_func(entry.name):
+                entries.append(entry_path)
+    return entries
+
+
+def get_git_subtree_for_dir(repo: Repository, dir: str | Path) -> Tree | None:
+    rel_path = get_path_relative_to_repo_root(repo, dir)
+
+    try:
+        head_commit = repo.head.peel(GIT_OBJ_COMMIT)
+    except pygit2.GitError:
+        # If the repo is empty, then the head commit will not exist
+        return None
+    head_tree: Tree = head_commit.tree
+
+    if rel_path == Path("."):
+        # If the relative path is the root of the repo, then
+        # the head_tree is what we want. Note we do this because
+        # Passing "." or "" as the path into the tree will raise.
+        return head_tree
+    return cast(Tree, head_tree[str(rel_path)])
+
+
+def get_path_relative_to_repo_root(repo: Repository, path: str | Path) -> Path:
+    path = Path(path).resolve()
+    return path.relative_to(Path(repo.workdir).resolve())
+
+
+def get_local_commit_hash() -> str:
+    try:
+        # Open the repository (assumes the current working directory is within the git repo)
+        repo = Repository(os.getcwd())
+
+        # Get the current HEAD commit
+        head = repo.head
+
+        # Get the commit object and return its hash as a string
+        return str(repo[head.target].id)
+    except pygit2.GitError:
+        return "unknown-local-commit"
+
+
+def _get_git_remote(repo: Repository) -> str | None:
+    if repo.remotes:
+        return str(repo.remotes[0].url)
+    return None
+
+
+async def _get_git_branch(repo: Repository) -> str | None:
+    try:
+        # Look for HEAD file in the .git directory
+        head_path = AsyncPath(os.path.join(repo.path, "HEAD"))
+
+        if not await head_path.exists():
+            return None
+
+        head_content_raw = await safe_read_file(head_path)
+        head_content = head_content_raw.strip()
+
+        if head_content.startswith("ref:"):
+            return head_content.split("refs/heads/")[-1]
+        else:
+            return None
+
+    except Exception:
+        return None
+
+
+class GitIgnoreHandler:
+    def __init__(
+        self, working_directory: str, default_ignores: list[str] | None = None
+    ):
+        self.checkers = {}
+
+        if default_ignores:
+            self.checkers[working_directory] = self._parse_ignore_extra(
+                working_directory, default_ignores
+            )
+
+    async def read_ignorefile(self, path: str) -> None:
+        new_ignore = await self._get_ignored_checker(path)
+
+        if new_ignore:
+            self.checkers[path] = new_ignore
+
+    def filter(
+        self,
+        relpaths: list[str],
+        root: str,
+    ) -> list[str]:
+        result = []
+
+        for relpath in relpaths:
+            if relpath.startswith(".git"):
+                continue
+
+            path = os.path.join(root, relpath)
+
+            if self.is_ignored(path):
+                continue
+
+            result.append(relpath)
+
+        return result
+
+    def is_ignored(self, path: str) -> bool:
+        return any(
+            self.checkers[dp](path)
+            for dp in self.checkers
+            if self._is_subpath(path, dp)
+        )
+
+    def _parse_ignore_extra(
+        self, working_directory: str, ignore_extra: list[str]
+    ) -> Callable[[str], bool]:
+        rules: list[IgnoreRule] = []
+
+        for pattern in ignore_extra:
+            if (
+                rule := rule_from_pattern(pattern, base_path=working_directory)
+            ) is not None:
+                rules.append(rule)
+
+        def rule_handler(file_path: str) -> bool:
+            nonlocal rules
+            return bool(handle_negation(file_path, rules))
+
+        return rule_handler
+
+    async def _get_ignored_checker(self, dir_path: str) -> Callable[[str], bool] | None:
+        new_ignore = self._parse_gitignore(dir_path)
+
+        existing_ignore = self.checkers.get(dir_path)
+
+        if existing_ignore and new_ignore:
+            return self._or(new_ignore, existing_ignore)
+
+        return new_ignore or existing_ignore
+
+    @staticmethod
+    def _parse_gitignore(directory: str) -> Callable[[str], bool] | None:
+        gitignore_path = os.path.join(directory, ".gitignore")
+
+        if os.path.isfile(gitignore_path):
+            return cast(Callable[[str], bool], parse_gitignore(gitignore_path))
+
+        return None
+
+    @staticmethod
+    def _or(
+        a: Callable[[str], bool], b: Callable[[str], bool]
+    ) -> Callable[[str], bool]:
+        def or_handler(file_path: str) -> bool:
+            return a(file_path) or b(file_path)
+
+        return or_handler
+
+    @staticmethod
+    def _is_subpath(path: str, parent: str) -> bool:
+        """
+        Check if a path is a subpath of another path.
+        """
+        return os.path.commonpath([path, parent]) == parent

exponent/core/remote_execution/http_fetch.py

@@ -0,0 +1,94 @@
+"""HTTP fetch implementation for remote execution client."""
+
+import logging
+
+import httpx
+
+from exponent.core.remote_execution.cli_rpc_types import (
+    HttpRequest,
+    HttpResponse,
+)
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_TIMEOUT = 30.0
+DEFAULT_USER_AGENT = "Indent-HTTP-Client/1.0"
+
+
+async def fetch_http_content(http_request: HttpRequest) -> HttpResponse:
+    """
+    Fetch content from an HTTP URL and return the response.
+
+    Args:
+        http_request: HttpRequest containing URL, method, headers, and timeout
+
+    Returns:
+        HttpResponse with status code, content, and error message if any
+    """
+    logger.info(f"Fetching {http_request.method} {http_request.url}")
+
+    try:
+        # Set up timeout
+        timeout = (
+            http_request.timeout
+            if http_request.timeout is not None
+            else DEFAULT_TIMEOUT
+        )
+
+        # Set up headers with default User-Agent
+        headers = http_request.headers or {}
+        if "User-Agent" not in headers:
+            headers["User-Agent"] = DEFAULT_USER_AGENT
+
+        # Create HTTP client with timeout
+        async with httpx.AsyncClient(timeout=timeout) as client:
+            # Make the HTTP request
+            response = await client.request(
+                method=http_request.method,
+                url=http_request.url,
+                headers=headers,
+            )
+
+            # Get response content as text
+            try:
+                content = response.text
+            except UnicodeDecodeError:
+                # If content can't be decoded as text, provide a fallback
+                content = f"Binary content ({len(response.content)} bytes)"
+                logger.warning(
+                    f"Could not decode response content as text for {http_request.url}"
+                )
+
+            logger.info(
+                f"HTTP {http_request.method} {http_request.url} -> {response.status_code}"
+            )
+
+            return HttpResponse(
+                status_code=response.status_code,
+                content=content,
+                error_message=None,
+            )
+
+    except httpx.TimeoutException:
+        error_msg = f"Request to {http_request.url} timed out after {timeout} seconds"
+        return HttpResponse(
+            status_code=None,
+            content="",
+            error_message=error_msg,
+        )
+
+    except httpx.RequestError as e:
+        error_msg = f"Request error for {http_request.url}: {e!s}"
+        return HttpResponse(
+            status_code=None,
+            content="",
+            error_message=error_msg,
+        )
+
+    except Exception as e:
+        error_msg = f"Unexpected error fetching {http_request.url}: {e!s}"
+        return HttpResponse(
+            status_code=None,
+            content="",
+            error_message=error_msg,
+        )

exponent/core/remote_execution/languages/python_execution.py

@@ -0,0 +1,239 @@
+import asyncio
+import logging
+import queue
+import re
+import sys
+import threading
+import time
+from collections.abc import AsyncGenerator, Callable
+from typing import Any
+
+from jupyter_client.client import KernelClient
+from jupyter_client.manager import KernelManager
+
+from exponent.core.remote_execution.languages.types import (
+    PythonExecutionResult,
+    StreamedOutputPiece,
+)
+from exponent.core.remote_execution.types import PythonEnvInfo
+
+logger = logging.getLogger(__name__)
+
+
+class IOChannelHandler:
+    ESCAPE_SEQUENCE = re.compile(r"\x1B\[[0-?]*[ -/]*[@-~]")
+
+    def __init__(self, user_interrupted: Callable[[], bool] | None = None) -> None:
+        self.output_buffer: queue.Queue[str] = queue.Queue()
+        self.user_interrupted = user_interrupted
+
+    def add_message(self, message: dict[str, Any]) -> None:
+        logger.debug(f"Jupyter kernel message received: {message}")
+        output = None
+        if message["msg_type"] == "stream":
+            output = message["content"]["text"]
+        elif message["msg_type"] == "error":
+            raw_content = "\n".join(message["content"]["traceback"])
+            content = self.ESCAPE_SEQUENCE.sub("", raw_content)
+            output = content
+        if output:
+            self.output_buffer.put(output)
+
+    @staticmethod
+    def is_idle(message: dict[str, Any]) -> bool:
+        return bool(
+            message["header"]["msg_type"] == "status"
+            and message["content"]["execution_state"] == "idle"
+        )
+
+
+class Kernel:
+    def __init__(self, working_directory: str) -> None:
+        self._manager: KernelManager | None = None
+        self._client: KernelClient | None = None
+        self.io_handler: IOChannelHandler = IOChannelHandler()
+        self.working_directory = working_directory
+        self.interrupted_by_user: bool = False
+
+    @property
+    def manager(self) -> KernelManager:
+        if not self._manager:
+            self._manager = KernelManager(kernel_name="python3")
+            self._manager.start_kernel(cwd=self.working_directory)
+        return self._manager
+
+    @property
+    def client(self) -> KernelClient:
+        if not self._client:
+            self._client = self.manager.client()
+
+            while not self._client.is_alive():
+                time.sleep(0.1)
+
+            self._client.start_channels()
+        return self._client
+
+    async def wait_for_ready(self, timeout: int = 5) -> None:
+        manager = self.manager
+        start_time = time.time()
+        while not manager.is_alive():
+            if time.time() - start_time > timeout:
+                raise Exception("Kernel took too long to start")
+            await asyncio.sleep(0.05)
+        await asyncio.sleep(0.5)
+
+    def _clear_channels(self) -> None:
+        """Clear all pending messages from kernel channels."""
+        # First clear shell and control channels
+        channels = [
+            self.client.shell_channel,
+            self.client.control_channel,
+        ]
+        for channel in channels:
+            try:
+                while True:
+                    channel.get_msg(timeout=0.1)
+            except queue.Empty:
+                continue
+
+        # Then process iopub channel until we get an idle state
+        iterations = 0
+        while True:
+            try:
+                msg = self.client.iopub_channel.get_msg(timeout=0.1)
+                self.io_handler.add_message(msg)
+                if self.io_handler.is_idle(msg):
+                    break
+            except queue.Empty:
+                iterations += 1
+                if iterations > 10:
+                    logger.info("Kernel took too long to become idle")
+                    break
+
+    def iopub_listener(self, client: KernelClient) -> None:
+        while True:
+            try:
+                if (
+                    self.io_handler.user_interrupted
+                    and self.io_handler.user_interrupted()
+                ):
+                    logger.info("External halt signal received")
+                    self.manager.interrupt_kernel()
+                    self.interrupted_by_user = True
+
+                    # Wait for kernel to push any final output
+                    time.sleep(0.5)
+
+                    # Clear all channels to reset kernel state
+                    self._clear_channels()
+
+                    break
+
+                try:
+                    msg = client.iopub_channel.get_msg(timeout=1)
+                    logger.debug(f"Received message from kernel: {msg}")
+                    self.io_handler.add_message(msg)
+
+                    if self.io_handler.is_idle(msg):
+                        logger.debug("Kernel is idle.")
+                        break
+                except queue.Empty:
+                    continue
+
+            except Exception as e:
+                logger.info(f"Error getting message from kernel: {e}")
+                break
+
+    # Deprecated, use execute_code_streaming
+    async def execute_code(self, code: str) -> str:
+        async for result in self.execute_code_streaming(code):
+            if isinstance(result, PythonExecutionResult):
+                return result.output
+        # should be unreachable
+        raise Exception("No result from kernel")
+
+    async def execute_code_streaming(
+        self, code: str, user_interrupted: Callable[[], bool] | None = None
+    ) -> AsyncGenerator[StreamedOutputPiece | PythonExecutionResult, None]:
+        await self.wait_for_ready()
+        self.interrupted_by_user = False
+
+        self.io_handler = IOChannelHandler(user_interrupted=user_interrupted)
+
+        client = self.client
+        client.connect_iopub()
+        iopub_thread = threading.Thread(target=self.iopub_listener, args=(client,))
+        logger.info("Starting IO listener thread.")
+        iopub_thread.start()
+
+        logger.info("Executing code in kernel.")
+        client.execute(code)
+
+        stopping = False
+
+        results = []
+        while True:
+            stopping = not iopub_thread.is_alive()
+
+            while not self.io_handler.output_buffer.empty():
+                output = self.io_handler.output_buffer.get()
+                logger.info("Execution output: %s", output)
+                yield StreamedOutputPiece(content=output)
+                results.append(output)
+
+            if stopping:
+                break
+
+            await asyncio.sleep(0.05)
+
+        # Wait for thread to fully exit
+        iopub_thread.join(timeout=1.0)
+        yield PythonExecutionResult(
+            output="".join(results), halted=self.interrupted_by_user
+        )
+
+    def close(self) -> None:
+        if self._client:
+            self._client.stop_channels()
+            self._client = None
+        if self._manager:
+            self._manager.shutdown_kernel()
+            self._manager = None
+
+
+async def execute_python(code: str, kernel: Kernel) -> str:
+    return await kernel.execute_code(code)
+
+
+async def execute_python_streaming(
+    code: str, kernel: Kernel, user_interrupted: Callable[[], bool] | None = None
+) -> AsyncGenerator[StreamedOutputPiece | PythonExecutionResult, None]:
+    async for result in kernel.execute_code_streaming(
+        code, user_interrupted=user_interrupted
+    ):
+        yield result
+
+
+### Environment Helpers
+
+
+def get_python_env_info() -> PythonEnvInfo:
+    return PythonEnvInfo(
+        interpreter_path=get_active_python_interpreter_path(),
+        interpreter_version=get_active_python_interpreter_version(),
+    )
+
+
+def get_active_python_interpreter_path() -> str | None:
+    return sys.executable
+
+
+def get_active_python_interpreter_version() -> str | None:
+    version = sys.version
+
+    match = re.search(r"(\d+\.\d+\.\d+).*", version)
+
+    if match:
+        return match.group(1)
+
+    return None