crabcode 0.1.6__py3-none-win_amd64.whl

crabcode_core/io_worker.py

@@ -0,0 +1,211 @@
+"""Bounded process isolation for filesystem calls that may never return.
+
+Neither process creation nor pipe I/O runs on the event loop or its default
+executor. A daemon supervisor owns each child until it is reaped. Cancelled
+children still consume a slot: a broken mount must not produce unlimited
+processes, threads, or open pipes. SIGKILL is a request, not a successful reap.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from dataclasses import asdict
+import json
+import math
+import os
+import signal
+import subprocess
+import sys
+import threading
+from typing import Any, Callable
+
+from crabcode_core.subprocess_utils import managed_process_command, subprocess_group_options
+from crabcode_core.types.config import DEFAULT_FILESYSTEM_TIMEOUT
+
+_SLOTS = threading.BoundedSemaphore(8)
+_ACTIVE_GUARD = threading.Lock()
+_ACTIVE_ABORTS: set[Callable[[], None]] = set()
+IO_TIMEOUT = DEFAULT_FILESYSTEM_TIMEOUT
+
+# Do not resolve/stat package paths in the caller. Imports and path resolution
+# happen in the child, including for editable installs on network storage.
+_BOOTSTRAP = """
+import sys
+sys.path[:0] = sys.argv[1:]
+from crabcode_core.io_worker import _worker_main
+_worker_main()
+"""
+
+
+def _kill(process: subprocess.Popen[bytes]) -> None:
+    if process.returncode is not None:
+        return
+    try:
+        if os.name == "nt":
+            process.kill()  # The Windows supervisor owns a kill-on-close job.
+        else:
+            os.killpg(process.pid, signal.SIGKILL)
+    except ProcessLookupError:
+        pass
+
+
+def abort_io_workers() -> None:
+    """Request worker termination during emergency exit, without waiting."""
+    with _ACTIVE_GUARD:
+        callbacks = tuple(_ACTIVE_ABORTS)
+    for abort in callbacks:
+        abort()
+
+
+async def run_io(module: str, function: str, *args: Any,
+                 timeout: float | None = IO_TIMEOUT, **kwargs: Any) -> Any:
+    """Call an internal function in an expendable process; None disables the deadline."""
+    slots = _SLOTS
+    if not slots.acquire(blocking=False):
+        raise OSError("Filesystem workers are busy or still blocked; stop retrying until storage recovers")
+    loop = asyncio.get_running_loop()
+    result = loop.create_future()
+    guard = threading.Lock()
+    abandoned = False
+    process: subprocess.Popen[bytes] | None = None
+
+    def abort() -> None:
+        nonlocal abandoned
+        with guard:
+            abandoned = True
+            child = process
+        if child is not None:
+            _kill(child)
+
+    def deliver(value: Any, error: BaseException | None) -> None:
+        if not result.done():
+            if error is None:
+                result.set_result(value)
+            else:
+                result.set_exception(error)
+
+    def supervise() -> None:
+        nonlocal process
+        value, error = None, None
+        child = None
+        try:
+            payload = json.dumps([module, function, args, kwargs, os.getcwd()]).encode("utf-8")
+            # Preserve editable-install search roots, but never search the
+            # target workspace implicitly via an empty sys.path entry.
+            roots = [p for p in sys.path if p and os.path.isabs(p)]
+            child = subprocess.Popen(
+                managed_process_command([sys.executable, "-I", "-c", _BOOTSTRAP, *roots]),
+                stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                cwd=os.path.abspath(os.sep), **subprocess_group_options(),
+            )
+            with guard:
+                process = child
+                cancelled = abandoned
+            if cancelled:
+                _kill(child)
+                payload = None  # Never begin an operation after its caller left.
+            stdout, stderr = child.communicate(payload)
+            if child.returncode:
+                raise OSError(f"Filesystem worker failed ({child.returncode}): "
+                              + stderr.decode("utf-8", errors="replace")[-2000:])
+            reply = json.loads(stdout)
+            if "error" in reply:
+                raise OSError(reply["error"])
+            value = reply["result"]
+        except BaseException as exc:
+            error = exc
+        finally:
+            # Even exceptional pipe handling must not release a live child slot.
+            if child is not None:
+                try:
+                    if child.poll() is None:
+                        _kill(child)
+                    child.wait()  # Only this daemon thread can wait indefinitely.
+                finally:
+                    for stream in (child.stdin, child.stdout, child.stderr):
+                        if stream is not None:
+                            stream.close()
+            with _ACTIVE_GUARD:
+                _ACTIVE_ABORTS.discard(abort)
+            slots.release()
+            try:
+                loop.call_soon_threadsafe(deliver, value, error)
+            except RuntimeError:
+                pass  # Caller and its event loop have already exited.
+
+    try:
+        with _ACTIVE_GUARD:
+            _ACTIVE_ABORTS.add(abort)
+        threading.Thread(target=supervise, name="crabcode-fs-worker", daemon=True).start()
+    except BaseException:
+        with _ACTIVE_GUARD:
+            _ACTIVE_ABORTS.discard(abort)
+        slots.release()
+        raise
+    try:
+        return await asyncio.wait_for(result, timeout=timeout)
+    finally:
+        if not result.done() or result.cancelled():
+            abort()
+
+
+async def run_file_tool(name: str, tool_input: dict[str, Any], context: Any) -> Any:
+    from crabcode_core.types.tool import ToolResult
+
+    # All session objects/callbacks stay in the parent. Snapshot metadata is
+    # explicit; filesystem changes and their backups execute in the same child.
+    fields = {key: getattr(context, key) for key in (
+        "cwd", "session_id", "snapshot_enabled", "snapshot_max_size_mb", "env",
+        "filesystem_timeout",
+    )}
+    timeout = context.filesystem_timeout
+    if name == "Bash" and "timeout" in tool_input:
+        try:
+            timeout = float(tool_input["timeout"])
+            if not math.isfinite(timeout) or timeout <= 0:
+                raise ValueError
+        except (ValueError, TypeError):
+            return ToolResult(result_for_model="Error: timeout must be a positive number of seconds",
+                              is_error=True)
+    try:
+        data = await run_io(__name__, "_call_file_tool", name, tool_input, fields,
+                            timeout=timeout)
+        return ToolResult(**data)
+    except (OSError, asyncio.TimeoutError) as exc:
+        detail = str(exc) or (
+            f"timed out after {timeout:g}s" if timeout is not None else type(exc).__name__
+        )
+        if name in {"apply_patch", "Edit", "Write", "Bash"}:
+            detail += ". Write outcome is unknown; inspect affected files and snapshots before retrying"
+        return ToolResult(result_for_model=f"{name} filesystem operation failed: {detail}",
+                          is_error=True)
+
+
+async def _call_file_tool(name: str, tool_input: dict[str, Any], fields: dict[str, Any]) -> dict:
+    from importlib import import_module
+    from crabcode_core.types.tool import ToolContext
+
+    modules = {"Read": ("file_read", "FileReadTool"), "Write": ("file_write", "FileWriteTool"),
+               "apply_patch": ("apply_patch", "ApplyPatchTool"),
+               "Edit": ("file_edit", "FileEditTool"), "Glob": ("glob", "GlobTool"),
+               "Grep": ("grep", "GrepTool"), "Bash": ("bash", "BashTool"),
+               "Image": ("image", "ImageTool")}
+    module, cls = modules[name]
+    tool = getattr(import_module(f"crabcode_core.tools.{module}"), cls)()
+    return asdict(await tool._call_local(tool_input, ToolContext(**fields)))
+
+
+def _worker_main() -> None:
+    import importlib
+    import inspect
+
+    try:
+        module, function, args, kwargs, caller_cwd = json.load(sys.stdin)
+        os.chdir(caller_cwd)
+        value = getattr(importlib.import_module(module), function)(*args, **kwargs)
+        if inspect.isawaitable(value):
+            value = asyncio.run(value)
+        reply = {"result": value}
+    except Exception as exc:
+        reply = {"error": f"{type(exc).__name__}: {exc}"}
+    sys.stdout.write(json.dumps(reply, ensure_ascii=True))

crabcode_core/logging_utils.py

@@ -0,0 +1,168 @@
+"""Shared runtime logging configuration for CrabCode."""
+
+from __future__ import annotations
+
+import json
+import logging
+import copy
+import queue
+import threading
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from crabcode_core.paths import get_config_home
+
+if TYPE_CHECKING:
+    from crabcode_core.types.config import LoggingSettings
+
+
+LOG_NAMESPACE = "crabcode"
+LOG_KEY = "crabcode"
+LOG_FILE_NAME = "crabcode.log"
+LOG_INDEX_NAME = "index.json"
+LOGS_DIR_NAME = ".crabcode/logs"
+
+_configured_signature: tuple[str, str, str] | None = None
+
+
+class _AsyncFileHandler(logging.Handler):
+    """Keep disk writes and traceback source lookups off application threads.
+
+    A stalled sink has bounded memory and a daemon owner. It must never hold
+    the logging handler lock or make logging.shutdown wait for remote storage.
+    """
+
+    def __init__(self, path: Path) -> None:
+        super().__init__()
+        self._path = path
+        self._records: queue.Queue[logging.LogRecord | None] = queue.Queue(maxsize=1024)
+        self._stopped = threading.Event()
+        self._writer = threading.Thread(target=self._write, name="crabcode-log-writer", daemon=True)
+        self._writer.start()
+
+    def emit(self, record: logging.LogRecord) -> None:
+        if self._stopped.is_set():
+            return
+        try:
+            # Formatting exceptions can read source files; do it in the writer.
+            self._records.put_nowait(copy.copy(record))
+        except queue.Full:
+            pass
+
+    def _write(self) -> None:
+        try:
+            with open(self._path, "a", encoding="utf-8") as stream:
+                while True:
+                    try:
+                        record = self._records.get(timeout=0.1)
+                    except queue.Empty:
+                        if self._stopped.is_set():
+                            break
+                        continue
+                    if record is None:
+                        break
+                    stream.write(self.format(record) + "\n")
+                    stream.flush()
+        except Exception:
+            # Do not recursively log a failed/full logging sink.
+            self._stopped.set()
+
+    def close(self) -> None:
+        self._stopped.set()
+        try:
+            self._records.put_nowait(None)
+        except queue.Full:
+            pass
+        if threading.current_thread() is not self._writer:
+            self._writer.join(timeout=0.2)
+        super().close()
+
+
+def get_logger(name: str | None = None) -> logging.Logger:
+    """Return a logger under the shared CrabCode namespace."""
+    if not name:
+        return logging.getLogger(LOG_NAMESPACE)
+    if name.startswith(f"{LOG_NAMESPACE}."):
+        return logging.getLogger(name)
+    return logging.getLogger(f"{LOG_NAMESPACE}.{name}")
+
+
+def get_logs_dir(cwd: str) -> Path:
+    """Return the per-project logs directory."""
+    logs_dir = Path(cwd).resolve() / LOGS_DIR_NAME
+    try:
+        logs_dir.mkdir(parents=True, exist_ok=True)
+    except OSError:
+        # cwd may be read-only (e.g. "/" on macOS) — fall back to config home
+        logs_dir = get_config_home() / "logs"
+        logs_dir.mkdir(parents=True, exist_ok=True)
+    return logs_dir
+
+
+def get_log_path(cwd: str, settings: LoggingSettings | None = None) -> Path:
+    """Resolve the main CrabCode log path."""
+    if settings and settings.file:
+        configured = Path(settings.file)
+        if not configured.is_absolute():
+            configured = Path(cwd).resolve() / configured
+        try:
+            configured.parent.mkdir(parents=True, exist_ok=True)
+        except OSError:
+            configured = get_config_home() / "logs" / configured.name
+            configured.parent.mkdir(parents=True, exist_ok=True)
+        return configured
+    return get_logs_dir(cwd) / LOG_FILE_NAME
+
+
+def configure_logging(cwd: str, settings: LoggingSettings | None = None) -> Path:
+    """Configure the shared CrabCode logger to write to a project log file."""
+    global _configured_signature
+
+    from crabcode_core.types.config import LoggingSettings
+
+    logging_settings = settings or LoggingSettings()
+    level_name = logging_settings.level.upper()
+    log_path = get_log_path(cwd, logging_settings).resolve()
+    signature = (str(Path(cwd).resolve()), level_name, str(log_path))
+    logger = logging.getLogger(LOG_NAMESPACE)
+
+    if _configured_signature == signature and logger.handlers:
+        _register_log(cwd, LOG_KEY, log_path)
+        return log_path
+
+    for old_handler in logger.handlers[:]:
+        logger.removeHandler(old_handler)
+        old_handler.close()
+    logger.setLevel(getattr(logging, level_name, logging.WARNING))
+    logger.propagate = False
+
+    handler = _AsyncFileHandler(log_path)
+    handler.setLevel(logger.level)
+    handler.setFormatter(
+        logging.Formatter(
+            fmt="%(asctime)s %(levelname)s %(name)s %(message)s",
+            datefmt="%Y-%m-%d %H:%M:%S",
+        )
+    )
+    logger.addHandler(handler)
+
+    _configured_signature = signature
+    _register_log(cwd, LOG_KEY, log_path)
+    logger.debug("Logging configured: level=%s path=%s", level_name, log_path)
+    return log_path
+
+
+def _register_log(cwd: str, key: str, log_path: Path) -> None:
+    """Register a log file so the CLI /logs command can discover it."""
+    index_path = get_logs_dir(cwd) / LOG_INDEX_NAME
+    try:
+        data = json.loads(index_path.read_text(encoding="utf-8")) if index_path.exists() else {}
+    except (OSError, json.JSONDecodeError):
+        data = {}
+    if not isinstance(data, dict):
+        data = {}
+    data[key] = str(log_path.resolve())
+    index_path.write_text(
+        json.dumps(data, ensure_ascii=False, indent=2),
+        encoding="utf-8",
+    )

crabcode_core/lsp/__init__.py

@@ -0,0 +1,6 @@
+"""LSP (Language Server Protocol) client support."""
+
+from crabcode_core.lsp.client import LSPClient
+from crabcode_core.lsp.manager import LSPManager
+
+__all__ = ["LSPClient", "LSPManager"]