crabcode 0.1.6__py3-none-win_amd64.whl

crabcode_core/types/tool.py

@@ -0,0 +1,208 @@
+"""Tool system types for CrabCode."""
+
+from __future__ import annotations
+
+import asyncio
+import base64
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import TYPE_CHECKING, Any, Callable, Coroutine
+
+from crabcode_core.types.config import DEFAULT_FILESYSTEM_TIMEOUT
+
+if TYPE_CHECKING:
+    from crabcode_core.tools.loading import ToolCatalog
+    from crabcode_core.types.message import AssistantMessage, Message
+
+
+class PermissionBehavior(str, Enum):
+    ALLOW = "allow"
+    DENY = "deny"
+    ASK = "ask"
+
+
+ToolEventCallback = Callable[[str, str, dict[str, Any]], None]
+"""Signature: (tool_name, event_type, data)"""
+
+
+# Keep this aligned with the Gateway's ImageAttachment validation limit.  The
+# limit applies to decoded bytes, not the larger Base64 representation.
+MAX_INLINE_IMAGE_BYTES = 20 * 1024 * 1024
+
+
+@dataclass
+class PermissionResult:
+    behavior: PermissionBehavior = PermissionBehavior.ALLOW
+    reason: str | None = None
+    updated_input: dict[str, Any] | None = None
+    permission_key: str | None = None
+
+
+@dataclass
+class ToolResult:
+    """Result from executing a tool."""
+
+    data: Any = None
+    result_for_model: str = ""
+    result_for_display: str | None = None
+    is_error: bool = False
+    # Inline image attachments for the conversation UI. Each entry carries
+    # ``media_type``, base64-encoded ``data``, and an optional ``description``.
+    images: list[dict[str, str]] = field(default_factory=list)
+
+
+@dataclass
+class ToolContext:
+    """Context passed to tool execution."""
+
+    cwd: str = "."
+    messages: list[Message] = field(default_factory=list)
+    abort_signal: Any | None = None
+    session_id: str = ""
+    env: dict[str, str] = field(default_factory=dict)
+    on_event: ToolEventCallback | None = None
+    tool_config: dict[str, Any] = field(default_factory=dict)
+    choice_queue: Any | None = None  # asyncio.Queue[ChoiceResponseEvent]
+    tool_event_queue: Any | None = None  # asyncio.Queue[CoreEvent] — for tools to emit events mid-execution
+    agent_id: str | None = None
+    tool_use_id: str | None = None
+    agent_depth: int = 0
+    agent_manager: Any | None = None
+    lsp_manager: Any | None = None  # LSPManager — session-scoped, set during CoreSession.initialize()
+    team_id: str | None = None  # Team ID the current agent belongs to
+    team_manager: Any | None = None  # TeamManager — session-scoped
+    schedule_manager: Any | None = None  # ScheduleManager — session-scoped
+    session: Any | None = None  # CoreSession — for checkpoint/revert operations
+    api_adapter: Any | None = None  # Active turn's provider, including subagent/model switches
+    model: str | None = None
+    tool_catalog: ToolCatalog | None = None
+    snapshot_enabled: bool = True
+    snapshot_max_size_mb: int = 1024
+    filesystem_timeout: float | None = DEFAULT_FILESYSTEM_TIMEOUT
+    # ImageContent emitted by a tool during this invocation.  This is kept on
+    # the per-call context so concurrent tools cannot leak attachments into
+    # one another.
+    emitted_images: list[dict[str, str]] = field(default_factory=list, repr=False)
+
+    def emit_image(self, data: bytes, mime_type: str, *, description: str = "") -> None:
+        """Append an inline image to this tool invocation's result.
+
+        This mirrors Codex's ``emitImage({ bytes, mimeType })`` boundary:
+        callers provide raw bytes and the transport carries a separate image
+        content block instead of embedding binary data in text or Markdown.
+        """
+        if not isinstance(data, (bytes, bytearray, memoryview)):
+            raise TypeError("image data must be bytes")
+        raw = bytes(data)
+        if not raw:
+            raise ValueError("image data must not be empty")
+        if len(raw) > MAX_INLINE_IMAGE_BYTES:
+            raise ValueError("image data exceeds the 20MB limit")
+        media_type = str(mime_type or "").strip()
+        if not media_type.lower().startswith("image/"):
+            raise ValueError("mime_type must be an image MIME type")
+        self.emitted_images.append({
+            "media_type": media_type,
+            "data": base64.b64encode(raw).decode("ascii"),
+            **({"description": description} if description else {}),
+        })
+
+
+CanUseToolFn = Callable[
+    ["Tool", dict[str, Any], "ToolContext"],
+    Coroutine[Any, Any, PermissionResult],
+]
+
+
+class Tool(ABC):
+    """Base class for all CrabCode tools."""
+
+    name: str = ""
+    description: str = ""
+    input_schema: dict[str, Any] = {}
+    is_read_only: bool = False
+    is_concurrency_safe: bool = False
+    is_enabled: bool = True
+    uses_tool_permission_policy: bool = False
+
+    _cached_prompt: str | None = None
+    _background_task: asyncio.Task[Any] | None = None
+    _setup_context: ToolContext | None = None
+
+    @abstractmethod
+    async def call(
+        self,
+        tool_input: dict[str, Any],
+        context: ToolContext,
+    ) -> ToolResult:
+        """Execute the tool with given input."""
+        ...
+
+    async def setup(self, context: ToolContext) -> None:
+        """Called once during session init. Override for heavy initialization."""
+        self._setup_context = context
+
+    async def close(self) -> None:
+        """Called during session shutdown to release resources."""
+        return None
+
+    def emit_event(self, event_type: str, data: dict[str, Any] | None = None) -> None:
+        """Emit a tool event (progress, ready, error, etc.)."""
+        if self._setup_context and self._setup_context.on_event:
+            self._setup_context.on_event(self.name, event_type, data or {})
+
+    async def wait_ready(self) -> None:
+        """Wait for background initialization to complete."""
+        if self._background_task and not self._background_task.done():
+            await self._background_task
+
+    async def get_prompt(self, **kwargs: Any) -> str:
+        """Return the tool description for the system prompt / API tools list."""
+        return self.description
+
+    async def validate_input(
+        self, tool_input: dict[str, Any]
+    ) -> str | None:
+        """Validate input. Return error string or None if valid."""
+        return None
+
+    def is_available(self, context: ToolContext) -> bool:
+        """Whether the active provider/context supports this tool."""
+        return self.is_enabled
+
+    async def check_permissions(
+        self,
+        tool_input: dict[str, Any],
+        context: ToolContext,
+    ) -> PermissionResult:
+        """Check if the tool can be used with the given input."""
+        return PermissionResult()
+
+    def get_permission_key(self, tool_input: dict[str, Any]) -> str:
+        """Return the permission key used for session-scoped allow rules."""
+        return self.name
+
+    def to_api_schema(self) -> dict[str, Any]:
+        """Convert to API tool schema for model requests.
+
+        Uses the detailed prompt from get_prompt() when available,
+        falling back to the short description.
+        """
+        return {
+            "name": self.name,
+            "description": self._cached_prompt or self.description,
+            "input_schema": self.input_schema,
+        }
+
+    async def resolve_prompt(self, **kwargs: Any) -> None:
+        """Pre-resolve the detailed prompt for use in to_api_schema().
+
+        Call this once during setup so to_api_schema() can include the
+        full tool description without being async.
+        """
+        prompt = await self.get_prompt(**kwargs)
+        if prompt and prompt != self.description:
+            self._cached_prompt = prompt
+        else:
+            self._cached_prompt = None

crabcode_core/usage.py

@@ -0,0 +1,289 @@
+"""Request-level, provider-reported token ledger and local calendar aggregation.
+
+This store deliberately does not estimate missing usage or rewrite session totals.
+Each network attempt has its own row, including retries and interrupted streams.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from contextlib import closing
+import logging
+import os
+import sqlite3
+import time
+import uuid
+from datetime import date, datetime, time as day_time, timedelta, timezone
+from pathlib import Path
+from typing import Any, AsyncIterator
+from zoneinfo import ZoneInfo
+
+from crabcode_core.api.base import APIAdapter, ModelConfig, StreamChunk, usage_int_field
+from crabcode_core.api.error_logging import log_api_error
+from crabcode_core.paths import get_config_home
+
+logger = logging.getLogger(__name__)
+recording_error: str | None = None
+
+
+class UsageStore:
+    def __init__(self, path: Path | None = None) -> None:
+        self.path = path or get_config_home() / "usage.sqlite3"
+
+    @property
+    def error_marker(self) -> Path:
+        return self.path.with_name("usage-recording-error.txt")
+
+    def mark_error(self) -> None:
+        """Make a write gap visible to another CLI/Gateway process when possible."""
+        try:
+            self.path.parent.mkdir(parents=True, exist_ok=True)
+            self.error_marker.write_text(
+                "Usage recording failed in a CrabCode process. Historical totals may be incomplete.\n",
+                encoding="utf-8",
+            )
+        except OSError:
+            logger.warning("Unable to persist usage recording failure marker", exc_info=True)
+
+    def _connect(self) -> sqlite3.Connection:
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        deadline = time.monotonic() + 3
+        while True:
+            db = sqlite3.connect(self.path, timeout=3)
+            try:
+                db.execute("PRAGMA busy_timeout=3000")
+                if db.execute("PRAGMA user_version").fetchone()[0] == 0:
+                    db.execute("PRAGMA journal_mode=WAL")
+                    db.execute("""CREATE TABLE IF NOT EXISTS requests (
+                        id TEXT PRIMARY KEY, started_ms INTEGER NOT NULL, ended_ms INTEGER,
+                        project_key TEXT, project_path TEXT, requested_model TEXT NOT NULL,
+                        model_id TEXT NOT NULL, provider TEXT NOT NULL, purpose TEXT NOT NULL,
+                        session_id TEXT, operation_id TEXT, input_tokens INTEGER, output_tokens INTEGER,
+                        reported_total_tokens INTEGER, cache_read_tokens INTEGER,
+                        cache_write_tokens INTEGER, reasoning_tokens INTEGER,
+                        usage_status TEXT NOT NULL, status TEXT NOT NULL, stop_reason TEXT
+                    )""")
+                    db.execute("CREATE INDEX IF NOT EXISTS requests_time ON requests(started_ms)")
+                    db.execute("CREATE INDEX IF NOT EXISTS requests_project_time ON requests(project_key, started_ms)")
+                    db.execute("CREATE TABLE IF NOT EXISTS metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL)")
+                    db.execute("INSERT OR REPLACE INTO metadata VALUES ('schema_version', '1')")
+                    db.execute("PRAGMA user_version=1")
+                    db.commit()
+                return db
+            except sqlite3.OperationalError as exc:
+                db.close()
+                if "locked" not in str(exc).lower() or time.monotonic() >= deadline:
+                    raise
+                time.sleep(0.05)
+            except BaseException:
+                db.close()
+                raise
+
+    @staticmethod
+    def project_key(cwd: str | None) -> str | None:
+        return os.path.normcase(os.path.normpath(os.path.abspath(cwd))) if cwd else None
+
+    def begin(self, request_id: str, started_ms: int, *, cwd: str | None,
+              model: str, provider: str, purpose: str, session_id: str | None,
+              operation_id: str | None = None) -> None:
+        with closing(self._connect()) as db, db:
+            db.execute("INSERT OR IGNORE INTO metadata VALUES ('tracking_started_ms', ?)", (str(started_ms),))
+            db.execute("""INSERT INTO requests
+                (id, started_ms, project_key, project_path, requested_model, model_id,
+                 provider, purpose, session_id, operation_id, usage_status, status)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'missing', 'running')""",
+                (request_id, started_ms, self.project_key(cwd), cwd,
+                 model, model, provider, purpose, session_id, operation_id))
+
+    def snapshot(self, request_id: str, usage: dict[str, int], model_id: str | None = None) -> None:
+        """Persist changed cumulative usage, including before a process crashes."""
+        inputs = usage.get("total_input_tokens", usage.get("input_tokens"))
+        outputs = usage.get("output_tokens")
+        total = usage.get("total_tokens")
+        status = "complete" if inputs is not None and outputs is not None else "partial" if usage else "missing"
+        with closing(self._connect()) as db, db:
+            db.execute("""UPDATE requests SET input_tokens=?, output_tokens=?,
+                reported_total_tokens=?, cache_read_tokens=?, cache_write_tokens=?,
+                reasoning_tokens=?, usage_status=?, model_id=COALESCE(?, model_id) WHERE id=?""",
+                (inputs, outputs, total, usage.get("cache_read_tokens"),
+                 usage.get("cache_write_tokens"), usage.get("reasoning_tokens"),
+                 status, model_id, request_id))
+
+    def finish(self, request_id: str, usage: dict[str, int],
+               status: str, stop_reason: str | None = None, model_id: str | None = None) -> None:
+        inputs = usage.get("total_input_tokens", usage.get("input_tokens"))
+        outputs = usage.get("output_tokens")
+        usage_status = "complete" if inputs is not None and outputs is not None else "partial" if usage else "missing"
+        with closing(self._connect()) as db, db:
+            db.execute("""UPDATE requests SET ended_ms=?, status=?, stop_reason=?,
+                input_tokens=?, output_tokens=?, reported_total_tokens=?,
+                cache_read_tokens=?, cache_write_tokens=?, reasoning_tokens=?,
+                usage_status=?, model_id=COALESCE(?, model_id) WHERE id=?""",
+                (int(time.time() * 1000), status, stop_reason, inputs, outputs,
+                 usage.get("total_tokens"), usage.get("cache_read_tokens"),
+                 usage.get("cache_write_tokens"), usage.get("reasoning_tokens"),
+                 usage_status, model_id, request_id))
+
+    def daily(self, start: date, end: date, zone: ZoneInfo,
+              cwd: str | None = None) -> dict[str, Any]:
+        if end < start or (end - start).days >= 366:
+            raise ValueError("日期范围必须为 1–366 天")
+        lower = int(datetime.combine(start, day_time.min, zone).timestamp() * 1000)
+        upper = int(datetime.combine(end + timedelta(days=1), day_time.min, zone).timestamp() * 1000)
+        with closing(self._connect()) as db, db:
+            row = db.execute("SELECT value FROM metadata WHERE key='tracking_started_ms'").fetchone()
+            tracking_start = int(row[0]) if row else None
+            args: list[Any] = [lower, upper]
+            clause = ""
+            if cwd is not None:
+                clause = " AND project_key=?"
+                args.append(self.project_key(cwd))
+            rows = db.execute("""SELECT started_ms, provider, model_id, input_tokens,
+                output_tokens, reported_total_tokens, usage_status, status
+                FROM requests WHERE started_ms>=? AND started_ms<?""" + clause,
+                args).fetchall()
+            activity_rows = db.execute("""SELECT provider, model_id, COUNT(*)
+                FROM requests WHERE 1=1""" + clause + " GROUP BY provider, model_id",
+                [self.project_key(cwd)] if cwd is not None else []).fetchall()
+            bounds = db.execute("SELECT MIN(started_ms), MAX(started_ms) FROM requests WHERE 1=1" + clause,
+                                [self.project_key(cwd)] if cwd is not None else []).fetchone()
+
+        days: dict[str, dict[str, Any]] = {}
+        cursor = start
+        while cursor <= end:
+            key = cursor.isoformat()
+            days[key] = {"date": key, "input_tokens": 0, "output_tokens": 0,
+                         "total_tokens": None, "request_count": 0,
+                         "unknown_requests": 0, "missing_requests": 0,
+                         "partial_requests": 0, "coverage": "unavailable"}
+            cursor += timedelta(days=1)
+        by_model: dict[str, dict[str, int]] = {}
+        model_totals: dict[str, int] = {}
+        for started_ms, provider, model_id, input_tokens, output_tokens, reported_total, usage_status, status in rows:
+            key = datetime.fromtimestamp(started_ms / 1000, timezone.utc).astimezone(zone).date().isoformat()
+            day = days[key]
+            day["request_count"] += 1
+            if usage_status != "complete" or status not in {"completed", "output_limit"}:
+                day["unknown_requests"] += 1
+                day["missing_requests" if usage_status == "missing" else "partial_requests"] += 1
+            inputs = input_tokens or 0
+            outputs = output_tokens or 0
+            amount = reported_total if reported_total is not None else inputs + outputs
+            day["input_tokens"] += inputs
+            day["output_tokens"] += outputs
+            day["total_tokens"] = (day["total_tokens"] or 0) + amount
+            model_key = f"{provider}/{model_id or '未知模型'}"
+            model_day = by_model.setdefault(model_key, {})
+            model_day[key] = model_day.get(key, 0) + amount
+            model_totals[model_key] = model_totals.get(model_key, 0) + amount
+        for key, day in days.items():
+            day_start = int(datetime.combine(date.fromisoformat(key), day_time.min, zone).timestamp() * 1000)
+            if day["unknown_requests"]:
+                day["coverage"] = "partial"
+            elif tracking_start is None or day_start < tracking_start:
+                day["coverage"] = "partial" if day["request_count"] else "unavailable"
+            else:
+                day["coverage"] = "complete"
+            if day["coverage"] == "complete" and day["total_tokens"] is None:
+                day["total_tokens"] = 0
+        activity_counts = {f"{provider}/{model_id or '未知模型'}": count
+                           for provider, model_id, count in activity_rows}
+        models = [{"key": model_key, "provider": model_key.split("/", 1)[0],
+                    "model_id": model_key.split("/", 1)[1], "model": model_key,
+                    "total_tokens": model_totals[model_key],
+                    "recorded_request_count": activity_counts[model_key],
+                    "points": [{"date": key, "total_tokens": totals.get(key, 0) if days[key]["coverage"] != "unavailable" else None}
+                              for key in days]}
+                  for model_key, totals in sorted(by_model.items())]
+        return {"start": start.isoformat(), "end": end.isoformat(), "timezone": str(zone),
+                "scope": "project" if cwd is not None else "global",
+                "generated_at": datetime.now(timezone.utc).isoformat(),
+                "tracking_started_at": datetime.fromtimestamp(tracking_start / 1000, timezone.utc).isoformat() if tracking_start else None,
+                "first_recorded_at": datetime.fromtimestamp(bounds[0] / 1000, timezone.utc).isoformat() if bounds[0] else None,
+                "last_recorded_at": datetime.fromtimestamp(bounds[1] / 1000, timezone.utc).isoformat() if bounds[1] else None,
+                "days": list(days.values()), "models": models,
+                "summary": {"input_tokens": sum(d["input_tokens"] for d in days.values()),
+                            "output_tokens": sum(d["output_tokens"] for d in days.values()),
+                            "total_tokens": sum(d["total_tokens"] or 0 for d in days.values()),
+                            "request_count": sum(d["request_count"] for d in days.values()),
+                            "unknown_requests": sum(d["unknown_requests"] for d in days.values()),
+                            "missing_requests": sum(d["missing_requests"] for d in days.values()),
+                            "partial_requests": sum(d["partial_requests"] for d in days.values())}}
+
+
+async def tracked_stream_message(
+    adapter: APIAdapter, *, messages: Any, system: list[str], tools: list[dict[str, Any]],
+    config: ModelConfig, cwd: str | None = None, session_id: str | None = None,
+    purpose: str = "conversation", operation_id: str | None = None,
+) -> AsyncIterator[StreamChunk]:
+    """Yield unchanged provider chunks; persist one row per attempt on close."""
+    global recording_error
+    store = UsageStore()
+    request_id = uuid.uuid4().hex
+    provider = str(getattr(getattr(adapter, "config", None), "provider", None) or type(adapter).__name__)
+    recorded = False
+    try:
+        await asyncio.to_thread(store.begin, request_id, int(time.time() * 1000),
+                                cwd=cwd, model=config.model, provider=provider,
+                                purpose=purpose, session_id=session_id,
+                                operation_id=operation_id)
+        recorded = True
+    except Exception:
+        recording_error = "使用记录数据库无法写入；部分请求可能未被统计"
+        store.mark_error()
+        logger.warning("Unable to start usage record", exc_info=True)
+    usage: dict[str, int] = {}
+    outcome = "interrupted"
+    stop_reason: str | None = None
+    actual_model: str | None = None
+    source = None
+    try:
+        source = adapter.stream_message(messages=messages, system=system, tools=tools, config=config)
+        async for chunk in source:
+            if chunk.model_id:
+                actual_model = chunk.model_id
+            if chunk.usage:
+                changed = False
+                for key in ("total_input_tokens", "input_tokens", "output_tokens", "total_tokens",
+                            "cache_read_tokens", "cache_write_tokens", "reasoning_tokens"):
+                    value, present = usage_int_field(chunk.usage, key)
+                    if present and (key not in usage or value > usage[key]):
+                        usage[key] = value
+                        changed = True
+                if changed and recorded:
+                    try:
+                        await asyncio.to_thread(store.snapshot, request_id, usage.copy(), actual_model)
+                    except Exception:
+                        recording_error = "使用记录数据库无法写入；部分请求可能未被统计"
+                        store.mark_error()
+                        logger.warning("Unable to save usage snapshot", exc_info=True)
+            if chunk.stop_reason:
+                stop_reason = chunk.stop_reason
+            if chunk.type == "error":
+                outcome = "error"
+                log_api_error(adapter, session_id=session_id, model=config.model,
+                              attempt_id=request_id, purpose=purpose, chunk=chunk)
+            yield chunk
+        if outcome != "error":
+            outcome = "output_limit" if stop_reason in {"max_tokens", "max_output_tokens", "length"} else "completed"
+    except (GeneratorExit, asyncio.CancelledError):
+        outcome = "interrupted"
+        raise
+    except BaseException as exc:
+        outcome = "error"
+        if isinstance(exc, Exception):
+            log_api_error(adapter, session_id=session_id, model=config.model,
+                          attempt_id=request_id, purpose=purpose, exception=exc)
+        raise
+    finally:
+        try:
+            if source is not None:
+                await source.aclose()
+        finally:
+            if recorded:
+                try:
+                    await asyncio.to_thread(store.finish, request_id, usage.copy(), outcome, stop_reason, actual_model)
+                except Exception:
+                    recording_error = "使用记录数据库无法写入；部分请求可能未被统计"
+                    store.mark_error()
+                    logger.warning("Unable to finish usage record", exc_info=True)

crabcode_core/utf8_sanitize.py

@@ -0,0 +1,29 @@
+"""Normalize text for UTF-8 I/O (JSON, HTTP, terminals).
+
+Python allows lone surrogate code units in ``str``; strict UTF-8 encoders
+(API clients, ``sys.stdout`` on UTF-8, JSON lines) reject them.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+def safe_utf8_str(s: str) -> str:
+    """Replace lone surrogates so the string encodes as UTF-8."""
+    if not s:
+        return s
+    return s.encode("utf-8", errors="surrogatepass").decode("utf-8", errors="replace")
+
+
+def safe_utf8_json_tree(obj: Any) -> Any:
+    """Recursively sanitize strings for JSON or API payloads."""
+    if isinstance(obj, str):
+        return safe_utf8_str(obj)
+    if isinstance(obj, dict):
+        return {k: safe_utf8_json_tree(v) for k, v in obj.items()}
+    if isinstance(obj, list):
+        return [safe_utf8_json_tree(v) for v in obj]
+    if isinstance(obj, tuple):
+        return tuple(safe_utf8_json_tree(v) for v in obj)
+    return obj

crabcode_debugger/__init__.py

@@ -0,0 +1,5 @@
+"""CrabCode debugger tools."""
+
+from crabcode_debugger.tool import DebuggerTool, ProcessDebuggerTool
+
+__all__ = ["DebuggerTool", "ProcessDebuggerTool"]