monkeybot 2.1.1__py3-none-any.whl

monkeybot/core/messages/convert_provider.py

@@ -0,0 +1,41 @@
+"""Convert agent-facing messages into the provider-facing payload.
+
+Pipeline stage after :func:`transform_context`. Resolves attachments and applies
+optional pressure-tier tool-result shaping without mutating persisted history.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import Literal
+
+from monkeybot.core.attachments.resolve import resolve_messages_for_provider
+from monkeybot.core.attachments.store import AttachmentStore
+from monkeybot.core.context.tool_shapers import shape_messages_tool_results
+from monkeybot.core.llm.provider import Message
+
+
+def convert_to_provider(
+    messages: Sequence[Message],
+    *,
+    attachment_store: AttachmentStore | None,
+    session_id: str,
+    pressure_tier: Literal["none", "moderate", "aggressive"] = "none",
+    protect_recent: int = 6,
+) -> list[Message]:
+    """Agent messages → provider-ready messages (attachments resolved, shaped).
+
+    Persisted history is never mutated; shaping and resolution are view-only.
+    """
+    working: Sequence[Message] = messages
+    if pressure_tier in ("moderate", "aggressive"):
+        working = shape_messages_tool_results(
+            working,
+            protect_recent=protect_recent,
+            pressure_tier=pressure_tier,
+        )
+    return resolve_messages_for_provider(
+        working,
+        attachment_store=attachment_store,
+        session_id=session_id,
+    )

monkeybot/core/messages/tool_integrity.py

@@ -0,0 +1,262 @@
+"""In-memory repair for broken tool-call turns before provider replay.
+
+Repairs are applied on every harness turn after ``history.load()`` and are never
+persisted. Structured warnings are logged for Langfuse / log aggregation.
+"""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Sequence
+from typing import cast
+
+from monkeybot.core.llm.provider import Message
+from monkeybot.core.logging_utils import kv
+from monkeybot.core.types.content_blocks import (
+    ContentBlock,
+    Text,
+    ToolRequest,
+    ToolResponse,
+)
+
+logger = logging.getLogger(__name__)
+
+_SYNTHETIC_MISSING_RESULT = "Tool call interrupted or result missing"
+_UNKNOWN_TOOL = "unknown_tool"
+
+
+def _log_repair(action: str, **fields: object) -> None:
+    logger.warning("tool_integrity_repair %s", kv(action=action, **fields))
+
+
+def _tool_responses(msg: Message) -> list[ToolResponse]:
+    return [b for b in msg.content if isinstance(b, ToolResponse)]
+
+
+def _tool_requests(msg: Message) -> list[ToolRequest]:
+    return [b for b in msg.content if isinstance(b, ToolRequest)]
+
+
+def _repair_tool_request(block: ToolRequest) -> ToolRequest:
+    args = block.args
+    if isinstance(args, dict):
+        return block
+    _log_repair(
+        "malformed_stored_args",
+        call_id=block.id,
+        tool_name=block.name,
+    )
+    return ToolRequest(
+        id=block.id,
+        name=block.name,
+        args={},
+        parse_error=block.parse_error,
+        metadata=block.metadata,
+    )
+
+
+def _repair_assistant_message(msg: Message) -> Message:
+    new_blocks: list[ContentBlock] = []
+    changed = False
+    for block in msg.content:
+        if isinstance(block, ToolRequest):
+            fixed = _repair_tool_request(block)
+            if fixed is not block:
+                changed = True
+            new_blocks.append(fixed)
+        else:
+            new_blocks.append(block)
+    if not changed:
+        return msg
+    return Message(role=msg.role, content=new_blocks)
+
+
+def _tool_name_for_response(
+    block: ToolResponse,
+    request_by_id: dict[str, ToolRequest],
+) -> str:
+    name = (block.tool_name or "").strip()
+    if name:
+        return name
+    req = request_by_id.get(block.id)
+    if req is not None and req.name.strip():
+        return req.name
+    return _UNKNOWN_TOOL
+
+
+def _repair_tool_response(
+    block: ToolResponse,
+    request_by_id: dict[str, ToolRequest],
+) -> ToolResponse:
+    tool_name = _tool_name_for_response(block, request_by_id)
+    if tool_name == block.tool_name:
+        return block
+    _log_repair(
+        "empty_tool_name",
+        call_id=block.id,
+        tool_name=tool_name,
+    )
+    return ToolResponse(
+        id=block.id,
+        tool_name=tool_name,
+        result=list(block.result),
+        is_error=block.is_error,
+    )
+
+
+def _synthetic_error_response(req: ToolRequest) -> ToolResponse:
+    _log_repair(
+        "synthetic_tool_result",
+        call_id=req.id,
+        tool_name=req.name,
+    )
+    return ToolResponse(
+        id=req.id,
+        tool_name=req.name,
+        result=[Text(text=_SYNTHETIC_MISSING_RESULT)],
+        is_error=True,
+    )
+
+
+def _synthetic_tool_request(
+    block: ToolResponse,
+    request_by_id: dict[str, ToolRequest],
+) -> ToolRequest | None:
+    name = _tool_name_for_response(block, request_by_id)
+    if name == _UNKNOWN_TOOL:
+        _log_repair(
+            "skip_synthetic_tool_request",
+            call_id=block.id,
+            reason="unresolvable_tool_name",
+        )
+        return None
+    _log_repair(
+        "synthetic_tool_request",
+        call_id=block.id,
+        tool_name=name,
+    )
+    return ToolRequest(id=block.id, name=name, args={})
+
+
+def _synthetic_tool_requests(
+    blocks: Sequence[ToolResponse],
+    request_by_id: dict[str, ToolRequest],
+) -> list[ToolRequest]:
+    out: list[ToolRequest] = []
+    for block in blocks:
+        req = _synthetic_tool_request(block, request_by_id)
+        if req is not None:
+            out.append(req)
+    return out
+
+
+def repair_tool_turn_integrity(messages: Sequence[Message]) -> list[Message]:
+    """Return a copy of ``messages`` with tool-turn structural issues synthesized away."""
+    if not messages:
+        return []
+
+    out: list[Message] = []
+    i = 0
+    while i < len(messages):
+        msg = messages[i]
+
+        if msg.role == "assistant":
+            repaired = _repair_assistant_message(msg)
+            requests = _tool_requests(repaired)
+            if requests:
+                request_map = {r.id: r for r in requests}
+                request_ids = set(request_map)
+
+                if i + 1 < len(messages) and messages[i + 1].role == "user":
+                    user_msg = messages[i + 1]
+                    user_responses = _tool_responses(user_msg)
+                    orphan_responses = [b for b in user_responses if b.id not in request_ids]
+
+                    repaired_blocks: list[ContentBlock] = []
+                    user_changed = False
+                    for block in user_msg.content:
+                        if isinstance(block, ToolResponse):
+                            fixed = _repair_tool_response(block, request_map)
+                            if fixed is not block:
+                                user_changed = True
+                            repaired_blocks.append(fixed)
+                        else:
+                            repaired_blocks.append(block)
+
+                    response_ids = {b.id for b in repaired_blocks if isinstance(b, ToolResponse)}
+                    missing_ids = sorted(request_ids - response_ids)
+                    if missing_ids:
+                        for req_id in missing_ids:
+                            repaired_blocks.append(_synthetic_error_response(request_map[req_id]))
+                        user_changed = True
+
+                    repaired_user = Message(role="user", content=repaired_blocks)
+                    out.append(repaired)
+                    synthetic_orphans = _synthetic_tool_requests(orphan_responses, request_map)
+                    if synthetic_orphans:
+                        out.append(
+                            Message(
+                                role="assistant",
+                                content=cast(list[ContentBlock], synthetic_orphans),
+                            )
+                        )
+                    out.append(repaired_user if user_changed else user_msg)
+                    i += 2
+                    continue
+
+                out.append(repaired)
+                out.append(
+                    Message(
+                        role="user",
+                        content=[_synthetic_error_response(r) for r in requests],
+                    )
+                )
+                i += 1
+                continue
+
+            out.append(repaired)
+            i += 1
+            continue
+
+        if msg.role == "user":
+            user_responses = _tool_responses(msg)
+            if user_responses:
+                prior_requests = {
+                    b.id: b
+                    for m in reversed(out)
+                    if m.role == "assistant"
+                    for b in m.content
+                    if isinstance(b, ToolRequest)
+                }
+                orphan_responses = [b for b in user_responses if b.id not in prior_requests]
+                synthetic_orphans = _synthetic_tool_requests(orphan_responses, prior_requests)
+                if synthetic_orphans:
+                    out.append(
+                        Message(
+                            role="assistant",
+                            content=cast(list[ContentBlock], synthetic_orphans),
+                        )
+                    )
+
+                user_blocks: list[ContentBlock] = []
+                changed = False
+                for block in msg.content:
+                    if isinstance(block, ToolResponse):
+                        fixed = _repair_tool_response(block, prior_requests)
+                        if fixed is not block:
+                            changed = True
+                        user_blocks.append(fixed)
+                    else:
+                        user_blocks.append(block)
+
+                out.append(Message(role="user", content=user_blocks) if changed else msg)
+                i += 1
+                continue
+
+        out.append(msg)
+        i += 1
+
+    return out
+
+
+__all__ = ["repair_tool_turn_integrity"]

monkeybot/core/messages/transform_context.py

@@ -0,0 +1,84 @@
+"""Agent-facing history transforms (before provider conversion).
+
+Keeps UI-only / internal blocks out of the agent working set and repairs tool
+turn integrity for replay. Does not resolve attachments or apply pressure shaping
+— those belong in :mod:`monkeybot.core.messages.convert_provider`.
+"""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Sequence
+
+from monkeybot.core.llm.provider import Message
+from monkeybot.core.logging_utils import kv
+from monkeybot.core.messages.tool_integrity import repair_tool_turn_integrity
+from monkeybot.core.types.content_blocks import (
+    ActionRequired,
+    AttachmentDescriptor,
+    ContentBlock,
+    FrontendToolRequest,
+    SystemNotification,
+    ToolConfirmationRequest,
+)
+
+logger = logging.getLogger(__name__)
+
+# Blocks that may appear in agent/history views but must not reach providers.
+_PROVIDER_EXCLUDED_BLOCKS: tuple[type[ContentBlock], ...] = (
+    AttachmentDescriptor,
+    ToolConfirmationRequest,
+    FrontendToolRequest,
+    ActionRequired,
+    SystemNotification,
+)
+
+
+def _strip_provider_excluded(messages: Sequence[Message]) -> list[Message]:
+    out: list[Message] = []
+    for msg in messages:
+        kept = [b for b in msg.content if not isinstance(b, _PROVIDER_EXCLUDED_BLOCKS)]
+        if not kept and msg.content:
+            logger.debug(
+                "transform_context drop_ui_only_turn %s",
+                kv(action="drop_ui_only_turn", role=msg.role, blocks=len(msg.content)),
+            )
+            continue
+        if len(kept) == len(msg.content):
+            out.append(msg)
+        else:
+            out.append(Message(role=msg.role, content=kept))
+    return out
+
+
+def _coalesce_adjacent_same_role(messages: Sequence[Message]) -> list[Message]:
+    """Merge consecutive same-role messages after UI-only turns are dropped.
+
+    Dropping an interior all-UI row can leave adjacent equal roles
+    (``user → assistant → user(UI) → assistant`` → ``user → assistant → assistant``).
+    Anthropic and Gemini reject that wire shape, so coalesce content into one
+    message per run of the same role.
+    """
+    if not messages:
+        return []
+    out: list[Message] = [messages[0]]
+    for msg in messages[1:]:
+        prev = out[-1]
+        if msg.role == prev.role:
+            out[-1] = Message(role=prev.role, content=[*prev.content, *msg.content])
+        else:
+            out.append(msg)
+    return out
+
+
+def transform_context(messages: Sequence[Message]) -> list[Message]:
+    """Agent-facing history → cleaned message list for conversion.
+
+    Steps:
+    1. Repair broken tool-call / tool-result pairing (in-memory only).
+    2. Strip UI-only content blocks that must never reach a provider.
+    3. Coalesce adjacent same-role messages left by dropped UI-only turns.
+    """
+    repaired = repair_tool_turn_integrity(list(messages))
+    stripped = _strip_provider_excluded(repaired)
+    return _coalesce_adjacent_same_role(stripped)

monkeybot/core/path_safety.py

@@ -0,0 +1,11 @@
+"""Helpers for safe filesystem paths from user-supplied identifiers."""
+
+from __future__ import annotations
+
+
+def sanitize_path_component(name: str) -> str:
+    """Sanitize a string for use as a single path component (no directory separators)."""
+    return name.replace("/", "_").replace("..", "_")
+
+
+__all__ = ["sanitize_path_component"]

monkeybot/core/persistence/__init__.py

@@ -0,0 +1,17 @@
+"""SQLite and filesystem persistence for conversations and runs."""
+
+from monkeybot.core.persistence.backends import (
+    HistoryStore,
+    RunStore,
+    StorageBackend,
+    UsageStore,
+    create_storage_backend,
+)
+
+__all__ = [
+    "HistoryStore",
+    "RunStore",
+    "StorageBackend",
+    "UsageStore",
+    "create_storage_backend",
+]

monkeybot/core/persistence/backends.py

@@ -0,0 +1,236 @@
+"""Storage backend protocols and factory for monkeybot persistence.
+
+``backends.py`` is protocol-only — no implementation code is imported at module
+level. Both ``SQLiteStorageBackend`` and ``PostgresStorageBackend`` are lazy-
+imported inside :func:`create_storage_backend` so that neither SQLite nor
+Postgres implementation code loads until the factory is actually called.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Protocol, runtime_checkable
+from urllib.parse import parse_qs, unquote, urlparse
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+    from monkeybot.core.llm.provider import Message
+    from monkeybot.core.llm.usage import Usage, UsageSummary
+    from monkeybot.core.persistence.durable_runs import SubagentEnvelope, SubagentRunRow
+    from monkeybot.core.persistence.scheduled_loops import (
+        ScheduledLoopCreate,
+        ScheduledLoopRow,
+    )
+    from monkeybot.core.persistence.thread_summary import ChatThreadSummary
+
+
+@runtime_checkable
+class HistoryStore(Protocol):
+    """Protocol for conversation history persistence."""
+
+    async def load(self, thread_id: str, limit: int = 100) -> list[Message]: ...
+
+    async def append(self, thread_id: str, message: Message) -> None: ...
+
+    async def reset(self, thread_id: str, messages: list[Message]) -> None: ...
+
+    async def list_threads(self, limit: int = 50) -> list[ChatThreadSummary]: ...
+
+
+@runtime_checkable
+class UsageStore(Protocol):
+    """Protocol for per-turn token/cost accounting."""
+
+    async def record(
+        self,
+        thread_id: str,
+        model: str,
+        usage: Usage,
+        run_id: str | None = None,
+    ) -> None: ...
+
+    async def summary(
+        self,
+        thread_id: str | None = None,
+        since_ms: int | None = None,
+    ) -> UsageSummary: ...
+
+
+@runtime_checkable
+class RunStore(Protocol):
+    """Protocol for subagent run lifecycle tracking."""
+
+    async def record_pending(
+        self,
+        run_id: str,
+        parent_run_id: str | None,
+        script: str,
+        envelope: SubagentEnvelope,
+        scratch_dir: Path,
+    ) -> None: ...
+
+    async def record_started(
+        self,
+        run_id: str,
+        parent_run_id: str | None,
+        script: str,
+        envelope: SubagentEnvelope,
+        scratch_dir: Path,
+    ) -> None: ...
+
+    async def claim(self, run_id: str, worker_id: str) -> bool: ...
+
+    async def reset_stale_claims(self, stale_after_ms: int) -> int: ...
+
+    async def record_completed(self, run_id: str, result_json: str) -> None: ...
+
+    async def record_failed(self, run_id: str, error: str) -> None: ...
+
+    async def pending_runs(self) -> list[SubagentRunRow]: ...
+
+    async def get_run(self, run_id: str) -> SubagentRunRow | None: ...
+
+
+@runtime_checkable
+class ScheduledLoopStore(Protocol):
+    """Protocol for prompt-first scheduled agent loops."""
+
+    async def create(self, spec: ScheduledLoopCreate) -> ScheduledLoopRow: ...
+
+    async def get(self, loop_id: str) -> ScheduledLoopRow | None: ...
+
+    async def list_all(self) -> list[ScheduledLoopRow]: ...
+
+    async def list_due(self, now_ms: int) -> list[ScheduledLoopRow]: ...
+
+    async def claim_tick(self, loop_id: str, worker_id: str) -> ScheduledLoopRow | None: ...
+
+    async def release_stale_claims(self, stale_after_ms: int) -> int: ...
+
+    async def complete_tick(
+        self,
+        loop_id: str,
+        *,
+        worker_id: str,
+        error: str | None = None,
+    ) -> ScheduledLoopRow | None: ...
+
+    async def defer_tick(self, loop_id: str, *, worker_id: str, reason: str) -> None: ...
+
+    async def renew_tick_claim(self, loop_id: str, worker_id: str) -> bool: ...
+
+    async def pause(self, loop_id: str) -> bool: ...
+
+    async def resume(self, loop_id: str) -> bool: ...
+
+    async def stop(self, loop_id: str, *, stop_reason: str = "manual") -> bool: ...
+
+
+@runtime_checkable
+class SessionTurnLockStore(Protocol):
+    """Exclusive turn lock per session for multi-replica gateways."""
+
+    async def try_acquire(self, session_id: str, request_id: str) -> bool: ...
+
+    async def release(self, session_id: str, request_id: str) -> None: ...
+
+    async def is_busy(self, session_id: str) -> bool: ...
+
+    async def release_stale_claims(self, stale_after_ms: int) -> int: ...
+
+
+@runtime_checkable
+class StorageBackend(Protocol):
+    """Owns a storage connection lifecycle and vends all stores."""
+
+    async def open(self, *, run_schema: bool = True) -> None: ...
+
+    async def close(self) -> None: ...
+
+    def history(self) -> HistoryStore: ...
+
+    def usage(self) -> UsageStore: ...
+
+    def runs(self) -> RunStore: ...
+
+    def scheduled_loops(self) -> ScheduledLoopStore: ...
+
+    def session_turns(self) -> SessionTurnLockStore: ...
+
+
+@dataclass(frozen=True)
+class FirestoreConfig:
+    """Parsed ``firestore://`` storage URL."""
+
+    project: str
+    database: str
+    prefix: str = ""
+
+
+def _normalize_postgres_db_url(db_url: str) -> str | None:
+    """Return a ``postgresql://`` URL for Postgres backends, or ``None`` if not Postgres.
+
+    Accepts both ``postgresql://`` (SQLAlchemy/asyncpg style) and ``postgres://``
+    (common alias, e.g. Heroku-style URLs). asyncpg expects ``postgresql://``.
+    """
+    stripped = db_url.strip()
+    if stripped.startswith("postgresql://"):
+        return stripped
+    if stripped.startswith("postgres://"):
+        return "postgresql://" + stripped[len("postgres://") :]
+    return None
+
+
+def _parse_firestore_config(db_url: str) -> FirestoreConfig | None:
+    """Return Firestore settings for ``firestore://`` URLs, or ``None`` if not Firestore."""
+    stripped = db_url.strip()
+    if not stripped.startswith("firestore://"):
+        return None
+    parsed = urlparse(stripped)
+    project = unquote(parsed.hostname or "")
+    if not project:
+        raise ValueError(f"Firestore URL missing project id: {db_url!r}")
+    database = unquote(parsed.path.lstrip("/")) or "(default)"
+    prefix = parse_qs(parsed.query).get("prefix", [""])[0]
+    return FirestoreConfig(project=project, database=database, prefix=prefix)
+
+
+def create_storage_backend(db_url: str) -> StorageBackend:
+    """Factory that returns the right backend for ``db_url``.
+
+    Neither SQLite nor Postgres implementation code is imported until this
+    function is called with a matching URL scheme.
+
+    - ``sqlite://...`` → :class:`~monkeybot.core.persistence.sqlite_backend.SQLiteStorageBackend`
+      (zero extra deps beyond core)
+    - ``postgresql://...`` or ``postgres://...`` → :class:`~monkeybot.core.persistence.postgres.PostgresStorageBackend`
+      (requires ``pip install 'monkeybot[postgres]'``)
+    - ``firestore://PROJECT/DATABASE`` → :class:`~monkeybot.core.persistence.firestore.FirestoreStorageBackend`
+      (requires ``pip install 'monkeybot[firestore]'``)
+    """
+    if db_url.startswith("sqlite://"):
+        from monkeybot.core.persistence.sqlite_backend import SQLiteStorageBackend
+
+        return SQLiteStorageBackend(db_url)
+    pg_url = _normalize_postgres_db_url(db_url)
+    if pg_url is not None:
+        try:
+            from monkeybot.core.persistence.postgres import PostgresStorageBackend
+        except ImportError as exc:
+            raise RuntimeError(
+                "asyncpg is not installed. "
+                "Run: pip install 'monkeybot[postgres]'"
+            ) from exc
+        return PostgresStorageBackend(pg_url)
+    fs_config = _parse_firestore_config(db_url)
+    if fs_config is not None:
+        try:
+            from monkeybot.core.persistence.firestore import FirestoreStorageBackend
+        except ImportError as exc:
+            raise RuntimeError(
+                "google-cloud-firestore is not installed. "
+                "Run: pip install 'monkeybot[firestore]'"
+            ) from exc
+        return FirestoreStorageBackend(fs_config)
+    raise ValueError(f"Unsupported DB URL scheme: {db_url!r}")

monkeybot/core/persistence/db.py

@@ -0,0 +1,28 @@
+"""Legacy import path for SQLite persistence helpers.
+
+Prefer :mod:`monkeybot.core.persistence.sqlite` for new code. This module
+re-exports the same symbols that lived here before ``db.py`` was renamed to
+``sqlite.py``.
+"""
+
+from __future__ import annotations
+
+from monkeybot.core.persistence.sqlite import (
+    DEFAULT_DB_URL,
+    SCHEMA_DDLS,
+    _LEGACY_SCHEMA_MESSAGE,
+    apply_schema,
+    configure_connection,
+    open_connection,
+    sqlite_path_from_db_url,
+)
+
+__all__ = [
+    "DEFAULT_DB_URL",
+    "SCHEMA_DDLS",
+    "_LEGACY_SCHEMA_MESSAGE",
+    "apply_schema",
+    "configure_connection",
+    "open_connection",
+    "sqlite_path_from_db_url",
+]