agentsec-eval 0.9.1__py3-none-any.whl

agentsec/__init__.py

@@ -0,0 +1 @@
+"""AgentSec — Security assessment framework for AI agents."""

agentsec/adapters/__init__.py

@@ -0,0 +1,4 @@
+from .base import AgentAdapter, AgentResponse
+from .registry import available, get, register
+
+__all__ = ["AgentAdapter", "AgentResponse", "available", "get", "register"]

agentsec/adapters/base.py

@@ -0,0 +1,38 @@
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+
+
+@dataclass
+class AgentResponse:
+    """Response from a target agent.
+
+    Adapters MUST NOT raise on non-2xx HTTP; populate `status_code` / `error`
+    and let the judge / assertions decide pass/fail (spec §4, OE-AUD2-002).
+
+    `tool_events` carries adapter-extracted tool invocations in a uniform
+    `[{"tool": str, "args": str}, ...]` shape so `tool_event_not_invoked`
+    can fire on whatever the underlying agent reports — OpenAI-compatible
+    `choices[].message.tool_calls` for `OpenClawGatewayAdapter`, future
+    SSH/native-audit observers for Stage D adapters.
+    """
+    content: str
+    raw: dict = field(default_factory=dict)
+    latency_ms: float = 0.0
+    status_code: int | None = None
+    headers: dict[str, str] = field(default_factory=dict)
+    request_url: str | None = None
+    error: str | None = None
+    tool_events: list[dict] = field(default_factory=list)
+
+
+class AgentAdapter(ABC):
+    """Base interface every target-agent adapter must implement."""
+
+    name: str = "unnamed"
+
+    @abstractmethod
+    async def send(self, message: str, session_id: str) -> AgentResponse:
+        """Send a message to the agent and return its response."""
+
+    async def reset(self, session_id: str) -> None:
+        """Reset / clear the agent's session state. Override if supported."""

agentsec/adapters/http.py

@@ -0,0 +1,101 @@
+"""Generic HTTP adapter — covers agents that expose a simple chat REST API."""
+
+import time
+from typing import Callable
+
+import httpx
+
+from .base import AgentAdapter, AgentResponse
+
+
+class HttpAgentAdapter(AgentAdapter):
+    """
+    Adapter for agents that expose a JSON-over-HTTP chat endpoint.
+
+    Default contract:
+      POST {base_url}{path}
+      Body: {"session_id": "...", "message": "..."}
+      Response: {"reply": "..."}
+
+    Subclasses override `_build_request` / `_parse_response` for non-standard
+    schemas. The `path` parameter (default `/chat`) lets the OpenClaw gateway
+    adapter target `/v1/chat/completions` without rewriting `send` (spec §5.2).
+
+    Per OE-AUD2-002: 4xx/5xx do NOT raise; status_code/error are populated
+    and the judge decides whether the response is a failure.
+    """
+
+    def __init__(
+        self,
+        name: str,
+        base_url: str,
+        path: str = "/chat",
+        headers: dict[str, str] | None = None,
+        timeout: float = 30.0,
+    ):
+        self.name = name
+        self._base_url = base_url.rstrip("/")
+        self._path = path if path.startswith("/") else "/" + path
+        self._headers = headers or {}
+        self._timeout = timeout
+        self._client_factory: Callable[[], httpx.AsyncClient] | None = None
+
+    def _build_request(self, message: str, session_id: str) -> dict:
+        return {"session_id": session_id, "message": message}
+
+    def _parse_response(self, data: dict) -> str:
+        return data.get("reply") or data.get("content") or data.get("message") or str(data)
+
+    def _parse_tool_events(self, data: dict) -> list[dict]:
+        """Subclass hook: extract tool invocations from the parsed JSON body
+        into the uniform `[{"tool": str, "args": str}, ...]` shape consumed by
+        `tool_event_not_invoked`. Default returns `[]` since the generic body
+        schema is content-only."""
+        return []
+
+    def _make_client(self) -> httpx.AsyncClient:
+        if self._client_factory is not None:
+            return self._client_factory()
+        return httpx.AsyncClient(timeout=self._timeout)
+
+    async def send(self, message: str, session_id: str) -> AgentResponse:
+        url = f"{self._base_url}{self._path}"
+        start = time.monotonic()
+        status: int | None = None
+        headers: dict[str, str] = {}
+        data: dict = {}
+        content: str = ""
+        error: str | None = None
+        tool_events: list[dict] = []
+        try:
+            async with self._make_client() as client:
+                resp = await client.post(
+                    url,
+                    json=self._build_request(message, session_id),
+                    headers=self._headers or None,
+                )
+                status = resp.status_code
+                headers = dict(resp.headers)
+                if 200 <= resp.status_code < 300:
+                    try:
+                        data = resp.json()
+                        content = self._parse_response(data)
+                        tool_events = self._parse_tool_events(data)
+                    except ValueError as exc:
+                        error = f"non-JSON 2xx body: {exc}"
+                        content = resp.text
+                else:
+                    error = f"HTTP {resp.status_code}: {resp.text[:200]}"
+                    content = resp.text
+        except httpx.HTTPError as exc:
+            error = f"{type(exc).__name__}: {exc}"
+        return AgentResponse(
+            content=content,
+            raw=data,
+            latency_ms=(time.monotonic() - start) * 1000,
+            status_code=status,
+            headers=headers,
+            request_url=url,
+            error=error,
+            tool_events=tool_events,
+        )

agentsec/adapters/openclaw_gateway.py

@@ -0,0 +1,70 @@
+"""OpenClaw `/v1/chat/completions` adapter (spec §5.2).
+
+OpenClaw exposes an OpenAI-compatible chat-completions endpoint. The body shape
+differs from `HttpAgentAdapter`'s default `{"session_id", "message"}` schema, so
+we override `_build_request` and `_parse_response` while reusing the transport
+and 4xx/5xx handling from the base class (OE-AUD2-002).
+"""
+
+from .http import HttpAgentAdapter
+
+
+class OpenClawGatewayAdapter(HttpAgentAdapter):
+    def __init__(
+        self,
+        name: str,
+        base_url: str,
+        headers: dict[str, str] | None = None,
+        timeout: float = 30.0,
+    ):
+        super().__init__(
+            name=name,
+            base_url=base_url,
+            path="/v1/chat/completions",
+            headers=headers,
+            timeout=timeout,
+        )
+
+    def _build_request(self, message: str, session_id: str) -> dict:
+        return {
+            "model": "openclaw",
+            "messages": [{"role": "user", "content": message}],
+            "stream": False,
+            "metadata": {"session_id": session_id},
+        }
+
+    def _parse_response(self, data: dict) -> str:
+        try:
+            return data["choices"][0]["message"]["content"]
+        except (KeyError, IndexError, TypeError):
+            return str(data)
+
+    def _parse_tool_events(self, data: dict) -> list[dict]:
+        """Extract OpenAI-style `choices[0].message.tool_calls[*]` into the
+        uniform `[{"tool": fn.name, "args": fn.arguments}, ...]` shape. Per
+        OE-AUD2-002 a malformed body must not raise — unparseable entries are
+        silently skipped so `tool_event_not_invoked` only sees real, named
+        invocations."""
+        try:
+            calls = data["choices"][0]["message"].get("tool_calls") or []
+        except (KeyError, IndexError, TypeError):
+            return []
+        events: list[dict] = []
+        for call in calls:
+            if not isinstance(call, dict):
+                continue
+            fn = call.get("function")
+            if not isinstance(fn, dict):
+                continue
+            name = fn.get("name")
+            if not name:
+                continue
+            args = fn.get("arguments")
+            events.append({
+                "tool": name,
+                # OpenAI emits arguments as a JSON-encoded string; pass it
+                # through verbatim so `with_args_pattern` can regex against
+                # the raw payload without round-tripping through json.
+                "args": args if isinstance(args, str) else (args or ""),
+            })
+        return events

agentsec/adapters/registry.py

@@ -0,0 +1,43 @@
+"""Adapter registry — resolves CLI/config `adapter` strings to factory callables.
+
+Why: keeping construction in a registry means new adapters need no CLI edits and
+can be tested in isolation. Spec §5.2.
+"""
+
+from collections.abc import Callable
+
+from .base import AgentAdapter
+
+AdapterFactory = Callable[..., AgentAdapter]
+
+_REGISTRY: dict[str, AdapterFactory] = {}
+
+
+def register(name: str, factory: AdapterFactory) -> None:
+    if name in _REGISTRY:
+        raise ValueError(f"adapter {name!r} is already registered")
+    _REGISTRY[name] = factory
+
+
+def get(name: str) -> AdapterFactory:
+    if name not in _REGISTRY:
+        raise KeyError(
+            f"unknown adapter {name!r}; available: {sorted(_REGISTRY)}"
+        )
+    return _REGISTRY[name]
+
+
+def available() -> list[str]:
+    return sorted(_REGISTRY)
+
+
+def _register_builtins() -> None:
+    # Imports kept local to avoid circular imports at module load.
+    from .http import HttpAgentAdapter
+    from .openclaw_gateway import OpenClawGatewayAdapter
+
+    register("http", HttpAgentAdapter)
+    register("openclaw-gateway", OpenClawGatewayAdapter)
+
+
+_register_builtins()

agentsec/adapters/ws_adapter.py

@@ -0,0 +1,122 @@
+"""WebSocket agent adapter — sends messages and listens for responses over WS.
+
+Session correlation: the session_id is appended as a query parameter so the
+server can tie the WS connection to the same logical session as HTTP calls.
+
+MockWebSocketAdapter overrides send() and listen() without importing websockets,
+so tests run without a real WebSocket server.
+"""
+
+import asyncio
+import time
+
+from .base import AgentAdapter, AgentResponse
+
+
+class WebSocketAgentAdapter(AgentAdapter):
+    """Adapter that communicates with an agent over WebSocket.
+
+    URL convention: ws_url?session_id=<session_id>
+    Send payload:   {"message": "<text>"}
+    Receive:        first text frame is the agent reply
+    """
+
+    name = "websocket"
+
+    def __init__(
+        self,
+        ws_url: str,
+        extra_headers: dict[str, str] | None = None,
+        connect_timeout: float = 10.0,
+        recv_timeout: float = 30.0,
+    ):
+        self._ws_url = ws_url.rstrip("/")
+        self._extra_headers = extra_headers or {}
+        self._connect_timeout = connect_timeout
+        self._recv_timeout = recv_timeout
+
+    def _session_url(self, session_id: str) -> str:
+        sep = "&" if "?" in self._ws_url else "?"
+        return f"{self._ws_url}{sep}session_id={session_id}"
+
+    async def send(self, message: str, session_id: str) -> AgentResponse:
+        import json
+
+        import websockets  # lazy import — only needed for real connections
+
+        url = self._session_url(session_id)
+        start = time.monotonic()
+        try:
+            async with asyncio.timeout(self._connect_timeout + self._recv_timeout):
+                async with websockets.connect(url, additional_headers=self._extra_headers) as ws:
+                    await ws.send(json.dumps({"message": message}))
+                    raw_text = await asyncio.wait_for(ws.recv(), timeout=self._recv_timeout)
+        except TimeoutError:
+            return AgentResponse(content="", error="ws_timeout",
+                                 latency_ms=(time.monotonic() - start) * 1000)
+        except Exception as exc:  # noqa: BLE001
+            return AgentResponse(content="", error=f"ws_connect_failed: {exc}",
+                                 latency_ms=(time.monotonic() - start) * 1000)
+
+        return AgentResponse(
+            content=raw_text if isinstance(raw_text, str) else raw_text.decode(),
+            latency_ms=(time.monotonic() - start) * 1000,
+        )
+
+    async def listen(self, session_id: str, duration_s: float) -> str:
+        """Connect and collect all incoming WS messages for `duration_s` seconds."""
+        import websockets  # lazy import
+
+        url = self._session_url(session_id)
+        messages: list[str] = []
+        try:
+            async with asyncio.timeout(duration_s + self._connect_timeout):
+                async with websockets.connect(url, additional_headers=self._extra_headers) as ws:
+                    loop = asyncio.get_running_loop()
+                    deadline = loop.time() + duration_s
+                    while True:
+                        remaining = deadline - loop.time()
+                        if remaining <= 0:
+                            break
+                        try:
+                            frame = await asyncio.wait_for(ws.recv(), timeout=remaining)
+                            messages.append(frame if isinstance(frame, str) else frame.decode())
+                        except TimeoutError:
+                            break
+        except Exception:  # noqa: BLE001 — connection errors produce empty output
+            pass
+        return "\n".join(messages)
+
+
+class MockWebSocketAdapter(WebSocketAgentAdapter):
+    """Test double — no real WebSocket connection.
+
+    Usage:
+        mock = MockWebSocketAdapter(send_reply="Safe response", listen_data="no leak")
+        result = await mock.send("attack", "session-1")
+        assert result.content == "Safe response"
+    """
+
+    def __init__(
+        self,
+        send_reply: str = "",
+        listen_data: str = "",
+        send_error: str | None = None,
+    ):
+        super().__init__(ws_url="ws://mock.invalid")
+        self._send_reply = send_reply
+        self._listen_data = listen_data
+        self._send_error = send_error
+        self.send_calls: list[tuple[str, str]] = []
+        self.listen_calls: list[tuple[str, float]] = []
+
+    async def send(self, message: str, session_id: str) -> AgentResponse:
+        self.send_calls.append((message, session_id))
+        return AgentResponse(
+            content=self._send_reply,
+            error=self._send_error,
+        )
+
+    async def listen(self, session_id: str, duration_s: float) -> str:
+        self.listen_calls.append((session_id, duration_s))
+        return self._listen_data

agentsec/audit/__init__.py

@@ -0,0 +1,7 @@
+"""Audit module — SSH policy, redaction, server-audit checks."""
+
+from .findings import Finding, Severity
+from .redactor import Redactor
+from .types import RunResult
+
+__all__ = ["Finding", "Redactor", "RunResult", "Severity"]

agentsec/audit/args_schema.py

@@ -0,0 +1,134 @@
+"""Per-command args-schema validation (spec §6.1).
+
+Three role types in args_schema entries:
+  - "flags": list of allowed/forbidden short/long flags. Flags listed in
+            ``value_taking`` consume the next argv token as their value
+            (e.g. ``-n 5`` for ``head``); without ``value_taking`` the
+            value falls through to the next schema entry and gets matched
+            as a positional path.
+  - "positional": one positional arg with kind={path, pattern}
+  - "predicate_pairs": (used by find) -name VALUE / -mtime VALUE pairs
+
+Plus an alternative top-level form `allowed_argv: list[list[str]]` for
+commands whose entire invocation is enumerated explicitly (uname, ps, etc.).
+
+When a schema entry has must_match_allowed_paths=True, the corresponding
+token is checked through the PathMatcher.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from .path_matcher import PathMatcher
+
+
+def validate_args(
+    command: str,
+    argv: list[str],
+    schema: dict[str, Any],
+    matcher: PathMatcher | None,
+) -> str | None:
+    """Return None if argv satisfies the schema; else return a reason string."""
+    if not schema:
+        return f"command {command!r} not in policy"
+
+    if "allowed_argv" in schema:
+        return _check_allowed_argv(command, argv, schema["allowed_argv"], matcher)
+
+    if "args_schema" in schema:
+        return _check_args_schema(command, argv, schema["args_schema"], matcher)
+
+    return f"policy entry for {command!r} has neither allowed_argv nor args_schema"
+
+
+def _check_allowed_argv(
+    command: str,
+    argv: list[str],
+    allowed: list[list[str]],
+    matcher: PathMatcher | None,
+) -> str | None:
+    for template in allowed:
+        if len(template) != len(argv):
+            continue
+        ok = True
+        for tmpl_tok, real_tok in zip(template, argv):
+            if tmpl_tok == "<allowed_path>":
+                if matcher is None or matcher.matches(real_tok) is not None:
+                    ok = False
+                    break
+            elif tmpl_tok != real_tok:
+                ok = False
+                break
+        if ok:
+            return None
+    return f"argv {argv!r} does not match any allowed_argv for {command!r}"
+
+
+def _check_args_schema(
+    command: str,
+    argv: list[str],
+    schema: list[dict[str, Any]],
+    matcher: PathMatcher | None,
+) -> str | None:
+    if not argv or argv[0] != command:
+        return f"argv[0]={argv[0]!r} does not match command {command!r}"
+    rest = argv[1:]
+    cursor = 0
+
+    for entry in schema:
+        role = entry.get("role")
+        if role == "flags":
+            allowed = set(entry.get("allowed", []))
+            forbidden = set(entry.get("forbidden", []))
+            value_taking = set(entry.get("value_taking", []))
+            while cursor < len(rest) and rest[cursor].startswith("-"):
+                tok = rest[cursor]
+                if tok in forbidden:
+                    return f"flag {tok!r} is forbidden for {command!r}"
+                if tok not in allowed:
+                    return f"flag {tok!r} is not in allowed list for {command!r}"
+                cursor += 1
+                if tok in value_taking:
+                    # Consume the flag's value (e.g. "5" after "-n", "%y"
+                    # after "-c"). Without this branch the value would
+                    # fall through to the next schema entry and get
+                    # matched as a positional path, producing a
+                    # misleading "path '5' not in allowed_paths" reject.
+                    if cursor >= len(rest):
+                        return (
+                            f"flag {tok!r} requires a value but argv ended for {command!r}"
+                        )
+                    cursor += 1
+        elif role == "positional":
+            if cursor >= len(rest):
+                if entry.get("required", False):
+                    return f"missing required positional for {command!r}"
+                continue
+            tok = rest[cursor]
+            cursor += 1
+            if entry.get("kind") == "path" and entry.get("must_match_allowed_paths"):
+                if matcher is None:
+                    return "matcher required for path positional"
+                reason = matcher.matches(tok)
+                if reason is not None:
+                    return reason
+        elif role == "predicate_pairs":
+            allowed = set(entry.get("allowed_predicates", []))
+            forbidden = set(entry.get("forbidden_predicates", []))
+            while cursor < len(rest):
+                pred = rest[cursor]
+                if pred in forbidden:
+                    return f"predicate {pred!r} is forbidden for {command!r}"
+                if pred not in allowed:
+                    return f"predicate {pred!r} is not allowed for {command!r}"
+                cursor += 1
+                if cursor >= len(rest):
+                    return f"predicate {pred!r} missing value"
+                cursor += 1  # consume the value
+        else:
+            return f"unknown args_schema role {role!r}"
+
+    if cursor != len(rest):
+        return f"unexpected trailing argv tokens: {rest[cursor:]!r}"
+    return None

agentsec/audit/authorization.py

@@ -0,0 +1,160 @@
+"""AUTHORIZATION.txt parser + validator (spec §6.4).
+
+This file is the only thing standing between AgentSec and unauthorized
+execution against a real OpenClaw deployment. Any change must keep the
+canonicalization byte-for-byte consistent with the spec — a one-character
+drift breaks every previously-signed AUTHORIZATION.txt in the wild.
+"""
+
+from __future__ import annotations
+
+import base64
+import hashlib
+import hmac as _hmac
+import os
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Literal
+
+import yaml
+from pydantic import BaseModel, Field
+
+SignatureMode = Literal["hmac_sha256", "none"]
+
+
+class AuthorizationError(Exception):
+    """Raised when AUTHORIZATION.txt is malformed, missing fields, or fails validation."""
+
+
+class Authorization(BaseModel):
+    target_host: str
+    authorized_by: str
+    identity_provider: str = ""
+    identity_assertion: str = ""
+    valid_from: datetime
+    valid_until: datetime
+    scope: list[str]
+    report_output_path_prefix: str
+    signature_mode: SignatureMode = "none"
+    signature: str | None = None
+    signature_key_env: str | None = None
+
+    low_assurance: bool = Field(default=False, exclude=True)
+
+    def compute_signature(self, key: bytes) -> str:
+        msg = canonical_message(self).encode("utf-8")
+        return base64.b64encode(_hmac.new(key, msg, hashlib.sha256).digest()).decode("ascii")
+
+    def validate(
+        self,
+        *,
+        target_host: str,
+        report_output_path_prefix: str,
+        required_scopes: list[str],
+        now: datetime | None = None,
+    ) -> None:
+        """Run the seven-step chain from spec §6.4. Raises AuthorizationError on
+        the first failing step. On success, sets self.low_assurance based on
+        signature_mode and identity_assertion presence.
+
+        Steps 1-2 (file readable, YAML parses) ran in `load()`; this method
+        covers steps 3-7.
+        """
+        if self.target_host != target_host:
+            raise AuthorizationError(
+                f"target_host mismatch: AUTHORIZATION.txt={self.target_host!r}, "
+                f"requested={target_host!r}"
+            )
+        now = now or datetime.now(timezone.utc)
+        if not (self.valid_from <= now <= self.valid_until):
+            raise AuthorizationError(
+                f"current time {now.isoformat()} outside valid window "
+                f"[{self.valid_from.isoformat()}, {self.valid_until.isoformat()}]"
+            )
+        missing = [s for s in required_scopes if s not in self.scope]
+        if missing:
+            raise AuthorizationError(f"scope missing required capabilities: {missing}")
+        # Path-equality compare so `./report-2026-04-27/` (the form documented
+        # in AUTHORIZATION.txt.example and CLAUDE.md) matches the prefix the
+        # CLI builds from a Path-typed --output flag, which normalizes away
+        # `./` and the trailing slash.
+        if Path(self.report_output_path_prefix) != Path(report_output_path_prefix):
+            raise AuthorizationError(
+                f"report_output_path_prefix mismatch: "
+                f"AUTHORIZATION.txt={self.report_output_path_prefix!r}, "
+                f"requested={report_output_path_prefix!r}"
+            )
+        if self.signature_mode == "hmac_sha256":
+            if not self.signature_key_env:
+                raise AuthorizationError(
+                    "signature_mode=hmac_sha256 requires signature_key_env"
+                )
+            key_str = os.environ.get(self.signature_key_env)
+            if not key_str:
+                raise AuthorizationError(
+                    f"env var {self.signature_key_env!r} for signing key is empty"
+                )
+            if not self.signature:
+                raise AuthorizationError(
+                    "signature_mode=hmac_sha256 requires signature field"
+                )
+            expected = self.compute_signature(key_str.encode("utf-8"))
+            if not _hmac.compare_digest(expected, self.signature):
+                raise AuthorizationError("signature does not match (HMAC verification failed)")
+
+        self.low_assurance = (
+            self.signature_mode == "none" or not self.identity_assertion
+        )
+
+    @classmethod
+    def load(cls, path: Path | str) -> "Authorization":
+        p = Path(path)
+        try:
+            raw = p.read_text(encoding="utf-8")
+        except OSError as e:
+            raise AuthorizationError(f"cannot read {p}: {e}") from e
+        try:
+            data = yaml.safe_load(raw)
+        except yaml.YAMLError as e:
+            raise AuthorizationError(f"cannot parse {p} as YAML: {e}") from e
+        if not isinstance(data, dict):
+            raise AuthorizationError(f"{p}: top level must be a mapping")
+        try:
+            auth = cls.model_validate(data)
+        except Exception as e:
+            raise AuthorizationError(f"{p}: {e}") from e
+        # Pydantic v2 leaves datetimes without a tz suffix naive; comparing
+        # naive ↔ aware in validate() then raises a bare TypeError that the
+        # CLI doesn't catch, so a missing `Z` would surface as a Python
+        # stack trace instead of the friendly AuthorizationError every other
+        # rejection branch produces. Reject early with a clear message.
+        for field in ("valid_from", "valid_until"):
+            value = getattr(auth, field)
+            if value.tzinfo is None or value.tzinfo.utcoffset(value) is None:
+                raise AuthorizationError(
+                    f"{p}: {field} must include a UTC offset "
+                    f"(e.g. 2026-04-27T00:00:00Z), got naive datetime "
+                    f"{value.isoformat()}"
+                )
+        return auth
+
+
+def canonical_message(auth: "Authorization") -> str:
+    """Build the canonical-form string for HMAC signing (spec §6.4).
+
+    Field order is FIXED. Datetimes are serialized as ISO-8601 with explicit
+    timezone offset (the spec uses 'Z' but datetime.isoformat() emits
+    '+00:00'; both are equivalent and the spec's wording allows either as
+    long as the implementation is consistent — we standardize on '+00:00'
+    so all signatures verify against the same canonical bytes).
+    """
+    return "\n".join([
+        auth.target_host,
+        auth.authorized_by,
+        auth.identity_provider,
+        auth.identity_assertion,
+        auth.valid_from.isoformat(),
+        auth.valid_until.isoformat(),
+        ",".join(sorted(auth.scope)),
+        auth.report_output_path_prefix,
+    ])