switch-trust-cli 1.7.0__py3-none-any.whl

switch_trust/eval/common/converter_anthropic.py

@@ -0,0 +1,172 @@
+"""Convert between Anthropic Messages API types and internal schema."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from anthropic.types import Message as AnthropicMessage
+from anthropic.types import MessageParam, TextBlock, ThinkingBlock, ToolUseBlock
+
+from switch_trust.eval.common.schema import (
+    Content,
+    Message,
+    Part,
+    PartType,
+    Role,
+    ToolCall,
+    ToolResult,
+)
+
+# -- To internal --------------------------------------------------------------
+
+
+def to_content(message: dict[str, Any] | AnthropicMessage) -> Content:
+    """Convert an Anthropic message (param dict or response object) to Content."""
+    if isinstance(message, AnthropicMessage):
+        return _response_to_content(message)
+    return _param_to_content(message)
+
+
+def to_message(message: dict[str, Any] | AnthropicMessage) -> Message:
+    return Message(content=to_content(message))
+
+
+# -- From internal ------------------------------------------------------------
+
+
+def from_content(content: Content) -> dict[str, Any]:
+    """Convert internal Content to an Anthropic MessageParam dict.
+
+    System content is returned as ``{"role": "system", "content": ...}``
+    — the caller should extract system messages and pass them via the
+    ``system`` parameter of ``messages.create()``.
+    """
+    role = content.role.value
+
+    if role == "system":
+        return {"role": "system", "content": _parts_to_blocks(content.parts)}
+
+    return {"role": role, "content": _parts_to_blocks(content.parts)}
+
+
+def from_message(message: Message) -> dict[str, Any]:
+    return from_content(message.content)
+
+
+def from_content_obj(content: Content) -> MessageParam:
+    """Convert internal Content to a typed Anthropic MessageParam."""
+    return from_content(content)  # type: ignore[return-value]
+
+
+def from_message_obj(message: Message) -> MessageParam:
+    return from_content_obj(message.content)
+
+
+# -- Private: to internal -----------------------------------------------------
+
+
+def _response_to_content(msg: AnthropicMessage) -> Content:
+    parts: list[Part] = []
+    for block in msg.content:
+        if isinstance(block, TextBlock):
+            parts.append(Part.text_part(block.text))
+        elif isinstance(block, ToolUseBlock):
+            parts.append(
+                Part.tool_call_part(
+                    ToolCall(
+                        id=block.id,
+                        name=block.name,
+                        arguments=block.input if isinstance(block.input, dict) else {},
+                    )
+                )
+            )
+        elif isinstance(block, ThinkingBlock):
+            parts.append(Part.thinking_part(block.thinking))
+
+    if not parts:
+        parts.append(Part.text_part(""))
+    return Content(role=Role(msg.role), parts=parts)
+
+
+def _param_to_content(msg: dict[str, Any]) -> Content:
+    role = Role(msg["role"])
+    raw_content = msg.get("content", "")
+
+    parts: list[Part] = []
+
+    if isinstance(raw_content, str):
+        parts.append(Part.text_part(raw_content))
+    elif isinstance(raw_content, list):
+        for block in raw_content:
+            block_type = block.get("type", "text")
+            if block_type == "text":
+                parts.append(Part.text_part(block["text"]))
+            elif block_type == "thinking":
+                parts.append(Part.thinking_part(block["thinking"]))
+            elif block_type == "tool_use":
+                parts.append(
+                    Part.tool_call_part(
+                        ToolCall(
+                            id=block["id"],
+                            name=block["name"],
+                            arguments=block.get("input", {}),
+                        )
+                    )
+                )
+            elif block_type == "tool_result":
+                result_content = block.get("content", "")
+                if isinstance(result_content, list):
+                    # Extract text from content blocks
+                    texts = [
+                        b["text"] for b in result_content if b.get("type") == "text"
+                    ]
+                    result_content = " ".join(texts) if texts else ""
+                parts.append(
+                    Part.tool_result_part(
+                        ToolResult(
+                            tool_call_id=block["tool_use_id"],
+                            content=result_content,
+                            is_error=block.get("is_error", False),
+                        )
+                    )
+                )
+
+    if not parts:
+        parts.append(Part.text_part(""))
+
+    return Content(role=role, parts=parts)
+
+
+# -- Private: from internal ---------------------------------------------------
+
+
+def _parts_to_blocks(parts: list[Part]) -> list[dict[str, Any]]:
+    blocks: list[dict[str, Any]] = []
+    for p in parts:
+        if p.part_type == PartType.TEXT:
+            blocks.append({"type": "text", "text": p.text or ""})
+        elif p.part_type == PartType.THINKING:
+            blocks.append({"type": "thinking", "thinking": p.text or ""})
+        elif p.part_type == PartType.TOOL_CALL:
+            tc = p.tool_call
+            blocks.append(
+                {
+                    "type": "tool_use",
+                    "id": tc.id,
+                    "name": tc.name,
+                    "input": tc.arguments,
+                }
+            )
+        elif p.part_type == PartType.TOOL_RESULT:
+            tr = p.tool_result
+            blocks.append(
+                {
+                    "type": "tool_result",
+                    "tool_use_id": tr.tool_call_id,
+                    "content": tr.content
+                    if isinstance(tr.content, str)
+                    else str(tr.content),
+                    "is_error": tr.is_error,
+                }
+            )
+    return blocks

switch_trust/eval/common/converter_genai.py

@@ -0,0 +1,171 @@
+"""Convert between Google GenAI Content/Part types and internal schema."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from google.genai import types as genai_types
+
+from switch_trust.eval.common.schema import (
+    Content,
+    Message,
+    Part,
+    PartType,
+    Role,
+    ToolCall,
+    ToolResult,
+)
+
+_ROLE_MAP = {"user": Role.USER, "model": Role.ASSISTANT}
+_ROLE_MAP_REVERSE = {Role.USER: "user", Role.ASSISTANT: "model"}
+
+
+# -- To internal --------------------------------------------------------------
+
+
+def to_content(
+    content: dict[str, Any] | genai_types.Content,
+) -> Content:
+    """Convert a GenAI Content (object or dict) to internal Content."""
+    if isinstance(content, genai_types.Content):
+        return _content_obj_to_internal(content)
+    return _content_dict_to_internal(content)
+
+
+def to_message(
+    content: dict[str, Any] | genai_types.Content,
+) -> Message:
+    return Message(content=to_content(content))
+
+
+# -- From internal ------------------------------------------------------------
+
+
+def from_content(content: Content) -> dict[str, Any]:
+    """Convert internal Content to a GenAI-compatible dict."""
+    role = _ROLE_MAP_REVERSE.get(content.role, "user")
+    parts = [_part_to_genai(p) for p in content.parts]
+    return {"role": role, "parts": parts}
+
+
+def from_message(message: Message) -> dict[str, Any]:
+    return from_content(message.content)
+
+
+def from_content_obj(content: Content) -> genai_types.Content:
+    """Convert internal Content to a GenAI Content object."""
+    return genai_types.Content(**from_content(content))
+
+
+def from_message_obj(message: Message) -> genai_types.Content:
+    return from_content_obj(message.content)
+
+
+# -- Private: to internal -----------------------------------------------------
+
+
+def _content_obj_to_internal(
+    content: genai_types.Content,
+) -> Content:
+    role = _ROLE_MAP.get(content.role, Role.USER)
+    parts = [_genai_part_to_internal(p) for p in (content.parts or [])]
+    if not parts:
+        parts = [Part.text_part("")]
+    return Content(role=role, parts=parts)
+
+
+def _content_dict_to_internal(d: dict[str, Any]) -> Content:
+    role = _ROLE_MAP.get(d.get("role", "user"), Role.USER)
+    parts: list[Part] = []
+    for p in d.get("parts", []):
+        if isinstance(p, str):
+            parts.append(Part.text_part(p))
+        elif isinstance(p, dict):
+            parts.append(_genai_part_dict_to_internal(p))
+        else:
+            parts.append(_genai_part_to_internal(p))
+    if not parts:
+        parts = [Part.text_part("")]
+    return Content(role=role, parts=parts)
+
+
+def _genai_part_to_internal(part: genai_types.Part) -> Part:
+    if part.thought and part.text is not None:
+        return Part.thinking_part(part.text)
+    if part.text is not None:
+        return Part.text_part(part.text)
+    if part.function_call is not None:
+        fc = part.function_call
+        return Part.tool_call_part(
+            ToolCall(
+                id=fc.id or "",
+                name=fc.name or "",
+                arguments=dict(fc.args) if fc.args else {},
+            )
+        )
+    if part.function_response is not None:
+        fr = part.function_response
+        return Part.tool_result_part(
+            ToolResult(
+                tool_call_id=fr.id or "",
+                content=dict(fr.response) if fr.response else "",
+            )
+        )
+    return Part.text_part("")
+
+
+def _genai_part_dict_to_internal(d: dict[str, Any]) -> Part:
+    if "thought" in d and d.get("thought") and "text" in d:
+        return Part.thinking_part(d["text"])
+    if "text" in d:
+        return Part.text_part(d["text"])
+    if "function_call" in d:
+        fc = d["function_call"]
+        return Part.tool_call_part(
+            ToolCall(
+                id=fc.get("id", ""),
+                name=fc.get("name", ""),
+                arguments=fc.get("args", {}),
+            )
+        )
+    if "function_response" in d:
+        fr = d["function_response"]
+        return Part.tool_result_part(
+            ToolResult(
+                tool_call_id=fr.get("id", ""),
+                content=fr.get("response", ""),
+            )
+        )
+    return Part.text_part("")
+
+
+# -- Private: from internal ---------------------------------------------------
+
+
+def _part_to_genai(part: Part) -> dict[str, Any]:
+    if part.part_type == PartType.THINKING:
+        return {"text": part.text or "", "thought": True}
+    if part.part_type == PartType.TEXT:
+        return {"text": part.text or ""}
+    if part.part_type == PartType.TOOL_CALL:
+        tc = part.tool_call
+        return {
+            "function_call": {
+                "id": tc.id,
+                "name": tc.name,
+                "args": tc.arguments,
+            },
+        }
+    if part.part_type == PartType.TOOL_RESULT:
+        tr = part.tool_result
+        response = (
+            tr.content if isinstance(tr.content, dict) else {"result": tr.content}
+        )
+        return {
+            "function_response": {
+                "id": tr.tool_call_id,
+                "name": "",
+                "response": response,
+            },
+        }
+    return {"text": ""}

switch_trust/eval/common/converter_openai.py

@@ -0,0 +1,198 @@
+"""Convert between OpenAI ChatCompletion messages and internal schema."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from openai.types.chat import ChatCompletionMessage, ChatCompletionMessageParam
+
+from switch_trust.eval.common.schema import (
+    Content,
+    Message,
+    Part,
+    PartType,
+    Role,
+    ToolCall,
+    ToolResult,
+)
+
+# -- To internal --------------------------------------------------------------
+
+
+def to_content(message: dict[str, Any] | ChatCompletionMessage) -> Content:
+    """Convert an OpenAI message (dict param or response object) to Content."""
+    if isinstance(message, ChatCompletionMessage):
+        return _response_to_content(message)
+    return _param_to_content(message)
+
+
+def to_message(message: dict[str, Any] | ChatCompletionMessage) -> Message:
+    return Message(content=to_content(message))
+
+
+# -- From internal ------------------------------------------------------------
+
+
+def from_content(content: Content) -> dict[str, Any]:
+    """Convert internal Content to an OpenAI message param dict."""
+    role = content.role.value
+
+    if role == "system":
+        return _content_to_system(content)
+    if role == "user":
+        return _content_to_user(content)
+    return _content_to_assistant_or_tool(content)
+
+
+def from_message(message: Message) -> dict[str, Any]:
+    return from_content(message.content)
+
+
+def from_content_obj(content: Content) -> ChatCompletionMessageParam:
+    """Convert internal Content to a typed OpenAI message param."""
+    return from_content(content)  # type: ignore[return-value]
+
+
+def from_message_obj(message: Message) -> ChatCompletionMessageParam:
+    return from_content_obj(message.content)
+
+
+# -- Private: to internal -----------------------------------------------------
+
+
+def _response_to_content(msg: ChatCompletionMessage) -> Content:
+    parts: list[Part] = []
+    if msg.content:
+        parts.append(Part.text_part(msg.content))
+    if msg.tool_calls:
+        for tc in msg.tool_calls:
+            args = (
+                json.loads(tc.function.arguments)
+                if isinstance(tc.function.arguments, str)
+                else tc.function.arguments
+            )
+            parts.append(
+                Part.tool_call_part(
+                    ToolCall(
+                        id=tc.id,
+                        name=tc.function.name,
+                        arguments=args,
+                    )
+                )
+            )
+    if not parts:
+        parts.append(Part.text_part(""))
+    return Content(role=Role.ASSISTANT, parts=parts)
+
+
+def _param_to_content(msg: dict[str, Any]) -> Content:
+    role_str = msg["role"]
+    role = Role(role_str) if role_str != "tool" else Role.USER
+
+    parts: list[Part] = []
+
+    if role_str == "tool":
+        parts.append(
+            Part.tool_result_part(
+                ToolResult(
+                    tool_call_id=msg["tool_call_id"],
+                    content=msg.get("content", ""),
+                )
+            )
+        )
+        return Content(role=Role.USER, parts=parts)
+
+    content = msg.get("content")
+    if isinstance(content, str):
+        parts.append(Part.text_part(content))
+    elif isinstance(content, list):
+        for item in content:
+            if item.get("type") == "text":
+                parts.append(Part.text_part(item["text"]))
+
+    tool_calls = msg.get("tool_calls")
+    if tool_calls:
+        for tc in tool_calls:
+            func = tc["function"] if isinstance(tc, dict) else tc
+            args = func["arguments"]
+            if isinstance(args, str):
+                args = json.loads(args)
+            parts.append(
+                Part.tool_call_part(
+                    ToolCall(
+                        id=tc["id"],
+                        name=func["name"],
+                        arguments=args,
+                    )
+                )
+            )
+
+    if not parts:
+        parts.append(Part.text_part(""))
+
+    return Content(role=role, parts=parts)
+
+
+# -- Private: from internal ---------------------------------------------------
+
+
+def _content_to_system(content: Content) -> dict[str, Any]:
+    text = _collect_text(content)
+    return {"role": "system", "content": text}
+
+
+def _content_to_user(content: Content) -> dict[str, Any] | list[dict[str, Any]]:
+    # If content has tool results, emit them as separate tool messages.
+    tool_results = [p for p in content.parts if p.part_type == PartType.TOOL_RESULT]
+    if tool_results:
+        messages = []
+        for p in tool_results:
+            tr = p.tool_result
+            result_content = (
+                tr.content if isinstance(tr.content, str) else json.dumps(tr.content)
+            )
+            messages.append(
+                {
+                    "role": "tool",
+                    "tool_call_id": tr.tool_call_id,
+                    "content": result_content,
+                }
+            )
+        return messages if len(messages) > 1 else messages[0]
+
+    text = _collect_text(content)
+    return {"role": "user", "content": text}
+
+
+def _content_to_assistant_or_tool(content: Content) -> dict[str, Any]:
+    text_parts = [p for p in content.parts if p.part_type == PartType.TEXT]
+    tool_call_parts = [p for p in content.parts if p.part_type == PartType.TOOL_CALL]
+
+    msg: dict[str, Any] = {"role": "assistant"}
+    msg["content"] = (
+        " ".join(p.text for p in text_parts if p.text) if text_parts else None
+    )
+
+    if tool_call_parts:
+        msg["tool_calls"] = []
+        for p in tool_call_parts:
+            tc = p.tool_call
+            msg["tool_calls"].append(
+                {
+                    "id": tc.id,
+                    "type": "function",
+                    "function": {
+                        "name": tc.name,
+                        "arguments": json.dumps(tc.arguments),
+                    },
+                }
+            )
+
+    return msg
+
+
+def _collect_text(content: Content) -> str:
+    return " ".join(
+        p.text for p in content.parts if p.part_type == PartType.TEXT and p.text
+    )

switch_trust/eval/common/log.py

@@ -0,0 +1,116 @@
+import logging
+import os
+import re
+
+_REDACT_PATTERNS = re.compile(
+    r"("
+    r"Bearer\s+[A-Za-z0-9\-_.~+/]+=*"
+    r"|sk-[A-Za-z0-9]{10,}"
+    r"|key-[A-Za-z0-9]{10,}"
+    r"|pat-[A-Za-z0-9]{10,}"
+    r"|AIza[A-Za-z0-9_\-]{35,}"
+    r"|xox[bspra]-[A-Za-z0-9\-]+"
+    r"|ghp_[A-Za-z0-9]{36,}"
+    r"|-----BEGIN\s+(?:RSA\s+)?PRIVATE\s+KEY-----"
+    r"[\s\S]*?-----END"
+    r")",
+    re.IGNORECASE,
+)
+
+_REPLACEMENT = "[REDACTED]"
+
+
+def redact(text: str) -> str:
+    """Replace any known secret pattern in ``text`` with ``[REDACTED]``."""
+    return _REDACT_PATTERNS.sub(_REPLACEMENT, text)
+
+
+class RedactingFilter(logging.Filter):
+    """Scrub secrets from log records before they reach handlers."""
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        if isinstance(record.msg, str):
+            record.msg = redact(record.msg)
+        if record.args:
+            if isinstance(record.args, tuple):
+                record.args = tuple(
+                    redact(a) if isinstance(a, str) else a for a in record.args
+                )
+            elif isinstance(record.args, dict):
+                record.args = {
+                    k: redact(v) if isinstance(v, str) else v
+                    for k, v in record.args.items()
+                }
+        return True
+
+
+class RedactingFormatter(logging.Formatter):
+    """Wrap another formatter and scrub secrets from its rendered output.
+
+    A handler-level :class:`RedactingFilter` only sees ``record.msg`` and
+    ``record.args``, so secrets that surface elsewhere — an exception message or
+    traceback (e.g. an aiohttp connect error carrying a key in the URL), or a
+    stray ``extra=`` field — reach the handler unredacted. Redacting the fully
+    formatted string closes that gap regardless of how the wrapped formatter
+    renders exceptions.
+    """
+
+    def __init__(self, inner: logging.Formatter) -> None:
+        super().__init__()
+        self._inner = inner
+
+    def format(self, record: logging.LogRecord) -> str:
+        return redact(self._inner.format(record))
+
+
+_NOISY_LOGGERS = [
+    "anthropic",
+    "google.genai",
+    "google.adk",
+    "google_adk",
+    "google.auth",
+    "google.api_core",
+    "google_genai.models",
+    "httpcore",
+    "httpx",
+    "urllib3",
+    "opentelemetry",
+    "litellm",
+    "LiteLLM",
+    "openai",
+    "asyncio",
+]
+
+
+def silence_noisy_loggers() -> None:
+    for name in _NOISY_LOGGERS:
+        logging.getLogger(name).setLevel(logging.WARNING)
+    logging.captureWarnings(True)
+
+    os.environ.setdefault("HF_DATASETS_DISABLE_PROGRESS_BARS", "1")
+    os.environ.setdefault("TRANSFORMERS_VERBOSITY", "error")
+    os.environ.setdefault("DATASETS_VERBOSITY", "error")
+
+
+LOG_FORMAT = "%(asctime)s %(levelname)s %(name)s [%(filename)s:%(lineno)d]: %(message)s"
+
+
+def setup_file_logging(log_path: str) -> None:
+    root = logging.getLogger()
+    root.setLevel(logging.DEBUG)
+    root.handlers.clear()
+
+    file_handler = logging.FileHandler(log_path, mode="w")
+    file_handler.setLevel(logging.DEBUG)
+    file_handler.setFormatter(logging.Formatter(LOG_FORMAT))
+    file_handler.addFilter(RedactingFilter())
+    root.addHandler(file_handler)
+
+    os.environ["LITELLM_LOG"] = "CRITICAL"
+    try:
+        import litellm  # noqa: PLC0415
+
+        litellm.suppress_debug_info = True
+    except ImportError:
+        pass
+    silence_noisy_loggers()

switch_trust/eval/common/reference.py

@@ -0,0 +1,25 @@
+from dataclasses import dataclass
+from enum import Enum
+
+from dataclasses_json import dataclass_json
+
+
+class ReferenceType(str, Enum):
+    """The kind of object a Reference points to."""
+
+    DETECTOR = "detector"
+    EVALUATION = "evaluation"
+    EVALUATION_SUITE = "evaluation_suite"
+    MESSAGE_COLLECTION = "message_collection"
+    MODEL = "model"
+    MODEL_EVALUATION_SUITE = "model_evaluation_suite"
+    MODEL_EVALUATION_SUITE_RUN = "model_evaluation_suite_run"
+
+
+@dataclass_json
+@dataclass
+class Reference:
+    id: str
+    name: str
+    type: ReferenceType
+    description: str | None = None