evalrx 0.1.2__py3-none-any.whl

evalrx/models/paper_methods/vicrop.py

@@ -0,0 +1,137 @@
+"""LLaVA relative-attention ViCrop executor from *MLLMs Know Where to Look*.
+
+The implementation is intentionally limited to the paper's LLaVA-style
+attention selector: task/general attention ratio, adaptive local-contrast
+window, then answer from original image plus crop.  It is used by the local
+backend and can therefore be proposed as a read-only L3a auto-fix candidate.
+"""
+
+from __future__ import annotations
+
+import math
+from typing import TYPE_CHECKING, Any
+
+import numpy as np
+
+from evalrx.analyzers.attention.relative_attn import attention_heatmap
+from evalrx.core.case import FailureCase, Inputs
+
+if TYPE_CHECKING:
+    from evalrx.models.backends.hf_local import HFLocalModel
+
+
+GENERAL_PROMPT = "Describe the image generally. Do not focus on any particular question or detail."
+
+
+def relative_attention_map(model: "HFLocalModel", inputs: Inputs, layer: int | float) -> np.ndarray:
+    """Paper-style ratio of task-specific to general image-patch attention."""
+    image = getattr(inputs, "image", None)
+    if image is None or isinstance(image, (list, tuple)):
+        raise ValueError("ViCrop requires exactly one source image")
+    prompt = str(getattr(inputs, "prompt", ""))
+    specific = attention_heatmap(
+        model, FailureCase(id="vicrop-specific", inputs=Inputs(prompt, image)), layer=layer
+    )
+    general = attention_heatmap(
+        model, FailureCase(id="vicrop-general", inputs=Inputs(GENERAL_PROMPT, image)), layer=layer
+    )
+    if specific is None or general is None or specific.shape != general.shape:
+        raise RuntimeError("could not obtain compatible task/general image attention maps")
+    return specific / np.maximum(general, np.finfo(np.float64).eps)
+
+
+def sliding_window_box(
+    relative_map: np.ndarray,
+    image_size: tuple[int, int],
+    *,
+    bbox_size: int,
+) -> tuple[float, float, float, float]:
+    """Released ViCrop adaptive-window rule, returned as normalized xyxy."""
+    if relative_map.ndim != 2 or min(relative_map.shape) < 2:
+        raise ValueError("relative attention must be a 2-D patch grid")
+    h, w = relative_map.shape
+    width, height = image_size
+    block_width, block_height = width / w, height / h
+    candidates: list[tuple[float, tuple[int, int], tuple[int, int], float]] = []
+    for ratio in (1, 1.2, 1.4, 1.6, 1.8, 2):
+        box_w = min(int(bbox_size * ratio / block_width), w)
+        box_h = min(int(bbox_size * ratio / block_height), h)
+        if w - box_w < 1 and h - box_h < 1:
+            if ratio == 1:
+                return 0.0, 0.0, 1.0, 1.0
+            continue
+        scores = np.empty((h - box_h + 1, w - box_w + 1), dtype=float)
+        for top in range(scores.shape[0]):
+            for left in range(scores.shape[1]):
+                scores[top, left] = relative_map[top : top + box_h, left : left + box_w].sum()
+        top, left = np.unravel_index(np.argmax(scores), scores.shape)
+        neighbors = [
+            float(scores[neighbor_top, neighbor_left])
+            for neighbor_top, neighbor_left in (
+                (top, left - 1), (top, left + 1), (top - 1, left), (top + 1, left)
+            )
+            if 0 <= neighbor_top < scores.shape[0] and 0 <= neighbor_left < scores.shape[1]
+        ]
+        contrast = (float(scores[top, left]) - float(np.mean(neighbors))) / (box_w * box_h)
+        candidates.append((contrast, (left, top), (box_w, box_h), bbox_size * ratio))
+    _, (left, top), (box_w, box_h), selected_size = max(candidates, key=lambda item: item[0])
+    center_x = int(left * block_width + block_width * box_w / 2)
+    center_y = int(top * block_height + block_height * box_h / 2)
+    half = selected_size // 2
+    center_x = min(max(center_x, half), width - half)
+    center_y = min(max(center_y, half), height - half)
+    return (
+        max(0, center_x - half) / width,
+        max(0, center_y - half) / height,
+        min(width, center_x + half) / width,
+        min(height, center_y + half) / height,
+    )
+
+
+def crop_from_box(image: Any, box: tuple[float, float, float, float]):
+    """Load one source image and return its selected crop."""
+    from PIL import Image
+
+    if isinstance(image, Image.Image):
+        rgb = image.convert("RGB")
+    else:
+        with Image.open(image) as opened:
+            rgb = opened.convert("RGB")
+    width, height = rgb.size
+    left, top, right, bottom = box
+    x1 = max(0, min(width - 1, math.floor(left * width)))
+    y1 = max(0, min(height - 1, math.floor(top * height)))
+    x2 = max(x1 + 1, min(width, math.ceil(right * width)))
+    y2 = max(y1 + 1, min(height, math.ceil(bottom * height)))
+    return rgb.crop((x1, y1, x2, y2))
+
+
+def prepare_views(
+    model: "HFLocalModel", inputs: Inputs, *, layer: int | float = 14
+) -> tuple[Any, Any, str]:
+    """Return source image, selected crop, and ViCrop's original+crop prompt."""
+    image = getattr(inputs, "image", None)
+    if image is None or isinstance(image, (list, tuple)):
+        raise ValueError("ViCrop requires exactly one source image")
+    relative = relative_attention_map(model, inputs, layer)
+    from PIL import Image
+
+    if isinstance(image, Image.Image):
+        image_size = image.size
+    else:
+        with Image.open(image) as opened:
+            image_size = opened.size
+    box = sliding_window_box(relative, image_size, bbox_size=336)
+    crop = crop_from_box(image, box)
+    prompt = (
+        "The first image is the full scene and the second is a task-relative visual crop. "
+        "Use both views, prioritizing visible detail in the crop, then answer the question.\n\n"
+        + str(getattr(inputs, "prompt", ""))
+    )
+    return image, crop, prompt
+
+
+def generate(model: "HFLocalModel", inputs: Inputs, *, layer: int | float = 14) -> str:
+    """Answer from original image and a task-relative ViCrop crop."""
+    image, crop, prompt = prepare_views(model, inputs, layer=layer)
+    return str(model.generate(Inputs(prompt, [image, crop])))

evalrx/models/toolcodec.py

@@ -0,0 +1,143 @@
+"""ToolCallCodec — the ONLY backend/model-specific piece of agent tool-calling.
+
+The agent loop is identical across backends; what differs is how tools are
+encoded into the request and how a tool call is decoded out of the model's
+reply.  Two implementations:
+
+  * :class:`OpenAIToolCodec` — native structured tool_calls (API / ``vllm serve``).
+  * :class:`QwenToolCodec` — Hermes-style ``<tool_call>{...}</tool_call>`` text
+    parsing (Qwen2.5/Qwen3 and many template-based local models).
+
+``encode`` is shared (OpenAI tool schema, which HF ``apply_chat_template`` also
+accepts).  Message-threading helpers let each codec put the assistant turn and
+the tool result back into history in the form its backend expects.  Torch-free.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from abc import ABC, abstractmethod
+from typing import Optional
+
+from evalrx.core.tool import ChatTurn, Tool, ToolCall
+
+
+class ToolCallCodec(ABC):
+    """Encode tool schemas into a request; decode a tool call out of a reply."""
+
+    name: str = "codec"
+
+    def encode(self, tools: list[Tool]) -> list[dict]:
+        """OpenAI tool schema — accepted by both APIs and HF apply_chat_template."""
+        return [t.to_openai_schema() for t in tools]
+
+    @abstractmethod
+    def decode(self, turn: ChatTurn) -> Optional[ToolCall]:
+        """Return the FIRST tool call in *turn*, or ``None`` if it's a final answer."""
+
+    def decode_all(self, turn: ChatTurn) -> list[ToolCall]:
+        """All tool calls in *turn* (for parallel calling). Default: the first only."""
+        call = self.decode(turn)
+        return [call] if call else []
+
+    def final_text(self, turn: ChatTurn) -> str:
+        """The clean assistant text to record as a final answer (codecs may strip reasoning)."""
+        return turn.text or ""
+
+    # -- message threading (sensible defaults; OpenAI overrides) -------
+    def assistant_message(self, turn: ChatTurn, call: Optional[ToolCall]) -> dict:
+        return {"role": "assistant", "content": turn.text}
+
+    def tool_message(self, call: ToolCall, result) -> dict:
+        return {"role": "tool", "content": str(result)}
+
+
+class OpenAIToolCodec(ToolCallCodec):
+    """Native OpenAI-compatible tool calling (structured tool_calls)."""
+
+    name = "openai"
+
+    def decode(self, turn: ChatTurn) -> Optional[ToolCall]:
+        if not turn.raw_tool_calls:
+            return None
+        tc = turn.raw_tool_calls[0]
+        fn = tc.get("function", tc)
+        args = fn.get("arguments", {})
+        if isinstance(args, str):
+            try:
+                args = json.loads(args) if args.strip() else {}
+            except json.JSONDecodeError:
+                args = {"_raw": args}
+        return ToolCall(name=fn.get("name", ""), args=args or {}, id=tc.get("id"))
+
+    def assistant_message(self, turn: ChatTurn, call: Optional[ToolCall]) -> dict:
+        msg: dict = {"role": "assistant", "content": turn.text or None}
+        if turn.raw_tool_calls:
+            # The loop executes ONE call per turn; replaying more tool_calls than
+            # have matching tool results is a protocol violation on OpenAI-style APIs.
+            msg["tool_calls"] = turn.raw_tool_calls[:1]
+        return msg
+
+    def tool_message(self, call: ToolCall, result) -> dict:
+        return {"role": "tool", "tool_call_id": call.id or "call_0", "content": str(result)}
+
+
+class QwenToolCodec(ToolCallCodec):
+    """Qwen2.5 / Qwen3 tool-calling: Hermes-style ``<tool_call>…</tool_call>`` blocks.
+
+    Robust to Qwen's real output:
+      * captures the FULL JSON between the tags (handles **nested** ``arguments``,
+        unlike a ``{...}`` regex that stops at the first brace),
+      * supports **multiple** ``<tool_call>`` blocks (parallel calls) via ``decode_all``,
+      * strips ``<think>…</think>`` reasoning from the final answer.
+
+    Other families differ (Llama 3.1, etc.) — add a sibling codec and route it in
+    :func:`codec_for`.
+    """
+
+    name = "qwen"
+    # capture everything between the tags (non-greedy stops at the first </tool_call>,
+    # which nested JSON never contains) -> nested arguments parse correctly.
+    _BLOCK = re.compile(r"<tool_call>\s*(.*?)\s*</tool_call>", re.DOTALL)
+    _THINK = re.compile(r"<think>.*?</think>\s*", re.DOTALL)
+
+    def decode(self, turn: ChatTurn) -> Optional[ToolCall]:
+        calls = self.decode_all(turn)
+        return calls[0] if calls else None
+
+    def decode_all(self, turn: ChatTurn) -> list[ToolCall]:
+        out: list[ToolCall] = []
+        for block in self._BLOCK.findall(turn.text or ""):
+            try:
+                payload = json.loads(block)
+            except json.JSONDecodeError:
+                continue
+            args = payload.get("arguments", payload.get("args", {})) or {}
+            out.append(ToolCall(name=payload.get("name", ""), args=args))
+        return out
+
+    def final_text(self, turn: ChatTurn) -> str:
+        return self._THINK.sub("", turn.text or "").strip()
+
+
+# family -> local codec (default Hermes/Qwen style covers most template models)
+_LOCAL_CODECS = {
+    "qwen2": QwenToolCodec,
+    "qwen3": QwenToolCodec,
+    "qwen3_moe": QwenToolCodec,
+    "qwen3_vl": QwenToolCodec,
+    "qwen3_vl_moe": QwenToolCodec,
+}
+
+
+def codec_for(handle) -> ToolCallCodec:
+    """Pick a codec for *handle*: OpenAI for native-tool-call handles, family-routed for local."""
+    from evalrx.models.backends.api import APIModel
+
+    # APIModel, and any handle declaring native structured calls (e.g. GeminiModel).
+    if isinstance(handle, APIModel) or getattr(handle, "tool_call_style", "") == "native":
+        return OpenAIToolCodec()
+    spec = getattr(handle, "spec", None)
+    family = getattr(spec, "family", "")
+    return _LOCAL_CODECS.get(family, QwenToolCodec)()

evalrx/models/tools/__init__.py

@@ -0,0 +1,20 @@
+"""Subject-side tool implementations for the :class:`~evalrx.models.agent.Agent` loop.
+
+These are the tools an agent-under-test may call — distinct from
+``eval_agent.agentic`` (the host's *diagnosis* tools) and ``agent_runtime``
+(the analysis-side CLI coding agents).  Design constraints, because these runs
+are diagnosed, not just executed:
+
+* **deterministic** — no sampling, versioned weights, so the tool layer stays a
+  controlled variable under held-out re-evaluation and fix-vs-baseline pairing;
+* **model-agnostic schemas** — normalized ``[0,1]`` coordinates, conventions
+  spelled out in the tool description (the description is an L1 fix surface);
+* **structured results** — :class:`~evalrx.core.tool.ToolResult` with
+  model-visible text, re-injectable images, and host-side ``meta`` that lands
+  on the trajectory step.
+"""
+
+from evalrx.models.tools.perception import detect_tool, ocr_tool
+from evalrx.models.tools.visual import zoom_in_tool
+
+__all__ = ["zoom_in_tool", "ocr_tool", "detect_tool"]

evalrx/models/tools/perception.py

@@ -0,0 +1,300 @@
+"""Structured-perception tools — the T1 tier: ``image_ocr`` and ``image_detect``.
+
+Both follow the house tool constraints: deterministic engines (versioned
+weights, no sampling), normalized ``[0, 1]`` coordinates in results, engine
+failures surfaced through the standard executor error envelope, and
+:class:`~evalrx.core.tool.ToolResult` outputs whose ``meta`` records what
+actually ran.
+
+Engines are **injectable** (any callable with the documented signature) so the
+tools stay dependency-light and unit-testable; the default engines are built
+lazily on first call:
+
+* OCR   — ``easyocr`` (reuses the already-installed torch stack),
+* detect — open-vocabulary Grounding DINO via ``transformers``
+  (``IDEA-Research/grounding-dino-tiny``).
+"""
+
+from __future__ import annotations
+
+from typing import Any, Callable, Optional
+
+from evalrx.core.tool import Tool, ToolResult
+from evalrx.models.tools.visual import _sanitize_bbox, _to_pil
+
+# ----------------------------------------------------------------------
+# OCR
+# ----------------------------------------------------------------------
+OCR_DESCRIPTION = (
+    "Read the text in the CURRENT image (optical character recognition). "
+    "Optionally pass bbox=[x0, y0, x1, y1] (fractions of width/height in "
+    "[0, 1]) to read only that region. Returns the recognized text lines with "
+    "their locations. Use this for signs, labels, documents, or any question "
+    "about what text says."
+)
+
+OCR_PARAMETERS = {
+    "type": "object",
+    "properties": {
+        "bbox": {
+            "type": "array",
+            "items": {"type": "number"},
+            "minItems": 4,
+            "maxItems": 4,
+            "description": "Optional region [x0, y0, x1, y1], fractions in [0, 1].",
+        },
+    },
+    "required": [],
+}
+
+
+def default_ocr_engine(languages: "list[str] | None" = None, device: str = "cpu") -> Callable:
+    """Build the default OCR engine on ``easyocr`` (deterministic, local weights).
+
+    Returns ``engine(pil_image) -> list[(bbox_px, text, confidence)]`` where
+    ``bbox_px`` is a 4-point polygon in pixels (easyocr's native shape).
+    """
+    try:
+        import easyocr
+    except ImportError as exc:
+        raise ImportError(
+            "image_ocr's default engine needs easyocr (pip install easyocr), "
+            "or inject your own engine via ocr_tool(..., engine=...)."
+        ) from exc
+    import numpy as np
+
+    reader = easyocr.Reader(languages or ["en"], gpu=device != "cpu", verbose=False)
+
+    def _engine(pil_image):
+        return reader.readtext(np.array(pil_image))
+
+    return _engine
+
+
+def ocr_tool(
+    image: Any,
+    *,
+    engine: Optional[Callable] = None,
+    max_lines: int = 40,
+) -> Tool:
+    """Build the ``image_ocr`` tool bound to *image*.
+
+    *engine* is ``callable(pil_image) -> list[(polygon_px, text, confidence)]``;
+    defaults to :func:`default_ocr_engine` built lazily on first call.
+    """
+    base = _to_pil(image)
+    width, height = base.size
+    state = {"engine": engine}
+
+    def _get_engine() -> Callable:
+        if state["engine"] is None:
+            state["engine"] = default_ocr_engine()
+        return state["engine"]
+
+    def _ocr(bbox=None) -> ToolResult:
+        region = base
+        offset = (0, 0)
+        meta: dict = {"engine": "injected" if engine else "easyocr", "region": "full"}
+        if bbox is not None:
+            frac, mode = _sanitize_bbox(bbox, width, height)
+            px = (
+                int(round(frac[0] * width)),
+                int(round(frac[1] * height)),
+                int(round(frac[2] * width)),
+                int(round(frac[3] * height)),
+            )
+            region = base.crop(px)
+            offset = (px[0], px[1])
+            meta.update({"region": "bbox", "bbox_frac": [round(v, 4) for v in frac], "coord_mode": mode})
+
+        detections = _get_engine()(region)
+        lines: list[str] = []
+        for det in detections[:max_lines]:
+            poly, text, conf = det[0], det[1], (det[2] if len(det) > 2 else None)
+            xs = [p[0] + offset[0] for p in poly]
+            ys = [p[1] + offset[1] for p in poly]
+            loc = (
+                f"[{min(xs) / width:.2f}, {min(ys) / height:.2f}, "
+                f"{max(xs) / width:.2f}, {max(ys) / height:.2f}]"
+            )
+            conf_s = f" (conf {conf:.2f})" if isinstance(conf, (int, float)) else ""
+            lines.append(f"- {text!r} at {loc}{conf_s}")
+        n_total = len(detections)
+        meta["n_detections"] = n_total
+        if not lines:
+            text_out = "No text detected in the requested region."
+        else:
+            clipped = f" (showing {max_lines} of {n_total})" if n_total > max_lines else ""
+            text_out = (
+                f"Detected {n_total} text line(s){clipped}, locations as "
+                f"[x0, y0, x1, y1] fractions of the full image:\n" + "\n".join(lines)
+            )
+        return ToolResult(text=text_out, meta=meta)
+
+    return Tool(name="image_ocr", description=OCR_DESCRIPTION, parameters=OCR_PARAMETERS, fn=_ocr)
+
+
+# ----------------------------------------------------------------------
+# Open-vocabulary detection
+# ----------------------------------------------------------------------
+DETECT_DESCRIPTION = (
+    "Detect objects in the CURRENT image by name (open-vocabulary detection). "
+    "Pass query as the object name(s) to look for, separated by ' . ' for "
+    "multiple (e.g. 'dog . frisbee'). Returns each detected instance with a "
+    "confidence score and its box as [x0, y0, x1, y1] fractions of the image, "
+    "plus an annotated copy of the image with the boxes drawn. Use this for "
+    "counting, presence checks, and locating objects."
+)
+
+DETECT_PARAMETERS = {
+    "type": "object",
+    "properties": {
+        "query": {
+            "type": "string",
+            "description": "Object name(s) to detect, ' . '-separated for multiple.",
+        },
+    },
+    "required": ["query"],
+}
+
+DEFAULT_DETECTOR_ID = "IDEA-Research/grounding-dino-tiny"
+
+
+def default_detect_engine(
+    model_id: str = DEFAULT_DETECTOR_ID,
+    device: str = "cpu",
+    threshold: float = 0.3,
+    text_threshold: float = 0.25,
+) -> Callable:
+    """Build the default detector on Grounding DINO via transformers.
+
+    Returns ``engine(pil_image, query) -> list[{"label", "score", "box_px"}]``
+    (deterministic: pure forward pass, fixed thresholds, versioned weights).
+    """
+    try:
+        import torch
+        from transformers import AutoModelForZeroShotObjectDetection, AutoProcessor
+    except ImportError as exc:
+        raise ImportError(
+            "image_detect's default engine needs torch+transformers "
+            "(pip install 'evalrx[local]'), or inject an engine via "
+            "detect_tool(..., engine=...)."
+        ) from exc
+
+    processor = AutoProcessor.from_pretrained(model_id)
+    model = AutoModelForZeroShotObjectDetection.from_pretrained(model_id).to(device).eval()
+
+    def _engine(pil_image, query: str):
+        text = query.strip().rstrip(".") + "."  # grounding-dino wants lowercase, dot-terminated
+        inputs = processor(images=pil_image, text=text.lower(), return_tensors="pt").to(device)
+        with torch.no_grad():
+            outputs = model(**inputs)
+        results = processor.post_process_grounded_object_detection(
+            outputs,
+            inputs.input_ids,
+            threshold=threshold,
+            text_threshold=text_threshold,
+            target_sizes=[pil_image.size[::-1]],
+        )[0]
+        out = []
+        for label, score, box in zip(
+            results["text_labels" if "text_labels" in results else "labels"],
+            results["scores"],
+            results["boxes"],
+        ):
+            out.append(
+                {
+                    "label": str(label),
+                    "score": float(score),
+                    "box_px": [float(v) for v in box.tolist()],
+                }
+            )
+        return out
+
+    return _engine
+
+
+def _annotate(image, detections: list) -> Any:
+    """Draw detection boxes + labels on a copy of *image*."""
+    from PIL import ImageDraw
+
+    annotated = image.copy()
+    draw = ImageDraw.Draw(annotated)
+    for i, det in enumerate(detections):
+        x0, y0, x1, y1 = det["box_px"]
+        draw.rectangle([x0, y0, x1, y1], outline=(208, 59, 59), width=3)
+        draw.text((x0 + 3, max(0, y0 - 12)), f"{i}: {det['label']} {det['score']:.2f}", fill=(208, 59, 59))
+    return annotated
+
+
+def detect_tool(
+    image: Any,
+    *,
+    engine: Optional[Callable] = None,
+    annotate: bool = True,
+    max_detections: int = 20,
+) -> Tool:
+    """Build the ``image_detect`` tool bound to *image*.
+
+    *engine* is ``callable(pil_image, query) -> list[{"label", "score",
+    "box_px"}]``; defaults to :func:`default_detect_engine` built lazily on
+    first call (weights load once per tool instance).
+    """
+    base = _to_pil(image)
+    width, height = base.size
+    state = {"engine": engine}
+
+    def _get_engine() -> Callable:
+        if state["engine"] is None:
+            state["engine"] = default_detect_engine()
+        return state["engine"]
+
+    def _detect(query: str) -> ToolResult:
+        if not str(query).strip():
+            raise ValueError("query must name at least one object to detect")
+        detections = _get_engine()(base, str(query))
+        detections = sorted(detections, key=lambda d: -d["score"])[:max_detections]
+        meta: dict = {
+            "engine": "injected" if engine else DEFAULT_DETECTOR_ID,
+            "query": str(query),
+            "n_detections": len(detections),
+            "detections": [
+                {
+                    "label": d["label"],
+                    "score": round(d["score"], 3),
+                    "box_frac": [
+                        round(d["box_px"][0] / width, 4),
+                        round(d["box_px"][1] / height, 4),
+                        round(d["box_px"][2] / width, 4),
+                        round(d["box_px"][3] / height, 4),
+                    ],
+                }
+                for d in detections
+            ],
+        }
+        if not detections:
+            return ToolResult(
+                text=f"No instances of {query!r} detected (score threshold not reached).",
+                meta=meta,
+            )
+        lines = [
+            f"- #{i} {d['label']} (score {d['score']:.2f}) at "
+            f"[{d['box_frac'][0]:.2f}, {d['box_frac'][1]:.2f}, {d['box_frac'][2]:.2f}, {d['box_frac'][3]:.2f}]"
+            for i, d in enumerate(meta["detections"])
+        ]
+        images = []
+        text_out = (
+            f"Detected {len(detections)} instance(s) of {query!r}, boxes as "
+            f"[x0, y0, x1, y1] fractions of the image:\n" + "\n".join(lines)
+        )
+        if annotate:
+            images.append(_annotate(base, detections))
+            text_out += "\nAn annotated copy of the image with numbered boxes is attached."
+        return ToolResult(text=text_out, images=images, meta=meta)
+
+    return Tool(
+        name="image_detect",
+        description=DETECT_DESCRIPTION,
+        parameters=DETECT_PARAMETERS,
+        fn=_detect,
+    )