evalrx 0.1.2__py3-none-any.whl

evalrx/models/blackbox/gemini.py

@@ -0,0 +1,279 @@
+"""Gemini API model — black-box GENERATE + native tool-calling via Google's Gen AI SDK.
+
+Install the optional dependency::
+
+    pip install evalrx[gemini]
+
+Two roles:
+
+* judge — ``GeminiModel(api_key=...)`` as before (``generate()``), and
+* **agent base** — ``chat(messages, tools=...)`` speaks the agent loop's
+  protocol with Gemini's NATIVE function calling (not an OpenAI-compat
+  bridge, whose tool-call shims are historically quirky).  Images in
+  content-block messages are sent as inline PNG parts, so multimodal
+  tool loops (zoom crops fed back) work like on the other backends.
+
+The loop threads history in OpenAI-ish form (assistant ``tool_calls`` /
+``role="tool"`` results, via :class:`~evalrx.models.toolcodec.OpenAIToolCodec`
+— ``tool_call_style = "native"`` routes it); this module converts that history
+to ``google-genai`` contents per call.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+from typing import Any, Optional
+
+from evalrx.core.capability import Capability
+from evalrx.core.case import Inputs
+from evalrx.core.tool import ChatTurn
+from evalrx.models.blackbox.base import BlackboxModel, Trace
+
+
+def _png_bytes(image: Any) -> bytes:
+    """Encode a PIL image (or open a path) as PNG bytes."""
+    import io
+
+    if not (hasattr(image, "save") and hasattr(image, "mode")):
+        from PIL import Image
+
+        image = Image.open(image).convert("RGB")
+    buf = io.BytesIO()
+    image.save(buf, format="PNG")
+    return buf.getvalue()
+
+
+_AUDIO_MIME = {"wav": "audio/wav", "mp3": "audio/mp3", "flac": "audio/flac",
+               "ogg": "audio/ogg", "m4a": "audio/mp4"}
+
+
+def _audio_bytes(audio: Any) -> "tuple[bytes, str]":
+    """``(bytes, mime_type)`` for an audio path / raw bytes / waveform.
+
+    Reuses the endpoint path's encoder (a path keeps its own format; bytes are
+    taken as WAV; a waveform is written as 16-bit PCM WAV), so the two API
+    backends send the same bytes for the same case.
+    """
+    import base64
+
+    from evalrx.models.backends.openai_compat import _to_input_audio
+
+    encoded = _to_input_audio(audio)
+    return base64.b64decode(encoded["data"]), _AUDIO_MIME.get(encoded["format"], "audio/wav")
+
+
+def _to_genai_contents(messages: list, types: Any) -> "tuple[Optional[str], list]":
+    """Convert loop messages (OpenAI-ish history + content blocks) into
+    ``(system_instruction, genai contents)``.
+
+    ``role="tool"`` results become ``function_response`` parts; the function
+    name comes from the preceding assistant turn's tool call (the loop is
+    single-call-per-turn, so pairing by order is exact).
+    """
+    system: Optional[str] = None
+    contents: list = []
+    last_call_name = ""
+    for msg in messages:
+        role = msg.get("role")
+        content = msg.get("content")
+        if role == "system":
+            system = content if isinstance(content, str) else str(content)
+            continue
+        if role == "tool":
+            contents.append(
+                types.Content(
+                    role="user",
+                    parts=[
+                        types.Part(
+                            function_response=types.FunctionResponse(
+                                name=last_call_name or "tool",
+                                response={"result": content},
+                            )
+                        )
+                    ],
+                )
+            )
+            continue
+        if role == "assistant":
+            parts = []
+            if isinstance(content, str) and content:
+                parts.append(types.Part(text=content))
+            for tc in msg.get("tool_calls") or []:
+                fn = tc.get("function", tc)
+                args = fn.get("arguments", {})
+                if isinstance(args, str):
+                    try:
+                        args = json.loads(args) if args.strip() else {}
+                    except json.JSONDecodeError:
+                        args = {"_raw": args}
+                last_call_name = fn.get("name", "")
+                parts.append(
+                    types.Part(
+                        function_call=types.FunctionCall(name=last_call_name, args=args)
+                    )
+                )
+            if parts:
+                contents.append(types.Content(role="model", parts=parts))
+            continue
+        # user message: plain text or content blocks (text / image)
+        parts = []
+        if isinstance(content, list):
+            for block in content:
+                if isinstance(block, dict) and block.get("type") == "image":
+                    parts.append(
+                        types.Part.from_bytes(
+                            data=_png_bytes(block.get("image")), mime_type="image/png"
+                        )
+                    )
+                elif isinstance(block, dict) and block.get("type") == "audio":
+                    data, mime = _audio_bytes(block.get("audio"))
+                    parts.append(types.Part.from_bytes(data=data, mime_type=mime))
+                elif isinstance(block, dict) and block.get("type") == "text":
+                    parts.append(types.Part(text=block.get("text", "")))
+        else:
+            parts.append(types.Part(text=content if isinstance(content, str) else str(content)))
+        contents.append(types.Content(role="user", parts=parts))
+    return system, contents
+
+
+def _to_genai_tools(tools: Optional[list], types: Any) -> Optional[list]:
+    """OpenAI tool schemas -> genai Tool with FunctionDeclarations.
+
+    Prefers ``parameters_json_schema`` (raw JSON schema passthrough, newer
+    SDKs); falls back to ``parameters`` coercion on older ones.
+    """
+    if not tools:
+        return None
+    decls = []
+    for t in tools:
+        fn = t.get("function", t)
+        try:
+            decls.append(
+                types.FunctionDeclaration(
+                    name=fn.get("name", ""),
+                    description=fn.get("description", ""),
+                    parameters_json_schema=fn.get("parameters"),
+                )
+            )
+        except Exception:  # older SDK without parameters_json_schema
+            decls.append(
+                types.FunctionDeclaration(
+                    name=fn.get("name", ""),
+                    description=fn.get("description", ""),
+                    parameters=fn.get("parameters"),
+                )
+            )
+    return [types.Tool(function_declarations=decls)]
+
+
+class GeminiModel(BlackboxModel):
+    """Black-box wrapper around the Google Gemini API.
+
+    Args:
+        model_id:  Gemini model name (default: ``"gemini-2.5-flash"``).
+        api_key:   API key.  Falls back to the ``GEMINI_API_KEY`` environment
+                   variable when ``None``.
+        client:    An injected ``genai.Client``-shaped object (tests, custom
+                   transports); lazily created from *api_key* when ``None``.
+
+    Requires ``pip install google-genai`` (``evalrx[gemini]``).
+    """
+
+    capabilities = frozenset({Capability.GENERATE, Capability.TOOL_CALLS})
+    modalities   = frozenset({"text", "image"})
+    tool_call_style = "native"  # codec_for -> OpenAIToolCodec (structured calls)
+
+    def __init__(
+        self,
+        model_id: str = "gemini-2.5-flash",
+        api_key: str | None = None,
+        client: Any = None,
+    ) -> None:
+        super().__init__(
+            model_id=model_id,
+            api_key=api_key or os.getenv("GEMINI_API_KEY"),
+        )
+        self._client = client
+
+    # -- plumbing ------------------------------------------------------
+    def _genai(self):
+        try:
+            from google import genai
+            from google.genai import types
+        except ImportError as exc:
+            raise ImportError(
+                "GeminiModel requires the google-genai package. "
+                "Install it with: pip install 'evalrx[gemini]'"
+            ) from exc
+        if self._client is None:
+            if not self.api_key:
+                raise ValueError(
+                    "No Gemini API key found. Pass api_key= or set GEMINI_API_KEY."
+                )
+            self._client = genai.Client(api_key=self.api_key)
+        return self._client, types
+
+    # -- interface -----------------------------------------------------
+    def generate(self, inputs: Any, **kwargs) -> str:
+        client, types = self._genai()
+        image = getattr(inputs, "image", None) if isinstance(inputs, Inputs) else None
+        prompt = inputs.prompt if isinstance(inputs, Inputs) else str(inputs)
+        contents: list = []
+        if image is not None:
+            contents.append(types.Part.from_bytes(data=_png_bytes(image), mime_type="image/png"))
+        contents.append(prompt)
+        response = client.models.generate_content(
+            model=self.model_id, contents=contents, **kwargs
+        )
+        return response.text or ""
+
+    def chat(self, messages: list, tools=None) -> ChatTurn:
+        """One tool-aware turn via native Gemini function calling."""
+        client, types = self._genai()
+        system, contents = _to_genai_contents(messages, types)
+        config = types.GenerateContentConfig(
+            system_instruction=system,
+            tools=_to_genai_tools(tools, types),
+            temperature=0.0,  # diagnosed runs should be as deterministic as the API allows
+        )
+        response = client.models.generate_content(
+            model=self.model_id, contents=contents, config=config
+        )
+
+        texts: list[str] = []
+        raw_calls: list[dict] = []
+        candidates = getattr(response, "candidates", None) or []
+        parts = getattr(candidates[0].content, "parts", None) or [] if candidates else []
+        for part in parts:
+            if getattr(part, "text", None):
+                texts.append(part.text)
+            fc = getattr(part, "function_call", None)
+            if fc is not None:
+                raw_calls.append(
+                    {
+                        "id": getattr(fc, "id", None),
+                        "function": {
+                            "name": fc.name,
+                            "arguments": json.dumps(dict(fc.args or {})),
+                        },
+                    }
+                )
+        um = getattr(response, "usage_metadata", None)
+        usage = (
+            {
+                "prompt_tokens": int(getattr(um, "prompt_token_count", 0) or 0),
+                "completion_tokens": int(getattr(um, "candidates_token_count", 0) or 0),
+            }
+            if um is not None
+            else None
+        )
+        return ChatTurn(text="".join(texts), raw_tool_calls=raw_calls or None, usage=usage)
+
+    def forward(self, inputs: Any, capture: set[Capability], spec: Any = None) -> Trace:  # type: ignore[override]
+        raise NotImplementedError(
+            "GeminiModel is a black-box model; only generate()/chat() are available."
+        )
+
+    def __repr__(self) -> str:
+        return f"GeminiModel(model_id={self.model_id!r})"

evalrx/models/blackbox/llm_api.py

@@ -0,0 +1,17 @@
+"""Black-box LLM via API (OpenAI, Anthropic, Google, etc.).
+
+Planned for Stage 2.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from evalrx.models.blackbox.base import BlackboxModel
+
+
+class BlackboxLLM(BlackboxModel):
+    """API-based LLM — no local weights, output-level capabilities only."""
+
+    def generate(self, inputs: Any, **kwargs) -> str:
+        raise NotImplementedError("BlackboxLLM is planned for Stage 2.")

evalrx/models/blackbox/vlm_api.py

@@ -0,0 +1,17 @@
+"""Black-box VLM via API (GPT-4V, Gemini Vision, etc.).
+
+Planned for Stage 2.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from evalrx.models.blackbox.base import BlackboxModel
+
+
+class BlackboxVLM(BlackboxModel):
+    """API-based vision-language model."""
+
+    def generate(self, inputs: Any, **kwargs) -> str:
+        raise NotImplementedError("BlackboxVLM is planned for Stage 2.")

evalrx/models/compose.py

@@ -0,0 +1,66 @@
+"""compose() — combine a ModelSpec with a Backend into a Model.
+
+This is where the orthogonal decomposition becomes a single object:
+
+    handle = compose(spec, "hf_local", want={Capability.ATTENTION})
+
+Capability negotiation happens HERE (before any weights load): if you ask for
+``ATTENTION`` against the ``api`` backend, you get a clear :class:`CapabilityError`
+up front rather than a crash deep inside a hook.  ``api_only`` specs (closed
+weights) refuse every backend except ``api``.
+"""
+
+from __future__ import annotations
+
+from typing import Iterable
+
+from evalrx.core.capability import Capability, CapabilityError
+from evalrx.models.backends import BACKENDS, Backend, RuntimeConfig
+
+
+def negotiate(handle, want: Iterable[Capability], model_id: str, where: str):
+    """Raise ``CapabilityError`` if *handle* cannot provide every wanted capability.
+
+    Shared by :func:`compose` (spec/registry path) and :func:`evalrx.wrap`
+    (live-model path) so both negotiate capabilities identically.
+    """
+    missing = set(want) - set(handle.capabilities)
+    if missing:
+        raise CapabilityError(analyzer=where, model=model_id, missing=missing)
+    return handle
+
+
+def compose(
+    spec,
+    backend: "str | Backend" = "hf_local",
+    runtime: RuntimeConfig | None = None,
+    want: Iterable[Capability] = (),
+):
+    """Build a Model for *spec* on *backend*, negotiating capabilities first.
+
+    Args:
+        spec:    a :class:`~evalrx.core.spec.ModelSpec` (or a registry key str).
+        backend: backend name (``"api"`` / ``"hf_local"`` / ``"vllm_offline"``) or instance.
+        runtime: :class:`RuntimeConfig`; defaults applied if omitted.
+        want:    capabilities the caller needs; raises if the backend can't provide them.
+    """
+    if isinstance(spec, str):
+        from evalrx.specs import get_spec
+
+        spec = get_spec(spec)
+    if isinstance(backend, str):
+        if backend not in BACKENDS:
+            raise KeyError(f"Unknown backend {backend!r}. Known: {sorted(BACKENDS)}")
+        backend = BACKENDS[backend]()
+    runtime = runtime or RuntimeConfig()
+
+    if spec.api_only and backend.kind != "api":
+        raise ValueError(
+            f"{spec.key!r} is api-only (closed weights); cannot use backend {backend.kind!r}."
+        )
+
+    # Build first (lazy — no weights load), then negotiate against the ACTUAL
+    # handle capabilities.  This is precise for conditional caps like TOOL_CALLS,
+    # which depend on the model (chat template), not just the backend.
+    handle = backend.build(spec, runtime)
+    return negotiate(handle, want, model_id=spec.key, where=f"request@{backend.kind}")

evalrx/models/inference.py

@@ -0,0 +1,88 @@
+"""Infer a :class:`~evalrx.core.spec.ModelSpec` from a live, already-loaded model.
+
+The curated path (``evalrx.load("qwen...")``) looks a spec up by registry key.
+The public on-ramp (``evalrx.wrap(model, tokenizer)``) has no key — the user
+brings their own loaded HF model — so we *infer* the same ``ModelSpec`` fields from
+the live ``model.config`` and tokenizer instead.  Both paths then feed the identical
+:class:`~evalrx.models.backends.hf_local.HFLocalModel`, so every analyzer sees one
+contract.
+
+This module is torch-free: it only reads attributes off objects the caller already
+constructed.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from evalrx.core.spec import AttnSemantics, ModelSpec
+
+
+def _is_vlm(config: Any) -> bool:
+    """Best-effort detection of a vision-language config.
+
+    A text decoder-only config has no separate vision tower; VLM configs carry a
+    nested ``vision_config`` (Qwen-VL, LLaVA, GLM-V, …) or expose a distinct text
+    sub-config via ``get_text_config()``.
+    """
+    if getattr(config, "vision_config", None) is not None:
+        return True
+    if hasattr(config, "get_text_config"):
+        try:
+            text_cfg = config.get_text_config()
+        except Exception:  # pragma: no cover - defensive
+            text_cfg = None
+        if text_cfg is not None and text_cfg is not config:
+            return True
+    return False
+
+
+def _chat_template(tokenizer: Any) -> str:
+    """Return the tokenizer's chat template string (``""`` if none)."""
+    tok = getattr(tokenizer, "tokenizer", tokenizer)  # processor -> inner tokenizer
+    return getattr(tok, "chat_template", None) or ""
+
+
+def infer_spec(model: Any, tokenizer: Any) -> ModelSpec:
+    """Build a :class:`ModelSpec` describing an already-loaded HF causal LM.
+
+    Args:
+        model:     an instantiated ``transformers`` decoder-only model.
+        tokenizer: its tokenizer (or processor).
+
+    Returns:
+        A ``ModelSpec`` carrying only the fields ``HFLocalModel`` reads at analysis
+        time (identity, ``tool_calling``, ``attn_semantics``).  ``hf_repo`` is left
+        empty because the weights are already in memory — ``HFLocalModel`` never
+        calls ``load()`` on a wrapped model.
+
+    Raises:
+        NotImplementedError: if *model* looks like a VLM.  ``wrap()`` supports text
+            decoder-only models in this pass; VLM forward-capture (image tokens +
+            TokenTypeMap) is Stage 2 — consistent with ``HFLocalModel.forward``.
+    """
+    config = getattr(model, "config", None)
+    if config is None:
+        raise ValueError(
+            "wrap() expects a transformers model with a .config; "
+            f"got {type(model).__name__} with none."
+        )
+    if _is_vlm(config):
+        raise NotImplementedError(
+            "wrap() supports text decoder-only models; VLM internals capture "
+            "(image tokens + TokenTypeMap) is Stage 2."
+        )
+
+    model_type = getattr(config, "model_type", None) or "unknown"
+    key = getattr(config, "_name_or_path", "") or model_type or "wrapped-model"
+    tool_calling = "tools" in _chat_template(tokenizer)
+
+    return ModelSpec(
+        key=key,
+        family=model_type,
+        model_type=model_type,
+        hf_repo="",  # already in memory — never reloaded
+        tool_calling=tool_calling,
+        eager_required_for_attn=True,
+        attn_semantics=AttnSemantics.STANDARD,
+    )

evalrx/models/paper_methods/__init__.py

@@ -0,0 +1,8 @@
+"""Architecture-aware executors for published model-health repairs.
+
+These modules are deliberately small and explicit.  A method is exposed here
+only when its model access contract can be checked by the backend; an example
+or benchmark runner must not silently relabel a prompt trick as the paper
+method.
+"""
+

evalrx/models/paper_methods/aad.py

@@ -0,0 +1,53 @@
+"""Hugging Face adapter for AAD's released per-token contrastive sampler.
+
+AAD (Hsu et al. 2025, arXiv:2506.07233, "Reducing Object Hallucination in
+Large Audio-Language Models via Audio-Aware Decoding") contrasts the
+with-audio next-token distribution against the SAME prompt with the audio
+waveform replaced by silence (zeros), at every decode step:
+
+    logits' = (1 + alpha) * logit(y_t | audio, text, y<t)
+              - alpha * logit(y_t | silence, text, y<t)
+
+No adaptive-plausibility cutoff -- unlike VCD, the released formula (and this
+adapter, to stay faithful to it) does not mask sub-threshold candidates.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+class AADLogitsProcessor:
+    """Contrast clean (real-audio) scores with a cached silent-audio pass.
+
+    Mirrors VCDLogitsProcessor's KV-cache reuse (the released reference
+    implementation reruns the full growing sequence every step; caching the
+    silent branch's own past_key_values gets the identical formula without
+    that O(n^2) cost).
+    """
+
+    def __init__(self, model: Any, silent_inputs: dict[str, Any], *, alpha: float) -> None:
+        self.model = model
+        self.silent_inputs = silent_inputs
+        self.alpha = float(alpha)
+        self._silent_output: Any = None
+
+    def __call__(self, input_ids: Any, scores: Any) -> Any:
+        import torch
+
+        with torch.no_grad():
+            if self._silent_output is None:
+                self._silent_output = self.model(
+                    **self.silent_inputs,
+                    use_cache=True,
+                    return_dict=True,
+                )
+            else:
+                self._silent_output = self.model(
+                    input_ids=input_ids[:, -1:],
+                    past_key_values=self._silent_output.past_key_values,
+                    use_cache=True,
+                    return_dict=True,
+                )
+        silent_scores = self._silent_output.logits[:, -1, :]
+        return (1.0 + self.alpha) * scores - self.alpha * silent_scores