modeljury 0.1.0__py3-none-any.whl

modeljury/__init__.py

@@ -0,0 +1,4 @@
+from .claude import Claude
+from .jury import PROMPT, Juror, Result, Vote, convene
+
+__all__ = ["PROMPT", "Claude", "Juror", "Result", "Vote", "convene"]

modeljury/claude.py

@@ -0,0 +1,35 @@
+"""Claude jurors through the native Anthropic Messages API."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+from .jury import Juror
+
+
+@dataclass
+class Claude(Juror):
+    """A Claude model, e.g. Claude("claude-opus-5-5").
+
+    Credentials come from ANTHROPIC_API_KEY or an `ant auth login` profile
+    unless api_key is given. Panel-wide kwargs are OpenAI parameters, so they
+    are not sent here; put Messages API options in params instead, e.g.
+    params={"output_config": {"effort": "low"}}. Requires modeljury[claude].
+    """
+
+    def ask(self, prompt: str, timeout: float, params: dict[str, Any]) -> str:
+        if self.client is None:
+            from anthropic import Anthropic
+
+            kwargs = {"api_key": self.api_key, "base_url": self.base_url}
+            self.client = Anthropic(**{k: v for k, v in kwargs.items() if v is not None})
+        merged = {"max_tokens": 16000, "timeout": timeout, **self.params}
+        resp = self.client.messages.create(
+            **{k: v for k, v in merged.items() if v is not None},
+            model=self.model,
+            messages=[{"role": "user", "content": prompt}],
+        )
+        if resp.stop_reason == "refusal":
+            raise RuntimeError(f"refused: {getattr(resp.stop_details, 'category', None)}")
+        return "".join(b.text for b in resp.content if b.type == "text")

modeljury/jury.py

@@ -0,0 +1,248 @@
+"""Convene a panel of models, count the votes, and flag decisions that need a human."""
+
+from __future__ import annotations
+
+import difflib
+import json
+import os
+import warnings
+from collections import Counter
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass, field
+from typing import Any, Sequence
+
+PROMPT = """You are one juror on a panel. Decide the question using only the evidence.
+
+Question: {question}
+
+Evidence:
+{evidence}
+
+Allowed options: {options}
+
+Reply with only a JSON object:
+{{"choice": "<one allowed option, exactly as written>", "confidence": <0.0-1.0>, "reason": "<one line>"}}"""
+
+
+@dataclass
+class Juror:
+    """A model behind any OpenAI-compatible chat completions endpoint.
+
+    base_url and api_key default to OPENAI_BASE_URL and OPENAI_API_KEY.
+    params are passed straight to chat.completions.create (temperature,
+    max_tokens, reasoning_effort, extra_body, ...) and override the panel's.
+    Set a param to None to leave it out, e.g. {"temperature": None} for
+    reasoning models that reject temperature.
+    """
+
+    model: str
+    base_url: str | None = None
+    api_key: str | None = None
+    name: str | None = None
+    params: dict[str, Any] = field(default_factory=dict)
+    client: Any = field(default=None, repr=False)
+
+    def __post_init__(self) -> None:
+        self.name = self.name or self.model
+
+    def ask(self, prompt: str, timeout: float, params: dict[str, Any]) -> str:
+        if self.client is None:
+            from openai import OpenAI
+
+            self.client = OpenAI(
+                base_url=self.base_url or os.environ.get("OPENAI_BASE_URL"),
+                api_key=self.api_key or os.environ.get("OPENAI_API_KEY", "none"),
+            )
+        merged = {"temperature": 0, "timeout": timeout, **params, **self.params}
+        resp = self.client.chat.completions.create(
+            **{k: v for k, v in merged.items() if v is not None},
+            model=self.model,
+            messages=[{"role": "user", "content": prompt}],
+        )
+        return resp.choices[0].message.content or ""
+
+
+@dataclass
+class Vote:
+    juror: str
+    choice: str | None
+    confidence: float | None = None
+    reason: str = ""
+    error: str | None = None
+
+
+@dataclass
+class Result:
+    verdict: str | None
+    votes: list[Vote]
+    agreement: float
+    needs_review: bool
+
+    @property
+    def majority(self) -> bool:
+        """True when the verdict won more than half of the whole panel, failed jurors included."""
+        won = sum(v.choice == self.verdict for v in self.votes)
+        return self.verdict is not None and won * 2 > len(self.votes)
+
+    @property
+    def dissent(self) -> list[tuple[str, str, str]]:
+        return [
+            (v.juror, v.choice, v.reason)
+            for v in self.votes
+            if v.choice is not None and v.choice != self.verdict
+        ]
+
+    @property
+    def failed(self) -> list[Vote]:
+        return [v for v in self.votes if v.choice is None]
+
+
+def convene(
+    question: str,
+    evidence: str,
+    options: Sequence[str],
+    jurors: Sequence[Juror | str],
+    timeout: float = 60.0,
+    prompt: str = PROMPT,
+    **params: Any,
+) -> Result:
+    """Poll every juror in parallel and return the verdict with the full record.
+
+    needs_review is True unless every juror returned the same valid choice.
+    timeout applies to each attempt; the SDKs retry failed calls twice.
+
+    prompt is a str.format template with {question}, {evidence} and {options};
+    double any literal braces. Extra keyword arguments (temperature=0.3,
+    max_tokens=200, ...) go to every juror's chat.completions.create call;
+    a Juror's own params override them. temperature defaults to 0. Claude jurors
+    (any "claude-..." string, or Claude(...)) skip panel kwargs; see Claude.
+    """
+    if len(options) < 2:
+        raise ValueError("need at least two options")
+    if not jurors:
+        raise ValueError("need at least one juror")
+    if len(jurors) == 1:
+        warnings.warn(
+            "a single juror can't dissent, so needs_review only catches failures; use 3 or more jurors",
+            stacklevel=2,
+        )
+
+    panel = [j if isinstance(j, Juror) else _juror(j) for j in jurors]
+    names = _unique_names(panel)
+    text = prompt.format(
+        question=question, evidence=evidence, options=json.dumps(list(options))
+    )
+
+    with ThreadPoolExecutor(max_workers=len(panel)) as pool:
+        votes = list(
+            pool.map(lambda j, n: _poll(j, n, text, options, timeout, params), panel, names)
+        )
+
+    return tally(votes)
+
+
+def _juror(model: str) -> Juror:
+    if model.startswith("claude-"):
+        from .claude import Claude
+
+        return Claude(model)
+    return Juror(model)
+
+
+def _unique_names(panel: list[Juror]) -> list[str]:
+    """Juror names, with #2, #3, ... added to repeats so votes stay attributable."""
+    seen: Counter[str] = Counter()
+    names = []
+    for j in panel:
+        seen[j.name] += 1
+        names.append(j.name if seen[j.name] == 1 else f"{j.name}#{seen[j.name]}")
+    return names
+
+
+def tally(votes: list[Vote]) -> Result:
+    counts = Counter(v.choice for v in votes if v.choice is not None)
+    if not counts:
+        return Result(verdict=None, votes=votes, agreement=0.0, needs_review=True)
+
+    ranked = counts.most_common()
+    top_choice, top_count = ranked[0]
+    hung = len(ranked) > 1 and ranked[1][1] == top_count
+    agreement = top_count / sum(counts.values())
+    failed = any(v.choice is None for v in votes)
+
+    return Result(
+        verdict=None if hung else top_choice,
+        votes=votes,
+        agreement=agreement,
+        needs_review=hung or failed or agreement < 1.0,
+    )
+
+
+def _poll(
+    juror: Juror,
+    name: str,
+    prompt: str,
+    options: Sequence[str],
+    timeout: float,
+    params: dict[str, Any],
+) -> Vote:
+    try:
+        raw = juror.ask(prompt, timeout, params)
+    except Exception as e:  # one bad juror must not sink the panel
+        error = f"{type(e).__name__}: {e}{_model_hint(juror, e)}"
+        return Vote(juror=name, choice=None, error=error)
+    return parse_vote(name, raw, options)
+
+
+def _model_hint(juror: Juror, e: Exception) -> str:
+    """On a model-not-found error, list the endpoint's models. Best effort; "" if unknown."""
+    status = getattr(e, "status_code", None)
+    if status != 404 and not (status == 400 and "model" in str(e).lower()):
+        return ""
+    try:
+        ids = sorted(m.id for m in juror.client.models.list(timeout=10))
+    except Exception:
+        return ""
+    if not ids or juror.model in ids:
+        return ""
+    close = difflib.get_close_matches(juror.model, ids, n=3, cutoff=0.3)
+    if len(ids) <= 20:
+        return f"; available models: {', '.join(ids)}"
+    hint = f"; did you mean {', '.join(close)}?" if close else ""
+    return f"{hint} ({len(ids)} models available)"
+
+
+def _last_json_object(raw: str) -> dict | None:
+    """The last JSON object in raw, skipping prose, code fences and <think> blocks."""
+    decoder = json.JSONDecoder()
+    found, i = None, raw.find("{")
+    while i != -1:
+        try:
+            found, end = decoder.raw_decode(raw, i)
+        except json.JSONDecodeError:
+            end = i + 1
+        i = raw.find("{", end)
+    return found
+
+
+def parse_vote(juror: str, raw: str, options: Sequence[str]) -> Vote:
+    data = _last_json_object(raw)
+    if not isinstance(data, dict):
+        return Vote(juror=juror, choice=None, error=f"unparseable reply: {raw[:200]!r}")
+
+    choice = str(data.get("choice", "")).strip()
+    by_lower = {o.lower(): o for o in options}
+    if choice.lower() not in by_lower:
+        return Vote(juror=juror, choice=None, error=f"choice not in options: {choice!r}")
+
+    try:
+        confidence = float(data["confidence"])
+    except (KeyError, TypeError, ValueError):
+        confidence = None
+
+    return Vote(
+        juror=juror,
+        choice=by_lower[choice.lower()],
+        confidence=confidence,
+        reason=str(data.get("reason", "")).strip(),
+    )

modeljury-0.1.0.dist-info/METADATA

@@ -0,0 +1,271 @@
+Metadata-Version: 2.5
+Name: modeljury
+Version: 0.1.0
+Summary: Ask several models instead of one. Ship when they agree, send to a human when they don't.
+Project-URL: Homepage, https://github.com/YasserManss/modeljury
+Project-URL: Repository, https://github.com/YasserManss/modeljury
+Project-URL: Issues, https://github.com/YasserManss/modeljury/issues
+Author: YasserManss
+License-Expression: MIT
+License-File: LICENSE
+Keywords: ensemble,evaluation,human-in-the-loop,llm,llm-as-a-judge,voting
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3 :: Only
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.10
+Requires-Dist: openai>=1.0
+Provides-Extra: claude
+Requires-Dist: anthropic>=1.11.0; extra == 'claude'
+Description-Content-Type: text/markdown
+
+# modeljury
+
+Ask several models instead of one. Ship when they agree, send to a human when they don't.
+
+## Getting started
+
+Python 3.10 or newer.
+
+```sh
+pip install modeljury
+# with Claude support
+pip install "modeljury[claude]"
+```
+
+Point it at any OpenAI-compatible endpoint (OpenAI, Ollama, vLLM, OpenRouter, ...) and
+convene a panel. This one uses three models served by a local Ollama:
+
+```python
+from modeljury import Juror, convene
+
+local = "http://localhost:11434/v1"
+result = convene(
+    question="Is this review spam?",
+    evidence="'Best product ever!!! Visit cheap-deals.example for 90% off!!!'",
+    options=["spam", "not spam"],
+    jurors=[Juror(m, base_url=local) for m in ["llama3.1", "qwen3", "mistral"]],
+)
+
+print(result.verdict, result.needs_review)
+for juror, choice, reason in result.dissent:
+    print(f"{juror} disagreed: {choice}, {reason}")
+```
+
+For hosted models, set `OPENAI_API_KEY` (and `OPENAI_BASE_URL` for non-OpenAI providers) or
+pass `api_key=` and `base_url=` to each `Juror`. For Claude, set `ANTHROPIC_API_KEY`.
+
+## Usage
+
+```python
+from modeljury import Juror, convene
+
+result = convene(
+    question="Is this refund request fraudulent?",
+    evidence=ticket_text,
+    options=["fraud", "legit"],
+    jurors=[
+        Juror("gpt-4o-mini"),
+        Juror("llama3.1", base_url="http://localhost:11434/v1"),
+        Juror("mistral-small", base_url="https://api.mistral.ai/v1", api_key="..."),
+    ],
+)
+
+result.verdict       # "legit", or None on a hung jury
+result.majority      # True: the verdict won more than half of the whole panel
+result.needs_review  # True
+result.dissent       # [("mistral-small", "fraud", "Shipping address changed after purchase")]
+result.failed        # jurors that errored or gave an invalid answer
+```
+
+Any endpoint that speaks the OpenAI chat completions API works. A plain string juror
+(`"gpt-4o-mini"`) uses `OPENAI_BASE_URL` and `OPENAI_API_KEY`.
+
+## Examples
+
+These are real runs: five copies of one local 27B model, sampled at `temperature=1.0`.
+The evidence is slightly trimmed here, and the quoted reasons are the model's own words.
+A mixed panel of different models would disagree more often. See the warning at the end.
+
+### Refund with mixed signals
+
+```python
+convene(
+    question="Should this refund be approved?",
+    evidence="Customer of 6 years, 40 prior orders, 2 prior refunds (both legitimate). "
+    "Claims a $1,900 laptop arrived with a cracked screen. The photo's EXIF date is "
+    "2 days before delivery. Courier marked it 'left at door', no signature. "
+    "Customer says their phone clock is wrong and offers to send the laptop back.",
+    options=["approve", "deny", "escalate"],
+    jurors=jurors,
+)
+```
+
+**escalate, 4–1, needs review.** The majority wanted the EXIF date checked. The dissent
+made the case for approving:
+
+> ✗ approve: *Strong 6-year history with only 2 prior refunds outweighs the EXIF
+> inconsistency, which is plausibly explained by the customer's claim of a misconfigured
+> phone clock.*
+
+### Bug triage across four teams
+
+```python
+convene(
+    question="Which team should own this bug report?",
+    evidence="'After upgrading to v4.2, CSV exports from the dashboard are missing every "
+    "row created after 11pm. Only affects customers in Australia. Started the same day we "
+    "moved the export job to a new server.'",
+    options=["Frontend", "Data", "Infra", "Billing"],
+    jurors=jurors,
+)
+```
+
+**Data, 4–1, needs review.** Four jurors blamed timezone logic in the export pipeline.
+The dissent was the only one that used the strongest clue:
+
+> ✗ Infra: *The bug started immediately after moving the export job to a new server,
+> and the time-based filtering error likely stems from a timezone configuration issue
+> on that new infrastructure.*
+
+A reviewer who reads only the dissent knows where to look first.
+
+### Merging a fast but risky cache
+
+```python
+convene(
+    question="Should this pull request be merged as is?",
+    evidence="Adds a cache in front of the user-permissions lookup: p99 latency drops "
+    "from 900ms to 40ms. TTL is 10 minutes. Tests pass. A revoked admin keeps admin "
+    "rights for up to 10 minutes. Revocations happen about 3 times a month, and last "
+    "week's outage was caused by the slow lookup.",
+    options=["merge", "request changes", "merge with follow-up ticket"],
+    jurors=jurors,
+)
+```
+
+**merge with follow-up ticket, 5–0, ships.** All five weighed the latency gain against
+the 10-minute window and asked for a ticket to add cache invalidation on revocation.
+
+### Satire or threat
+
+```python
+convene(
+    question="Does this post violate the policy against threats of violence?",
+    evidence="Policy: remove credible threats of violence. Satire and hyperbole are allowed. "
+    "Post, replying to a council member's parking fee increase: 'If they raise parking "
+    "again I swear I'll show up at the next meeting and make sure he never votes on "
+    "anything again 🙂'. Author has no prior violations and often jokes about local politics.",
+    options=["violates", "allowed"],
+    jurors=jurors,
+)
+```
+
+**allowed, 5–0, ships.** This is the warning. The post names a target and a place, and
+a human moderator might well escalate it. All five jurors dismissed it for the same
+reason, mostly the smiley. Copies of one model share the same blind spots, so their
+agreement is weaker evidence than it looks. Use jurors from different model families,
+because a unanimous vote only means something if the jurors could have disagreed.
+
+## Model options
+
+Extra keyword arguments to `convene()` go to every juror's `chat.completions.create` call.
+A juror's own `params` override them. `temperature` defaults to `0`.
+
+```python
+convene(..., temperature=0.3, max_tokens=200, jurors=[
+    Juror("gpt-4o-mini"),                                       # temperature=0.3, max_tokens=200
+    Juror("o4-mini", params={"temperature": None,               # None drops a param
+                             "reasoning_effort": "low"}),       # adds a model-specific option
+    Juror("qwen3", base_url="http://localhost:8000/v1",
+          params={"temperature": 0.7, "extra_body": {"top_k": 20}}),  # overrides temperature
+])
+```
+
+## Claude
+
+Claude uses the native Anthropic Messages API. Install the extra with `pip install modeljury[claude]`.
+
+```python
+from modeljury import Claude
+
+jurors = [
+    "gpt-4o-mini",
+    "claude-sonnet-5-5",                                         # any "claude-..." string works
+    Claude("claude-opus-5-5", params={"output_config": {"effort": "low"}}),
+]
+```
+
+- Credentials come from `ANTHROPIC_API_KEY` or an `ant auth login` profile, unless you pass `api_key=`.
+- Panel-wide kwargs are OpenAI parameters, so Claude jurors don't receive them. Put Messages API
+  options in `params` instead. Current Claude models reject `temperature`, so none is sent.
+- `max_tokens` defaults to 16000. Set it lower with `params={"max_tokens": ...}` if you need a cost cap.
+  Thinking counts toward it.
+- If Claude refuses, the vote is recorded in `result.failed` and the decision goes to review.
+
+## Custom prompt
+
+`prompt=` takes a `str.format` template with `{question}`, `{evidence}` and `{options}`.
+Double any literal braces (`{{` and `}}`). The default is `modeljury.PROMPT`. Your template
+must still ask for `{"choice": ..., "confidence": ..., "reason": ...}` JSON, because that's what the parser reads.
+
+## Rules
+
+1. Each juror picks one allowed option and gives a one-line reason.
+2. The verdict is the option with the most votes.
+3. `needs_review` is true unless every juror returned the same valid choice.
+4. A tie is a hung jury: no verdict, review needed.
+
+`agreement` is the winning option's votes divided by valid votes. A juror that errors, refuses or
+answers with an option that isn't allowed is left out of the count and listed in `result.failed`.
+
+## Outcomes
+
+Three jurors, options `fraud` / `legit`:
+
+| Votes | `verdict` | `agreement` | `majority` | `needs_review` | Why |
+|---|---|---|---|---|---|
+| legit, legit, legit | legit | 100% | ✅ | ❌ | Everyone agrees: ships without review |
+| legit, legit, fraud | legit | 67% | ✅ | ✅ | Dissent; the reason is in `dissent` |
+| legit, fraud, *failed* | `None` | 50% | ❌ | ✅ | Tie, so hung jury |
+| legit, legit, *failed* | legit | 100% | ✅ | ✅ | A juror failed |
+| legit, *failed*, *failed* | legit | 100% | ❌ | ✅ | Only one valid vote out of three |
+| *failed* ×3 | `None` | 0% | ❌ | ✅ | No valid votes |
+
+`verdict` is the plurality winner, so with three or more options it can win without a majority:
+five votes split A, A, B, C, D give verdict A with 40% agreement and `majority` false.
+`majority` counts the whole panel, failed jurors included, so it means more than half of
+all jurors chose the verdict.
+
+`needs_review` is the strict rule. For a looser one, check `majority` or `agreement` yourself:
+
+```python
+if not result.majority:            # ship any decision that more than half the panel backs
+    send_to_human(result)
+```
+
+A single juror can't dissent, so its decisions only go to review when it fails. `convene()` warns if you pass only one juror.
+
+## Notes
+
+- If a model name is wrong, the juror's error in `result.failed` lists the models the endpoint serves, or suggests the closest names when there are more than 20.
+- Jurors with the same name (e.g. `llama3.1` on two endpoints) are recorded as `llama3.1`, `llama3.1#2`, ...
+- `timeout` applies to each attempt. The OpenAI and Anthropic SDKs retry failed calls twice, so one juror can take up to three times `timeout`.
+
+## Develop
+
+```sh
+uv sync
+uv run pytest
+```
+
+## License
+
+MIT. See [LICENSE](https://github.com/YasserManss/modeljury/blob/main/LICENSE).

modeljury-0.1.0.dist-info/RECORD

@@ -0,0 +1,7 @@
+modeljury/__init__.py,sha256=eJjurZcgkA_rHyj42yyr44AXw4xY7iK8SvyGRPzrtLI,152
+modeljury/claude.py,sha256=z3m_ugYGc3PJqQCt0mC2O0dvpv3tMJ4vdUP_wtEPY6I,1393
+modeljury/jury.py,sha256=VU9sHqhTY0Xf5osc_CVP4X1Td4ZnfpokiAIy1NpgOtA,8011
+modeljury-0.1.0.dist-info/METADATA,sha256=k1k8BMMnmPF0vA6v0BFOnaKl92D4hN7Qce2pxVXj7GM,10963
+modeljury-0.1.0.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+modeljury-0.1.0.dist-info/licenses/LICENSE,sha256=t4NshXpywUmS-Sjcjk6z1t_GA_UC8DtTdfydJBtFW7s,1079
+modeljury-0.1.0.dist-info/RECORD,,

modeljury-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.4
+Root-Is-Purelib: true
+Tag: py3-none-any

modeljury-0.1.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 modeljury contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.