evalring 0.2.0__py3-none-any.whl

EvalRing/agent/multi_role.py

@@ -0,0 +1,305 @@
+"""Reusable multi-role host orchestration for classification tasks."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import Any
+
+from .classification import (
+    normalize_probability_distribution,
+    parse_json_object,
+    resolve_classification_prediction,
+)
+
+
+@dataclass
+class RoleConfig:
+    """Configuration for one specialist role in the multi-role system."""
+
+    name: str
+    persona: str
+    model_name: str
+    temperature: float = 0.0
+    max_completion_tokens: int = 400
+
+
+class MultiRoleHostOrchestrator:
+    """Orchestrates role-by-role reviews plus bounded host-led questioning."""
+
+    def __init__(
+        self,
+        *,
+        client: Any,
+        labels: list[str],
+        task_name: str,
+        task_instructions: str,
+        host_model_name: str,
+        role_configs: list[RoleConfig],
+        host_temperature: float = 0.0,
+        host_max_completion_tokens: int = 500,
+        max_iterations: int = 10,
+    ):
+        if max_iterations < 1:
+            raise ValueError("max_iterations must be >= 1")
+        if not role_configs:
+            raise ValueError("role_configs cannot be empty")
+
+        self.client = client
+        self.labels = labels
+        self.task_name = task_name
+        self.task_instructions = task_instructions
+        self.host_model_name = host_model_name
+        self.role_configs = role_configs
+        self.role_map = {role.name: role for role in role_configs}
+        self.host_temperature = host_temperature
+        self.host_max_completion_tokens = host_max_completion_tokens
+        self.max_iterations = max_iterations
+
+    def _call_json(
+        self,
+        *,
+        model_name: str,
+        messages: list[dict[str, str]],
+        temperature: float,
+        max_completion_tokens: int,
+    ) -> dict[str, Any]:
+        response = self.client.chat.completions.create(
+            model=model_name,
+            messages=messages,
+            temperature=temperature,
+            max_completion_tokens=max_completion_tokens,
+            stream=False,
+        )
+        raw_output = response.choices[0].message.content or ""
+        parsed = parse_json_object(raw_output) or {}
+
+        usage = getattr(response, "usage", None)
+        token_usage = {
+            "prompt_tokens": int(getattr(usage, "prompt_tokens", 0) or 0),
+            "completion_tokens": int(getattr(usage, "completion_tokens", 0) or 0),
+            "total_tokens": int(getattr(usage, "total_tokens", 0) or 0),
+        }
+        return {"raw_output": raw_output, "parsed": parsed, "token_usage": token_usage}
+
+    def _one_hot(self, label: str) -> dict[str, float]:
+        return {candidate: (1.0 if candidate == label else 0.0) for candidate in self.labels}
+
+    def _normalize_role_judgment(self, parsed: Mapping[str, Any]) -> dict[str, Any]:
+        class_scores = parsed.get("class_scores")
+        conclusion = str(parsed.get("conclusion", "")).strip()
+
+        if isinstance(class_scores, Mapping):
+            prediction = resolve_classification_prediction(class_scores, valid_labels=self.labels)
+            label = prediction.label or (self.labels[0] if self.labels else "")
+            scores = normalize_probability_distribution(
+                {k: float(v) for k, v in prediction.class_scores.items()}
+                if prediction.class_scores
+                else self._one_hot(label)
+            )
+        else:
+            prediction = resolve_classification_prediction(conclusion, valid_labels=self.labels)
+            label = prediction.label or (self.labels[0] if self.labels else "")
+            scores = self._one_hot(label)
+
+        return {
+            "analysis": str(parsed.get("analysis", "")).strip(),
+            "conclusion": label,
+            "class_scores": scores,
+            "confidence": parsed.get("confidence", None),
+        }
+
+    def _aggregate_scores(self, role_reviews: list[dict[str, Any]]) -> dict[str, float]:
+        sums = dict.fromkeys(self.labels, 0.0)
+        if not role_reviews:
+            return sums
+
+        for review in role_reviews:
+            scores = review.get("class_scores", {})
+            for label in self.labels:
+                sums[label] += float(scores.get(label, 0.0))
+
+        return normalize_probability_distribution(sums)
+
+    def run_case(self, input_text: str) -> dict[str, Any]:
+        role_reviews: list[dict[str, Any]] = []
+        host_dialogue: list[dict[str, Any]] = []
+        total_prompt_tokens = 0
+        total_completion_tokens = 0
+        total_tokens = 0
+
+        labels_text = ", ".join(self.labels)
+
+        # 1) Independent role evaluations.
+        for role in self.role_configs:
+            call = self._call_json(
+                model_name=role.model_name,
+                messages=[
+                    {
+                        "role": "system",
+                        "content": (
+                            f"You are role: {role.name}. Persona: {role.persona}\n"
+                            f"Task: {self.task_name}\n"
+                            f"Allowed labels: {labels_text}\n"
+                            f"Instructions: {self.task_instructions}\n"
+                            "Return ONLY JSON with keys: analysis, conclusion, class_scores, confidence."
+                        ),
+                    },
+                    {"role": "user", "content": f"Case:\n\n{input_text}"},
+                ],
+                temperature=role.temperature,
+                max_completion_tokens=role.max_completion_tokens,
+            )
+            normalized = self._normalize_role_judgment(call["parsed"])
+            role_reviews.append(
+                {
+                    "role": role.name,
+                    "model": role.model_name,
+                    "raw_output": call["raw_output"],
+                    **normalized,
+                }
+            )
+
+            usage = call["token_usage"]
+            total_prompt_tokens += usage["prompt_tokens"]
+            total_completion_tokens += usage["completion_tokens"]
+            total_tokens += usage["total_tokens"]
+
+        # 2) Host iterative questioning (bounded).
+        for iteration in range(1, self.max_iterations + 1):
+            host_call = self._call_json(
+                model_name=self.host_model_name,
+                messages=[
+                    {
+                        "role": "system",
+                        "content": (
+                            "You are the host coordinator for a multi-role classifier.\n"
+                            f"Allowed labels: {labels_text}.\n"
+                            "Decide either:\n"
+                            "1) ask one role one follow-up question, or\n"
+                            "2) finalize with a conclusion.\n"
+                            "Return ONLY JSON with either:\n"
+                            '- {"action":"ask","target_role":"...","question":"..."}\n'
+                            "or\n"
+                            '- {"action":"finalize","conclusion":"...","class_scores":{...},"rationale":"..."}.'
+                        ),
+                    },
+                    {
+                        "role": "user",
+                        "content": (
+                            f"Case:\n{input_text}\n\n"
+                            f"Role reviews JSON:\n{role_reviews}\n\n"
+                            f"Dialogue so far JSON:\n{host_dialogue}\n\n"
+                            f"Iteration: {iteration}/{self.max_iterations}"
+                        ),
+                    },
+                ],
+                temperature=self.host_temperature,
+                max_completion_tokens=self.host_max_completion_tokens,
+            )
+
+            usage = host_call["token_usage"]
+            total_prompt_tokens += usage["prompt_tokens"]
+            total_completion_tokens += usage["completion_tokens"]
+            total_tokens += usage["total_tokens"]
+
+            action = str(host_call["parsed"].get("action", "")).strip().lower()
+            if action == "finalize":
+                final_scores_raw = host_call["parsed"].get("class_scores")
+                if isinstance(final_scores_raw, Mapping):
+                    normalized_scores = normalize_probability_distribution(
+                        {label: float(final_scores_raw.get(label, 0.0)) for label in self.labels}
+                    )
+                else:
+                    label = (
+                        resolve_classification_prediction(
+                            host_call["parsed"].get("conclusion", ""),
+                            valid_labels=self.labels,
+                        ).label
+                        or self.labels[0]
+                    )
+                    normalized_scores = self._one_hot(label)
+
+                final_prediction = resolve_classification_prediction(
+                    normalized_scores, valid_labels=self.labels
+                )
+                return {
+                    "final_label": final_prediction.label,
+                    "class_scores": normalized_scores,
+                    "rationale": str(host_call["parsed"].get("rationale", "")).strip(),
+                    "iterations_used": iteration,
+                    "role_reviews": role_reviews,
+                    "host_dialogue": host_dialogue,
+                    "token_usage": {
+                        "prompt_tokens": total_prompt_tokens,
+                        "completion_tokens": total_completion_tokens,
+                        "total_tokens": total_tokens,
+                    },
+                }
+
+            if action != "ask":
+                break
+
+            target_role = str(host_call["parsed"].get("target_role", "")).strip()
+            question = str(host_call["parsed"].get("question", "")).strip()
+            if not question or target_role not in self.role_map:
+                break
+
+            target_cfg = self.role_map[target_role]
+            role_reply = self._call_json(
+                model_name=target_cfg.model_name,
+                messages=[
+                    {
+                        "role": "system",
+                        "content": (
+                            f"You are role: {target_cfg.name}. Persona: {target_cfg.persona}\n"
+                            f"Allowed labels: {labels_text}\n"
+                            "Answer host questions. Return ONLY JSON with keys: answer, updated_conclusion, class_scores."
+                        ),
+                    },
+                    {
+                        "role": "user",
+                        "content": (
+                            f"Case:\n{input_text}\n\n"
+                            f"Host question:\n{question}\n\n"
+                            f"Your original review context:\n{role_reviews}"
+                        ),
+                    },
+                ],
+                temperature=target_cfg.temperature,
+                max_completion_tokens=target_cfg.max_completion_tokens,
+            )
+
+            usage = role_reply["token_usage"]
+            total_prompt_tokens += usage["prompt_tokens"]
+            total_completion_tokens += usage["completion_tokens"]
+            total_tokens += usage["total_tokens"]
+
+            host_dialogue.append(
+                {
+                    "iteration": iteration,
+                    "target_role": target_role,
+                    "question": question,
+                    "answer": role_reply["parsed"],
+                    "raw_answer": role_reply["raw_output"],
+                }
+            )
+
+        # 3) Fallback finalize if host never finalized.
+        fallback_scores = self._aggregate_scores(role_reviews)
+        fallback_prediction = resolve_classification_prediction(
+            fallback_scores, valid_labels=self.labels
+        )
+        return {
+            "final_label": fallback_prediction.label,
+            "class_scores": fallback_scores,
+            "rationale": "Fallback finalize from averaged role distributions.",
+            "iterations_used": self.max_iterations,
+            "role_reviews": role_reviews,
+            "host_dialogue": host_dialogue,
+            "token_usage": {
+                "prompt_tokens": total_prompt_tokens,
+                "completion_tokens": total_completion_tokens,
+                "total_tokens": total_tokens,
+            },
+        }

EvalRing/cli.py

@@ -0,0 +1,284 @@
+"""
+Command-line interface for EvalRing.
+
+Installed as the ``evalring`` console script. Every subcommand is a thin
+wrapper over the public Python API, so anything reachable here is also
+reachable from code.
+
+Run ``evalring --help`` for the full listing, or see ``docs/CLI.md``.
+"""
+
+from __future__ import annotations
+
+import argparse
+import importlib.util
+import json
+import logging
+import sys
+from collections.abc import Sequence
+from pathlib import Path
+
+from . import __version__
+from .config import CREDENTIAL_ENV_VARS, resolve_credentials, resolve_model_name
+from .dataset.base import BaseDataset
+from .logging_utils import configure_logging, get_logger
+
+logger = get_logger(__name__)
+
+SUPPORTED_DATASET_SUFFIXES = (".json", ".csv")
+
+
+def _load_dataset(
+    path: Path,
+    text_field: str,
+    label_field: str,
+    id_field: str | None,
+) -> BaseDataset:
+    """Load a dataset from disk, picking the reader from the file extension.
+
+    Args:
+        path: Path to a ``.json`` or ``.csv`` dataset file.
+        text_field: Column or key holding the model input.
+        label_field: Column or key holding the ground-truth label.
+        id_field: Column or key holding a unique sample ID, or ``None`` to
+            fall back to the row index.
+
+    Returns:
+        A loaded :class:`~EvalRing.dataset.base.BaseDataset` subclass instance.
+
+    Raises:
+        ValueError: If the file extension has no registered reader.
+    """
+    from .dataset import CSVDataset, JSONDataset
+
+    suffix = path.suffix.lower()
+    dataset: BaseDataset
+    if suffix == ".json":
+        dataset = JSONDataset(name=path.stem)
+    elif suffix == ".csv":
+        dataset = CSVDataset(name=path.stem)
+    else:
+        raise ValueError(
+            f"Unsupported dataset extension {suffix!r}. "
+            f"Supported: {', '.join(SUPPORTED_DATASET_SUFFIXES)}"
+        )
+
+    dataset.load_data(
+        path,
+        text_field=text_field,
+        label_field=label_field,
+        id_field=id_field,
+    )
+    return dataset
+
+
+def _cmd_info(args: argparse.Namespace) -> int:
+    """Print dataset statistics as JSON.
+
+    Args:
+        args: Parsed arguments for the ``info`` subcommand.
+
+    Returns:
+        Process exit code: 0 on success, 1 when the dataset cannot be read.
+    """
+    path = Path(args.dataset)
+    if not path.exists():
+        logger.error("Dataset not found: %s", path)
+        return 1
+
+    try:
+        dataset = _load_dataset(path, args.text_field, args.label_field, args.id_field)
+    except Exception as exc:
+        logger.error("Failed to load %s: %s", path, exc)
+        return 1
+
+    stats = dataset.get_statistics()
+    stats["valid"] = dataset.validate_data()
+    stats["path"] = str(path)
+    print(json.dumps(stats, indent=2, default=str))
+    return 0
+
+
+def _cmd_check(args: argparse.Namespace) -> int:
+    """Report the resolved provider configuration without revealing secrets.
+
+    Args:
+        args: Parsed arguments for the ``check`` subcommand.
+
+    Returns:
+        Process exit code: 0 when a credential was resolved, 1 otherwise.
+    """
+    credentials = resolve_credentials()
+    report = {
+        "evalring_version": __version__,
+        "python": sys.version.split()[0],
+        "api_key_found": bool(credentials.api_key),
+        "api_key_source": credentials.source,
+        "provider": credentials.provider,
+        "base_url": credentials.base_url or "<provider default>",
+        "model": resolve_model_name(default="<unset>"),
+        "recognized_key_variables": list(CREDENTIAL_ENV_VARS),
+    }
+
+    # find_spec only inspects the import system, so this stays fast even when
+    # a heavyweight optional package such as litellm is installed.
+    report["optional_packages"] = {
+        module: importlib.util.find_spec(module) is not None
+        for module in ("litellm", "openai", "matplotlib", "nest_asyncio")
+    }
+
+    print(json.dumps(report, indent=2))
+    if not credentials.api_key:
+        logger.warning("No API key resolved. Set one of: %s", ", ".join(CREDENTIAL_ENV_VARS))
+        return 1
+    return 0
+
+
+def _cmd_run_suite(args: argparse.Namespace) -> int:
+    """Run an evaluation script across every model in a model list.
+
+    Args:
+        args: Parsed arguments for the ``run-suite`` subcommand.
+
+    Returns:
+        Process exit code propagated from
+        :func:`EvalRing.utils.suite_runner.run_suite`.
+    """
+    from .utils.suite_runner import run_suite
+
+    return run_suite(
+        eval_script=args.eval_script,
+        model_list_path=args.models_file,
+        n_samples=args.n_samples,
+        max_workers=args.max_workers,
+        seed=args.seed,
+        agent_mode=args.agent_mode,
+        base_class=args.base_class,
+        host_model=args.host_model,
+        role_models_json=args.role_models_json,
+        max_host_iterations=args.max_host_iterations,
+        continue_runs=args.continue_runs,
+        cache=args.cache,
+        cache_mode=args.cache_mode,
+        ignore_errors=args.ignore_errors,
+        out_dir=args.out_dir,
+        data_path=args.data_path,
+        no_confirm=args.yes,
+    )
+
+
+def _cmd_models(args: argparse.Namespace) -> int:
+    """Generate a model-list JSON file from the OpenRouter catalogue.
+
+    Args:
+        args: Parsed arguments for the ``models`` subcommand.
+
+    Returns:
+        Process exit code: 0 on success, 1 on failure.
+    """
+    from .utils.generate_model_list import generate_model_list
+
+    output = Path(args.output)
+    output.parent.mkdir(parents=True, exist_ok=True)
+    try:
+        generate_model_list(output)
+    except Exception as exc:
+        logger.error("Failed to generate model list: %s", exc)
+        return 1
+    return 0
+
+
+def build_parser() -> argparse.ArgumentParser:
+    """Construct the top-level argument parser.
+
+    Returns:
+        The fully configured :class:`argparse.ArgumentParser`.
+    """
+    parser = argparse.ArgumentParser(
+        prog="evalring",
+        description="Evaluate agents and models across providers.",
+    )
+    parser.add_argument("--version", action="version", version=f"evalring {__version__}")
+    parser.add_argument("-v", "--verbose", action="store_true", help="Enable debug logging.")
+    parser.add_argument("-q", "--quiet", action="store_true", help="Only log warnings and errors.")
+
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    info = sub.add_parser("info", help="Show statistics for a dataset file.")
+    info.add_argument("--dataset", required=True, help="Path to a .json or .csv dataset.")
+    info.add_argument("--text-field", default="text", help="Input text column/key.")
+    info.add_argument("--label-field", default="label", help="Ground-truth column/key.")
+    info.add_argument("--id-field", default=None, help="Unique sample ID column/key.")
+    info.set_defaults(func=_cmd_info)
+
+    check = sub.add_parser(
+        "check",
+        help="Report the resolved provider configuration (no secrets printed).",
+    )
+    check.set_defaults(func=_cmd_check)
+
+    models = sub.add_parser("models", help="Generate a model-list JSON file.")
+    models.add_argument(
+        "--output",
+        default="model_list.json",
+        help="Destination path for the model list.",
+    )
+    models.set_defaults(func=_cmd_models)
+
+    suite = sub.add_parser(
+        "run-suite",
+        help="Run an evaluation script across every model in a model list.",
+    )
+    suite.add_argument("--eval-script", required=True, help="Path to the evaluation script.")
+    suite.add_argument("--models-file", required=True, help="Path to the model list JSON.")
+    suite.add_argument("--n-samples", type=int, default=10)
+    suite.add_argument("--max-workers", type=int, default=5)
+    suite.add_argument("--seed", type=int, default=42)
+    suite.add_argument("--agent-mode", default="single-class")
+    suite.add_argument("--base-class", default=None)
+    suite.add_argument("--host-model", default=None)
+    suite.add_argument("--role-models-json", default=None)
+    suite.add_argument("--max-host-iterations", type=int, default=10)
+    suite.add_argument("--continue-runs", action="store_true")
+    suite.add_argument("--cache", default=None)
+    suite.add_argument(
+        "--cache-mode",
+        default="both",
+        choices=["runs_only", "cache_file", "both", "none"],
+    )
+    suite.add_argument("--ignore-errors", action="store_true")
+    suite.add_argument("--out-dir", default=None)
+    suite.add_argument("--data-path", default=None)
+    suite.add_argument(
+        "-y", "--yes", action="store_true", help="Skip the pre-run cache confirmation."
+    )
+    suite.set_defaults(func=_cmd_run_suite)
+
+    return parser
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    """Entry point for the ``evalring`` console script.
+
+    Args:
+        argv: Argument vector to parse. Defaults to ``sys.argv[1:]``.
+
+    Returns:
+        Process exit code.
+    """
+    parser = build_parser()
+    args = parser.parse_args(argv)
+
+    if args.verbose:
+        level = logging.DEBUG
+    elif args.quiet:
+        level = logging.WARNING
+    else:
+        level = logging.INFO
+    configure_logging(level, verbose=args.verbose)
+
+    return int(args.func(args) or 0)
+
+
+if __name__ == "__main__":  # pragma: no cover
+    sys.exit(main())