ags-cli 0.1.0__py3-none-any.whl

harness/services/comparison.py

@@ -0,0 +1,345 @@
+"""Compare-and-merge workflow.
+
+``compare`` produces a side-by-side, metric-annotated view of two or more runs and
+(for ``judge``) a heuristic winner. ``merge`` synthesizes a canonical final answer
+and — when ``persist`` is set — writes it to durable workspace memory, linking the
+``Comparison`` row to the resulting ``memory_item``. This is how multiple providers'
+outputs become one persisted, vendor-neutral outcome.
+"""
+
+from __future__ import annotations
+
+import re
+import uuid
+
+from harness_sdk.base import SessionContext
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from harness.adapters.registry import build_adapter
+from harness.memory.ingest import ingest_memory
+from harness.memory.namespaces import workspace_ns
+from harness.models.comparison import Comparison
+from harness.models.enums import ComparisonStrategy, MemoryScope, MemoryType, RunStatus
+from harness.models.provider_config import ProviderConfig
+from harness.models.run import Run
+from harness.models.session import Session
+from harness.models.workspace import Workspace
+from harness.observability.audit import write_audit
+
+# How much of each candidate's output the judge gets to see. Generous on
+# purpose: a few hundred characters is not enough to tell two answers apart.
+PREVIEW_CHARS = 4000
+
+
+def _provider_of(graph_node_id: str | None) -> str:
+    """Agent name behind a graph node id ('claude-0' -> 'claude')."""
+    return re.sub(r"-\d+$", "", graph_node_id or "") or "unknown"
+
+
+def _winner_from_verdict(verdict: str, scored: list[dict]) -> str | None:
+    """Resolve the judge's prose answer to a run id.
+
+    Prefers the declared ``WINNER: <name>`` line, then any provider name or run
+    id mentioned anywhere in the text. Returns None when nothing matches, which
+    leaves the caller on its heuristic fallback.
+    """
+    if not verdict:
+        return None
+    by_name = {s["provider"].lower(): s["run_id"] for s in scored}
+
+    match = re.search(r"^\s*#*\s*\**\s*WINNER\s*\**\s*:\s*\**\s*(.+?)\s*\**\s*$",
+                      verdict, re.IGNORECASE | re.MULTILINE)
+    if match:
+        claimed = match.group(1).strip().strip("`").lower()
+        if claimed in by_name:
+            return by_name[claimed]
+        # The judge may decorate the name ("local (llama3.1)") — take the
+        # longest candidate name it contains so 'gpt' can't shadow 'gpt-fast'.
+        hits = [n for n in by_name if n and n in claimed]
+        if hits:
+            return by_name[max(hits, key=len)]
+
+    for s in scored:
+        if s["run_id"] in verdict:
+            return s["run_id"]
+    return None
+
+
+def _quality_score(run: Run) -> float:
+    """Cheap, deterministic heuristic for fallback."""
+    if run.status != RunStatus.succeeded:
+        return 0.0
+    out = run.final_output or ""
+    length_score = min(len(out) / 1000.0, 1.0)
+    cost = run.cost or {}
+    tokens = cost.get("total_tokens", 0) or 0
+    efficiency = 1.0 / (1.0 + tokens / 1000.0)
+    return 0.7 * length_score + 0.3 * efficiency
+
+
+async def _load_runs(db: AsyncSession, run_ids: list[uuid.UUID]) -> list[Run]:
+    rows = (await db.execute(select(Run).where(Run.id.in_(run_ids)))).scalars().all()
+    order = {rid: i for i, rid in enumerate(run_ids)}
+    return sorted(rows, key=lambda r: order.get(r.id, 0))
+
+
+async def _pick_provider(db: AsyncSession, name: str | None = None) -> ProviderConfig | None:
+    """The provider to use for an internal (judge/merge) LLM call.
+
+    *name* selects a specific agent; when it is omitted — or names one that is
+    unknown or disabled — this falls back to the first enabled provider, which
+    is the historical behaviour.
+    """
+    if name:
+        pc = (await db.execute(
+            select(ProviderConfig).where(
+                ProviderConfig.name == name, ProviderConfig.enabled.is_(True)
+            ).limit(1)
+        )).scalar_one_or_none()
+        if pc is not None:
+            return pc
+    return (await db.execute(
+        select(ProviderConfig).where(ProviderConfig.enabled.is_(True)).limit(1)
+    )).scalar_one_or_none()
+
+
+async def _invoke_llm(
+    db: AsyncSession,
+    session_id: uuid.UUID,
+    prompt: str,
+    *,
+    pc: ProviderConfig | None = None,
+    model: str | None = None,
+) -> str:
+    """Run a one-off, non-streaming completion for judging/merging.
+
+    *pc* is the provider to ask (resolved by the caller so it can report which
+    agent judged); omitted means the first enabled one. *model* overrides that
+    provider's configured model for this call only.
+    """
+    if pc is None:
+        pc = await _pick_provider(db)
+
+    if not pc:
+        return ""
+
+    if model:
+        from harness_sdk.base import AdapterConfig
+
+        from harness.adapters.registry import build_adapter_from_config
+        from harness.security.secrets import resolve_secret_ref
+        adapter = build_adapter_from_config(AdapterConfig(
+            name=pc.name,
+            kind=pc.kind,
+            base_url=pc.base_url,
+            model=model,
+            api_key=resolve_secret_ref(pc.secret_ref),
+            params=pc.params or {},
+            capabilities=pc.capabilities or {},
+        ))
+    else:
+        adapter = build_adapter(pc)
+    ctx = SessionContext(
+        run_id=str(uuid.uuid4()),
+        session_id=str(session_id),
+        workspace_slug="ephemeral",
+        objective="internal_llm_call",
+        constraints={},
+        memory_context=""
+    )
+    provider_session = await adapter.start_session(ctx)
+    result = ""
+    async for ev in adapter.stream_events(provider_session, prompt, stream=False):
+        if ev.type == "message":
+            result = ev.text
+    return result
+
+
+async def compare_runs(
+    db: AsyncSession,
+    *,
+    run_ids: list[uuid.UUID],
+    strategy: str = "judge",
+    judge_provider: str | None = None,
+    judge_model: str | None = None,
+) -> Comparison:
+    """Score candidate runs and, for ``judge``, ask an LLM to pick a winner.
+
+    *judge_provider* / *judge_model* choose the agent and model doing the
+    judging; both fall back to the first enabled provider on its own model.
+    The judge answers in prose (stored as ``result["verdict"]``) so the user
+    reads a rationale, not a bare id.
+    """
+    runs = await _load_runs(db, run_ids)
+    if not runs:
+        raise ValueError("no runs found for comparison")
+    session_id = runs[0].session_id
+
+    scored = [
+        {
+            "run_id": str(r.id),
+            "provider": _provider_of(r.graph_node_id),
+            "provider_role": r.role.value,
+            "adapter_kind": r.adapter_kind.value,
+            "status": r.status.value,
+            "score": round(_quality_score(r), 4),
+            "output_preview": (r.final_output or "")[:PREVIEW_CHARS],
+            "cost": r.cost or {},
+        }
+        for r in runs
+    ]
+
+    winner_run_id = None
+    verdict = ""
+    judge_meta: dict | None = None
+    if strategy == "judge":
+        pc = await _pick_provider(db, judge_provider)
+        judge_meta = {
+            "provider": pc.name if pc else None,
+            "model": judge_model or (pc.model if pc else None),
+            "requested_provider": judge_provider,
+        }
+        names = ", ".join(s["provider"] for s in scored)
+        prompt = (
+            "You are judging the outputs of several AI agents that were given the "
+            "same task. Decide which one did it best.\n\n"
+            "Answer in plain text (light markdown is fine). Structure it exactly like "
+            "this:\n"
+            f"1. A first line of exactly `WINNER: <name>`, where <name> is one of: {names}.\n"
+            "2. A short paragraph explaining why that one won.\n"
+            "3. One bullet per agent noting its main strength and weakness.\n\n"
+            "Do not output JSON. Do not quote run ids.\n\n"
+        )
+        for s in scored:
+            prompt += (
+                f"## Agent: {s['provider']} ({s['adapter_kind']}, status={s['status']})\n"
+                f"{s['output_preview'] or '(no output)'}\n\n"
+            )
+
+        verdict = (await _invoke_llm(db, session_id, prompt, pc=pc, model=judge_model)).strip()
+        winner_run_id = _winner_from_verdict(verdict, scored)
+
+        if not winner_run_id and scored:
+            # Judge unavailable or unparseable — fall back to the heuristic score.
+            winner_run_id = max(scored, key=lambda s: s["score"])["run_id"]
+
+    winner_provider = next(
+        (s["provider"] for s in scored if s["run_id"] == winner_run_id), None
+    )
+    comparison = Comparison(
+        session_id=session_id,
+        run_ids=run_ids,
+        strategy=ComparisonStrategy(strategy),
+        result={
+            "candidates": scored,
+            "winner_run_id": winner_run_id,
+            "winner_provider": winner_provider,
+            "verdict": verdict,
+            "judge": judge_meta,
+        },
+    )
+    db.add(comparison)
+    await db.flush()
+
+    await write_audit(
+        db,
+        actor="system",
+        action="compare_runs",
+        target_type="comparison",
+        target_id=str(comparison.id),
+        after={
+            "strategy": strategy,
+            "winner_run_id": winner_run_id,
+            "winner_provider": winner_provider,
+            "judge": judge_meta,
+        }
+    )
+
+    return comparison
+
+
+async def merge_runs(
+    db: AsyncSession,
+    *,
+    run_ids: list[uuid.UUID],
+    persist: bool = True,
+    merged_content: str | None = None,
+) -> Comparison:
+    """Merge runs into a canonical answer and optionally persist it as memory."""
+    runs = await _load_runs(db, run_ids)
+    if not runs:
+        raise ValueError("no runs found for merge")
+    session = (
+        await db.execute(select(Session).where(Session.id == runs[0].session_id))
+    ).scalar_one()
+    workspace = (
+        await db.execute(select(Workspace).where(Workspace.id == session.workspace_id))
+    ).scalar_one()
+
+    if merged_content is None:
+        prompt = (
+            "You are an AI merging the outputs of several language model runs into a "
+            "single, canonical, best answer. Synthesize the following outputs into a "
+            "cohesive final answer.\n\n"
+        )
+        for r in runs:
+            if r.final_output:
+                prompt += (
+                    f"### Output from {r.role.value} ({r.adapter_kind.value})\n"
+                    f"{r.final_output}\n\n"
+                )
+
+        ans = await _invoke_llm(db, session.id, prompt)
+        
+        if ans:
+            merged_content = ans.strip()
+        else:
+            # Fallback
+            ranked = sorted(runs, key=_quality_score, reverse=True)
+            sections = [
+                f"## Canonical answer (from {ranked[0].role.value})",
+                ranked[0].final_output or "",
+            ]
+            for r in ranked[1:]:
+                if r.final_output:
+                    sections.append(f"\n### Alternative ({r.role.value}/{r.adapter_kind.value})")
+                    sections.append(r.final_output)
+            merged_content = "\n".join(sections).strip()
+
+    memory_item_id: uuid.UUID | None = None
+    if persist:
+        item = await ingest_memory(
+            db,
+            namespace=workspace_ns(workspace.slug),
+            scope=MemoryScope.workspace,
+            mem_type=MemoryType.decision,
+            content=merged_content,
+            tags=["merged", "canonical", session.canonical_objective[:40]],
+            source_run_id=runs[0].id,
+        )
+        memory_item_id = item.id
+
+    comparison = Comparison(
+        session_id=session.id,
+        run_ids=run_ids,
+        strategy=ComparisonStrategy.merge,
+        result={"merged_content": merged_content, "persisted": persist},
+        canonical_memory_item_id=memory_item_id,
+    )
+    db.add(comparison)
+    await db.flush()
+
+    await write_audit(
+        db,
+        actor="system",
+        action="merge_runs",
+        target_type="comparison",
+        target_id=str(comparison.id),
+        after={
+            "persisted": persist,
+            "canonical_memory_item_id": str(memory_item_id) if memory_item_id else None,
+        },
+    )
+
+    return comparison

harness/services/doctor.py

@@ -0,0 +1,156 @@
+"""Doctor service — end-to-end provider preflight.
+
+Iterates all *enabled* ProviderConfig rows and for each produces a health
+report dict:
+
+    {
+        "provider":        str,           # provider name
+        "kind":            str,           # AdapterKind value string
+        "binary":          str | None,    # resolved binary path (CLI kinds only)
+        "version":         str | None,    # version string from probe (CLI kinds only)
+        "health_ok":       bool,
+        "latency_ms":      float,
+        "model":           str | None,
+        "mcp_ok":          bool | None,   # None for non-CLI kinds
+        "native_sessions": bool,          # whether decide_resume would enable this
+        "problems":        list[str],     # human-readable problem descriptions
+    }
+
+Probe/health exceptions are caught and appended as problem entries — this
+function never raises.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+
+from harness_sdk.cli_base import resolve_binary
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from harness.adapters.probe import probe_cli, supports_flag
+from harness.adapters.registry import build_adapter
+from harness.models.enums import AdapterKind
+from harness.models.provider_config import ProviderConfig
+
+# Kinds that are CLI-based (have a binary on PATH, support probe_cli, etc.)
+_CLI_KINDS = {AdapterKind.claude_code, AdapterKind.antigravity}
+
+# MCP flag checked per CLI kind
+_MCP_FLAG: dict[AdapterKind, str] = {
+    AdapterKind.claude_code: "--mcp-config",
+    AdapterKind.antigravity: "--mcp-config",
+}
+
+# Conversation / resume flag for antigravity native-sessions detection
+_CONVERSATION_FLAG = "--conversation"
+
+
+def _resolve_existing(binary_name: str) -> str | None:
+    """Resolve a CLI name to an existing executable path (PATH + common install dirs),
+    or None if nothing is found. Sync (filesystem checks) so ``_check_provider`` — which
+    is async — doesn't call ``os.path`` directly."""
+    resolved = resolve_binary(binary_name)
+    return resolved if os.path.isfile(resolved) else None
+
+
+async def doctor_report(db: AsyncSession) -> list[dict]:
+    """Run a preflight check on all enabled providers and return a report list."""
+    result = await db.execute(
+        select(ProviderConfig)
+        .where(ProviderConfig.enabled.is_(True))
+        .order_by(ProviderConfig.name)
+    )
+    configs = list(result.scalars())
+
+    results = await asyncio.gather(*[_check_provider(pc) for pc in configs])
+    return list(results)
+
+
+async def _check_provider(pc: ProviderConfig) -> dict:
+    """Probe a single provider config and return its report dict."""
+    problems: list[str] = []
+    is_cli = pc.kind in _CLI_KINDS
+
+    # --- Binary resolution (CLI kinds only) ---
+    binary_path: str | None = None
+    version: str | None = None
+    mcp_ok: bool | None = None
+    probe: dict | None = None
+
+    if is_cli:
+        # Determine the binary name from params or fallback default
+        if pc.kind == AdapterKind.claude_code:
+            binary_name = (pc.params or {}).get("bin", "claude")
+        else:  # antigravity
+            p = pc.params or {}
+            binary_name = p.get("bin") or p.get("cmd") or "agy"
+
+        # Resolve via PATH + common install dirs (matches the spawn path), so an
+        # installed-but-off-PATH CLI isn't wrongly reported as missing.
+        binary_path = _resolve_existing(binary_name)
+        if binary_path is None:
+            problems.append(f"binary not on PATH or common install dirs: {binary_name!r}")
+
+        # Run CLI probe (fast; handles missing binary gracefully)
+        try:
+            probe = await probe_cli(binary_name)
+            version = probe.get("version")
+        except Exception as exc:  # noqa: BLE001
+            problems.append(f"probe failed: {exc}")
+            probe = {"ok": False, "flags": set(), "version": None}
+
+        # MCP flag support check
+        mcp_flag = _MCP_FLAG.get(pc.kind, "--mcp-config")
+        if probe is not None and probe.get("ok"):
+            mcp_ok = supports_flag(probe, mcp_flag)
+            if not mcp_ok:
+                problems.append("MCP flag unsupported — servers will be skipped")
+        else:
+            mcp_ok = False
+
+    # --- Health check ---
+    health_ok = False
+    latency_ms = 0.0
+    try:
+        adapter = build_adapter(pc)
+        health = await adapter.health_check()
+        health_ok = health.ok
+        latency_ms = health.latency_ms
+        if not health.ok:
+            detail = health.detail or {}
+            problems.append(f"health check failed: {detail}")
+    except Exception as exc:  # noqa: BLE001
+        problems.append(f"health check error: {exc}")
+
+    # --- Native-sessions eligibility ---
+    # claude_code: enabled by default unless params explicitly disable it
+    # antigravity: enabled only when probe found --conversation flag or params enable it
+    # others: False
+    explicit = (pc.params or {}).get("native_sessions")
+    if explicit is not None:
+        native_sessions = bool(explicit)
+    elif pc.kind == AdapterKind.claude_code:
+        native_sessions = True
+    elif pc.kind == AdapterKind.antigravity:
+        probed_flags = (probe or {}).get("flags", set()) if probe else None
+        native_sessions = (
+            probed_flags is not None
+            and _CONVERSATION_FLAG in probed_flags
+        )
+    else:
+        native_sessions = False
+
+    return {
+        "provider": pc.name,
+        "kind": pc.kind.value,
+        "binary": binary_path,
+        "version": version,
+        "health_ok": health_ok,
+        "latency_ms": latency_ms,
+        "model": pc.model,
+        "mcp_ok": mcp_ok,
+        "native_sessions": native_sessions,
+        "problems": problems,
+    }