multi-forge 0.2.0__py3-none-any.whl

forge/review/consensus.py

@@ -0,0 +1,236 @@
+"""Two-round consensus workflow with role-assigned workers.
+
+Round 1: Each worker evaluates the subject from their assigned role.
+         Blinded (``resume_id=None``). Workers don't see each other.
+
+Round 2: Each worker receives the reconciliation brief (all Round 1
+         positions) and produces a reconciled recommendation.
+         Still blinded (no conversation context).
+
+Both rounds delegate to ``run_multi_review()`` for parallel fan-out.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from forge.core.reactive.structured_output import extract_json_from_response
+
+from .engine import run_multi_review
+from .models import ConsensusOutput, ModelSpec, RoleSpec
+from .routing import WorkerRoutingPlan
+
+ROLE_MARKER = "{role_prompt}"
+
+CONSENSUS_GUARDRAIL = (
+    "\n\nIMPORTANT: You are participating in a structured consensus-building exercise. "
+    "Provide your honest expert assessment from your assigned perspective. "
+    "Support claims with evidence and reasoning. Do not fabricate evidence "
+    "or misrepresent trade-offs. When you lack certainty, say so explicitly."
+)
+
+_MAX_EXCERPT_LEN = 1500
+
+_ROUND2_OUTPUT_CONTRACT = (
+    "\n## Required Output Format\n\n"
+    "Respond with your reconciled assessment in JSON wrapped in a ```json code fence:\n\n"
+    "```\n"
+    "{\n"
+    '  "position": "SUPPORT" | "SUPPORT_WITH_CONDITIONS" | "OPPOSE",\n'
+    '  "confidence": "LOW" | "MEDIUM" | "HIGH",\n'
+    '  "agreements": ["point of agreement 1", ...],\n'
+    '  "disagreements": ["unresolved point 1", ...],\n'
+    '  "recommendation": "1-2 sentence reconciled recommendation",\n'
+    '  "conditions": ["condition 1", ...]\n'
+    "}\n"
+    "```\n"
+)
+
+
+def validate_resource(resource_path: str) -> str:
+    """Load a resource file and verify it contains the role marker.
+
+    Raises ValueError if the marker is missing.
+    """
+    content = Path(resource_path).read_text()
+    if ROLE_MARKER not in content:
+        raise ValueError(f"Resource {resource_path} must contain '{ROLE_MARKER}' marker " "for role injection.")
+    return content
+
+
+def _build_reconciliation_brief(
+    round1_results: list,
+    role_map: dict[str, str],
+    original_subject: str = "",
+) -> str:
+    """Build a structured reconciliation brief from Round 1 positions.
+
+    Each worker's output is labeled by role (not model name) to minimize
+    anchoring bias. Parse-resilient: tries JSON extraction with fallback
+    to truncated raw text. Includes the original subject and output contract
+    so Round 2 workers retain scope and produce parseable output.
+    """
+    sections: list[str] = []
+
+    if original_subject:
+        sections.append(f"# Original Subject\n\n{original_subject}\n")
+
+    sections.append("# Round 1 Positions\n")
+
+    for result in round1_results:
+        role = role_map.get(result.model_name, "unknown")
+        section = f"## {role} perspective\n\n"
+
+        if not result.success:
+            section += f"Status: failed ({result.error})\n"
+            sections.append(section)
+            continue
+
+        section += "Status: success\n"
+
+        # Try structured extraction; fall back to truncated text
+        parsed = extract_json_from_response(result.stdout)
+        if parsed is not None:
+            section += f"Position: {json.dumps(parsed, indent=2)}\n"
+        else:
+            excerpt = result.stdout[:_MAX_EXCERPT_LEN]
+            if len(result.stdout) > _MAX_EXCERPT_LEN:
+                excerpt += "..."
+            section += f"Position: {excerpt}\n"
+
+        sections.append(section)
+
+    sections.append(
+        "\n---\n\n"
+        "# Reconciliation Task\n\n"
+        "You have seen all initial positions above. Now:\n\n"
+        "1. Identify the specific points of AGREEMENT across perspectives.\n"
+        "2. Identify the specific points of DISAGREEMENT.\n"
+        "3. For each disagreement, assess which position has stronger evidence.\n"
+        "4. Propose a RECONCILED RECOMMENDATION that incorporates the strongest "
+        "points from each perspective.\n"
+        "5. If genuine consensus is not possible on a point, explicitly state "
+        "'NO CONSENSUS' for that point and explain why.\n\n"
+        "Maintain your assigned role perspective but be willing to update your "
+        "position based on compelling evidence from other perspectives.\n"
+    )
+
+    sections.append(_ROUND2_OUTPUT_CONTRACT)
+
+    return "\n".join(sections)
+
+
+def run_consensus(
+    resource_path: str,
+    roles: list[RoleSpec],
+    *,
+    timeout_seconds: int = 600,
+    cwd: str | None = None,
+    original_subject: str = "",
+    via: str | None = None,
+    routing_plan: WorkerRoutingPlan | None = None,
+) -> ConsensusOutput:
+    """Run two-round consensus workflow with role-assigned workers.
+
+    Round 1: Each worker evaluates the subject from their assigned role,
+    blinded. Round 2: Each worker receives the reconciliation brief and
+    produces a reconciled recommendation, still blinded.
+
+    Args:
+        original_subject: The raw subject/target text (before template
+            wrapping). Included in the reconciliation brief so Round 2
+            workers retain scope context.
+        via: Route all workers through this proxy (passed to routing).
+            Ignored when routing_plan is provided.
+        routing_plan: Pre-resolved routing plan. When provided, skips
+            internal routing resolution and reuses the same route decisions
+            for both rounds; Round 2 changes prompts but not route-bearing
+            model fields or order.
+
+    Raises ValueError if the resource lacks the role marker.
+    """
+    from forge.review.routing import resolve_invocation_routing
+
+    template = validate_resource(resource_path)
+
+    # --- Build Round 1 specs ---
+    specs_r1: list[ModelSpec] = []
+    seen: dict[str, int] = {}
+    for role_spec in roles:
+        filled = template.replace(
+            ROLE_MARKER,
+            role_spec.role_prompt + CONSENSUS_GUARDRAIL,
+        )
+        label = role_spec.effective_label
+        base_id = f"{role_spec.model.name}-{label}"
+        count = seen.get(base_id, 0)
+        seen[base_id] = count + 1
+        worker_id = base_id if count == 0 else f"{base_id}-{count}"
+        specs_r1.append(
+            ModelSpec(
+                name=role_spec.model.name,
+                model_id=role_spec.model.model_id,
+                family=role_spec.model.family,
+                provider_refs=role_spec.model.provider_refs,
+                description=f"{label} role via {role_spec.model.name}",
+                preferred_proxy=role_spec.model.preferred_proxy,
+                prompt=filled,
+                worker_id=worker_id,
+            )
+        )
+
+    role_map = {spec.effective_worker_id: r.effective_label for spec, r in zip(specs_r1, roles)}
+
+    plan_r1 = routing_plan if routing_plan is not None else resolve_invocation_routing(specs_r1, via=via)
+
+    # --- Round 1: Independent positions (blinded) ---
+    round1_output = run_multi_review(
+        prompt="",
+        models=specs_r1,
+        routing_plan=plan_r1,
+        timeout_seconds=timeout_seconds,
+        cwd=cwd,
+        resume_id=None,
+    )
+
+    # --- Build reconciliation brief ---
+    brief = _build_reconciliation_brief(round1_output.results, role_map, original_subject=original_subject)
+
+    # --- Build Round 2 specs (same worker_ids for correlation) ---
+    specs_r2: list[ModelSpec] = []
+    for spec_r1, role_spec in zip(specs_r1, roles):
+        reconciliation_prompt = f"[ROLE: {role_spec.effective_label}]\n" f"{role_spec.role_prompt}\n\n" f"{brief}"
+        specs_r2.append(
+            ModelSpec(
+                name=spec_r1.name,
+                model_id=spec_r1.model_id,
+                family=spec_r1.family,
+                provider_refs=spec_r1.provider_refs,
+                description=f"{role_spec.effective_label} reconciliation via {spec_r1.name}",
+                preferred_proxy=spec_r1.preferred_proxy,
+                prompt=reconciliation_prompt,
+                worker_id=spec_r1.effective_worker_id,
+            )
+        )
+
+    plan_r2 = routing_plan if routing_plan is not None else resolve_invocation_routing(specs_r2, via=via)
+
+    # --- Round 2: Reconciliation (blinded) ---
+    round2_output = run_multi_review(
+        prompt="",
+        models=specs_r2,
+        routing_plan=plan_r2,
+        timeout_seconds=timeout_seconds,
+        cwd=cwd,
+        resume_id=None,
+    )
+
+    return ConsensusOutput(
+        subject=original_subject or resource_path,
+        roles=[r.effective_label for r in roles],
+        round1_results=round1_output.results,
+        round2_results=round2_output.results,
+        role_map=role_map,
+        reconciliation_brief=brief,
+    )

forge/review/engine.py

@@ -0,0 +1,356 @@
+"""Multi-model review engine with parallel fan-out.
+
+Spawns N ``claude -p`` subprocesses in parallel via ThreadPoolExecutor,
+one per model backend. Each subprocess runs in its own process group
+(``start_new_session=True``) so that cleanup via ``os.killpg`` can
+terminate orphaned children if the parent is interrupted.
+
+Routing is pre-resolved: the engine receives a ``WorkerRoutingPlan``
+and passes each worker its ``RoutingResult``. No per-worker registry
+lookups during fan-out.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import shutil
+import signal
+import subprocess
+import threading
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
+from forge.core.auth.capabilities import CREDENTIALS, format_missing_credential_error
+from forge.core.auth.template_secrets import resolve_env_or_credential
+from forge.core.reactive.env import (
+    build_claude_env,
+    can_use_bare,
+    should_spawn_subprocesses,
+)
+from forge.core.reactive.routing import RoutingResult
+from forge.review.routing import (
+    WorkerRoutingPlan,
+    resolve_invocation_routing,
+    resolve_model_flag,
+)
+from forge.session.direct_model import direct_model_env
+
+from .models import (
+    DEFAULT_MODELS,
+    ModelSpec,
+    MultiReviewOutput,
+    ReviewResult,
+)
+
+_log = logging.getLogger(__name__)
+
+
+def preflight_check(
+    specs: list[ModelSpec],
+    routing_plan: WorkerRoutingPlan | None = None,
+) -> list[str]:
+    """Validate routing before spawning workers.
+
+    When a routing_plan is provided, validates each result has a route.
+    Otherwise falls back to check_model_availability().
+
+    Returns a list of error strings (empty means all OK).
+    """
+    errors: list[str] = []
+
+    if should_spawn_subprocesses() and shutil.which("claude") is None:
+        errors.append(
+            "claude CLI not found in PATH. `forge workflow` workers run through local `claude -p`, "
+            "even for proxy-routed models; install Claude Code or expose `claude` on PATH in the "
+            "environment running `forge workflow`."
+        )
+
+    if routing_plan is not None:
+        for spec, result in zip(specs, routing_plan.routes):
+            if result.route is None:
+                reason = result.warning or "No compatible route found"
+                errors.append(f"{spec.name}: {reason}")
+                continue
+
+            credential_error = _credential_preflight_error(spec, result)
+            if credential_error:
+                errors.append(credential_error)
+        return errors
+
+    from .models import check_model_availability
+
+    availabilities = check_model_availability(specs)
+    for avail in availabilities:
+        if avail.status == "ready":
+            continue
+        if avail.spec.preferred_proxy:
+            hint = f" Run 'forge proxy create {avail.spec.preferred_proxy}' to set it up."
+        else:
+            hint = " Run 'forge auth login -c anthropic-api' or use --models to select only proxy-backed models."
+        errors.append(f"{avail.spec.name}: {avail.reason}.{hint}")
+    return errors
+
+
+def _credential_preflight_error(spec: ModelSpec, result: RoutingResult) -> str | None:
+    """Return an actionable missing-credential error for direct workflow routes."""
+    route = result.route
+    if route is None or route.provider != "direct":
+        return None
+
+    credential = CREDENTIALS.get(route.credential)
+    if credential is None:
+        return None
+
+    missing_vars = [
+        env_var.name
+        for env_var in credential.env_vars
+        if env_var.required and not resolve_env_or_credential(env_var.name)
+    ]
+    if not missing_vars:
+        return None
+
+    return format_missing_credential_error(
+        credential,
+        missing_vars=missing_vars,
+        context=f"Workflow model '{spec.name}'",
+    )
+
+
+def run_multi_review(
+    prompt: str,
+    *,
+    models: list[ModelSpec] | None = None,
+    routing_plan: WorkerRoutingPlan | None = None,
+    timeout_seconds: int = 600,
+    cwd: str | None = None,
+    resume_id: str | None = None,
+) -> MultiReviewOutput:
+    """Fan out a review prompt to multiple models in parallel.
+
+    Args:
+        prompt: The review prompt to send to each model.
+        models: Model specs to use. Defaults to DEFAULT_MODELS values.
+        routing_plan: Pre-resolved routing for all workers. When None,
+            resolves routing once at the top before the thread pool.
+        timeout_seconds: Per-model timeout in seconds.
+        cwd: Working directory for each subprocess.
+        resume_id: If set, adds ``--resume <id>`` to each subprocess.
+
+    Returns:
+        MultiReviewOutput with per-model results in input order.
+        Returns empty results if FORGE_DEPTH limit reached.
+    """
+    if not should_spawn_subprocesses():
+        _log.debug("Skipping ensemble review at FORGE_DEPTH limit")
+        return MultiReviewOutput(prompt=prompt)
+
+    specs = models if models is not None else list(DEFAULT_MODELS.values())
+
+    if not specs:
+        return MultiReviewOutput(prompt=prompt)
+
+    # Resolve routing once if not provided by caller
+    if routing_plan is None:
+        try:
+            routing_plan = resolve_invocation_routing(specs)
+        except Exception as e:
+            _log.warning("Routing resolution failed: %s", e)
+            return MultiReviewOutput(
+                prompt=prompt,
+                results=[
+                    ReviewResult(
+                        model_name=s.effective_worker_id,
+                        stdout="",
+                        stderr="",
+                        success=False,
+                        duration_seconds=0.0,
+                        error=str(e),
+                    )
+                    for s in specs
+                ],
+            )
+
+    # Thread-safe list for tracking child processes
+    children: list[subprocess.Popen[str]] = []
+    children_lock = threading.Lock()
+
+    def _cleanup() -> None:
+        """Terminate and reap all running children. SIGTERM -> wait -> SIGKILL."""
+        with children_lock:
+            for proc in children:
+                if proc.poll() is None:
+                    try:
+                        os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
+                    except (OSError, ProcessLookupError):
+                        pass
+            for proc in children:
+                try:
+                    proc.wait(timeout=5)
+                except subprocess.TimeoutExpired:
+                    try:
+                        os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
+                        proc.wait(timeout=2)
+                    except (OSError, ProcessLookupError, subprocess.TimeoutExpired):
+                        pass
+                except OSError:
+                    pass
+
+    def _run_single(spec: ModelSpec, routing_result: RoutingResult) -> ReviewResult:
+        """Run a single model review with pre-resolved routing."""
+        start = time.monotonic()
+        if spec.prompt is None:
+            worker_prompt = prompt
+        elif spec.prompt_mode == "prefix":
+            worker_prompt = f"{spec.prompt}\n\n{prompt}" if prompt else spec.prompt
+        else:
+            worker_prompt = spec.prompt
+
+        extra_env: dict[str, str] = {}
+        if not os.environ.get("ANTHROPIC_API_KEY"):
+            ak = resolve_env_or_credential("ANTHROPIC_API_KEY")
+            if ak:
+                extra_env["ANTHROPIC_API_KEY"] = ak
+
+        route = routing_result.route
+        if route is None:
+            duration = time.monotonic() - start
+            return ReviewResult(
+                model_name=spec.effective_worker_id,
+                stdout="",
+                stderr="",
+                success=False,
+                duration_seconds=duration,
+                error=f"No route resolved for '{spec.name}'",
+            )
+
+        if route.provider == "direct":
+            try:
+                extra_env.update(direct_model_env(route.model_ref))
+            except ValueError as e:
+                duration = time.monotonic() - start
+                return ReviewResult(
+                    model_name=spec.effective_worker_id,
+                    stdout="",
+                    stderr="",
+                    success=False,
+                    duration_seconds=duration,
+                    error=str(e),
+                )
+            env = build_claude_env(direct=True, extra_vars=extra_env or None)
+        else:
+            env = build_claude_env(base_url=routing_result.base_url, extra_vars=extra_env or None)
+
+        cmd = ["claude", "-p"]
+        if can_use_bare(env):
+            cmd.append("--bare")
+        if resume_id:
+            cmd.extend(["--resume", resume_id])
+
+        model_flag = resolve_model_flag(route)
+        if model_flag:
+            cmd.extend(["--model", model_flag])
+
+        try:
+            proc = subprocess.Popen(
+                cmd,
+                stdin=subprocess.PIPE,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                text=True,
+                cwd=cwd,
+                env=env,
+                start_new_session=True,
+            )
+            with children_lock:
+                children.append(proc)
+
+            stdout, stderr = proc.communicate(input=worker_prompt, timeout=timeout_seconds)
+            duration = time.monotonic() - start
+
+            if proc.returncode != 0:
+                error_msg = stderr.strip() or f"Exit code {proc.returncode}"
+                return ReviewResult(
+                    model_name=spec.effective_worker_id,
+                    stdout=stdout,
+                    stderr=stderr,
+                    success=False,
+                    duration_seconds=duration,
+                    error=error_msg,
+                )
+
+            return ReviewResult(
+                model_name=spec.effective_worker_id,
+                stdout=stdout.strip(),
+                stderr=stderr,
+                success=True,
+                duration_seconds=duration,
+            )
+
+        except subprocess.TimeoutExpired:
+            try:
+                os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
+                proc.wait(timeout=5)
+            except (OSError, ProcessLookupError, subprocess.TimeoutExpired):
+                pass
+            return ReviewResult(
+                model_name=spec.effective_worker_id,
+                stdout="",
+                stderr="",
+                success=False,
+                duration_seconds=float(timeout_seconds),
+                error=f"Timeout after {timeout_seconds}s",
+            )
+
+        except FileNotFoundError:
+            duration = time.monotonic() - start
+            return ReviewResult(
+                model_name=spec.effective_worker_id,
+                stdout="",
+                stderr="",
+                success=False,
+                duration_seconds=duration,
+                error="claude CLI not found in PATH",
+            )
+
+        except (OSError, subprocess.SubprocessError) as e:
+            duration = time.monotonic() - start
+            return ReviewResult(
+                model_name=spec.effective_worker_id,
+                stdout="",
+                stderr="",
+                success=False,
+                duration_seconds=duration,
+                error=str(e),
+            )
+
+    # Fan out with ThreadPoolExecutor, preserving input order and duplicate workers.
+    result_map: dict[int, ReviewResult] = {}
+    max_workers = min(len(specs), 5)
+
+    try:
+        with ThreadPoolExecutor(max_workers=max_workers) as executor:
+            future_to_item = {
+                executor.submit(_run_single, spec, routing_plan.routes[idx]): (idx, spec)
+                for idx, spec in enumerate(specs)
+            }
+            for future in as_completed(future_to_item):
+                idx, spec = future_to_item[future]
+                wid = spec.effective_worker_id
+                try:
+                    result_map[idx] = future.result()
+                except Exception as e:
+                    result_map[idx] = ReviewResult(
+                        model_name=wid,
+                        stdout="",
+                        stderr="",
+                        success=False,
+                        duration_seconds=0.0,
+                        error=f"Thread error: {e}",
+                    )
+    finally:
+        _cleanup()
+
+    # Return in deterministic input order
+    ordered = [result_map[idx] for idx in range(len(specs)) if idx in result_map]
+    return MultiReviewOutput(prompt=prompt, results=ordered)