multi-model-debate 1.0.1__py3-none-any.whl

multi_model_debate/phases/base.py

@@ -0,0 +1,236 @@
+"""Base classes for review phases."""
+
+from __future__ import annotations
+
+import json
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from jinja2 import Environment, FileSystemLoader, StrictUndefined
+
+from multi_model_debate.response_parser import extract_json_block, is_valid_response
+
+if TYPE_CHECKING:
+    from multi_model_debate.config import Config
+
+
+@dataclass
+class PhaseArtifact:
+    """Represents a file artifact produced by a phase.
+
+    Used for checkpoint validation - a phase is complete when all
+    its required artifacts exist and are valid.
+
+    Attributes:
+        name: Artifact identifier.
+        path: Path for the artifact file.
+        min_length: Minimum content length for non-JSON responses.
+        is_json: If True, save JSON cleanly without fences and use .json extension.
+    """
+
+    name: str
+    path: Path
+    min_length: int = 100
+    is_json: bool = False
+
+    def exists(self) -> bool:
+        """Check if the artifact file exists."""
+        return self.path.exists()
+
+    def is_valid(self) -> bool:
+        """Check if artifact exists and meets validity criteria.
+
+        Uses JSON-aware validation that accepts:
+        - Valid JSON responses (any length)
+        - "NO NEW ISSUES" format for convergence
+        - Non-JSON responses meeting minimum length
+
+        Returns:
+            True if artifact exists and is valid.
+        """
+        if not self.path.exists():
+            return False
+        content = self.path.read_text()
+        return is_valid_response(content, self.min_length)
+
+    def read(self) -> str:
+        """Read the artifact content.
+
+        Returns:
+            The artifact file content.
+
+        Raises:
+            FileNotFoundError: If artifact doesn't exist.
+        """
+        if not self.path.exists():
+            raise FileNotFoundError(f"Artifact not found: {self.path}")
+        return self.path.read_text()
+
+    def write(self, content: str) -> None:
+        """Write content to the artifact file.
+
+        If is_json=True, extracts clean JSON from markdown fences
+        before saving. This ensures artifacts are stored as valid
+        JSON files without markdown wrapper.
+
+        Args:
+            content: The content to write.
+        """
+        if self.is_json:
+            # Try to extract clean JSON from markdown fences
+            json_content = extract_json_block(content)
+            if json_content is not None:
+                self.path.write_text(json_content)
+                return
+
+        # Write content as-is (either not JSON or no fences to strip)
+        self.path.write_text(content)
+
+
+class Phase(ABC):
+    """Base class for review phases.
+
+    Each phase represents a stage in the adversarial review workflow.
+    Phases produce artifacts (files) that can be validated for checkpoint/resume.
+    """
+
+    def __init__(self, run_dir: Path, config: Config) -> None:
+        """Initialize the phase.
+
+        Args:
+            run_dir: Directory for this run's artifacts.
+            config: Configuration settings.
+        """
+        self.run_dir = run_dir
+        self.config = config
+        self._template_env: Environment | None = None
+
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        """Phase identifier for checkpointing (e.g., 'PHASE_1')."""
+        ...
+
+    @property
+    @abstractmethod
+    def display_name(self) -> str:
+        """Human-readable phase name for logging."""
+        ...
+
+    @abstractmethod
+    def required_artifacts(self) -> list[PhaseArtifact]:
+        """Artifacts that must exist for phase to be complete.
+
+        Returns:
+            List of PhaseArtifact instances.
+        """
+        ...
+
+    @abstractmethod
+    def run(self) -> None:
+        """Execute the phase.
+
+        May be partial if resuming - implementations should check
+        if individual artifacts already exist before generating them.
+        """
+        ...
+
+    def is_complete(self) -> bool:
+        """Check if all required artifacts are valid.
+
+        Returns:
+            True if phase is complete.
+        """
+        return all(artifact.is_valid() for artifact in self.required_artifacts())
+
+    @property
+    def template_env(self) -> Environment:
+        """Get the Jinja2 template environment.
+
+        Lazily creates the environment on first access.
+        """
+        if self._template_env is None:
+            prompts_dir = Path(__file__).parent.parent / "prompts"
+            self._template_env = Environment(
+                loader=FileSystemLoader(prompts_dir),
+                undefined=StrictUndefined,
+                trim_blocks=True,
+                lstrip_blocks=True,
+            )
+        return self._template_env
+
+    def render_template(self, template_name: str, **kwargs: object) -> str:
+        """Render a Jinja2 template.
+
+        Args:
+            template_name: Name of template file (e.g., 'critic_1_lens.md.j2').
+            **kwargs: Template variables.
+
+        Returns:
+            Rendered template string.
+        """
+        template = self.template_env.get_template(template_name)
+        return template.render(**kwargs)
+
+    def get_game_plan(self) -> str:
+        """Read the game plan from the run directory.
+
+        Returns:
+            The game plan content.
+        """
+        return (self.run_dir / "00_game_plan.md").read_text()
+
+    def artifact(
+        self,
+        name: str,
+        filename: str | None = None,
+        *,
+        is_json: bool = False,
+    ) -> PhaseArtifact:
+        """Create a PhaseArtifact for this phase.
+
+        Args:
+            name: Artifact name.
+            filename: File name. If is_json=True, defaults to name + '.json',
+                     otherwise defaults to name + '.md'.
+            is_json: If True, save as clean JSON with .json extension.
+
+        Returns:
+            PhaseArtifact instance.
+        """
+        if filename is None:
+            extension = ".json" if is_json else ".md"
+            filename = f"{name}{extension}"
+
+        return PhaseArtifact(
+            name=name,
+            path=self.run_dir / filename,
+            min_length=self.config.models.min_response_length,
+            is_json=is_json,
+        )
+
+    def journal_response(self, round_num: int, response: str) -> None:
+        """Journal a Strategist response for audit trail.
+
+        Appends entry to strategist_journal.jsonl in JSONL format.
+        Used by DefensePhase and FinalPositionPhase to record Strategist outputs.
+
+        See REQUIREMENTS_V2.md Section 5 for journaling rationale.
+
+        Args:
+            round_num: Round number within the phase (0 for initial/only).
+            response: The Strategist's response text.
+        """
+        journal_path = self.run_dir / "strategist_journal.jsonl"
+        entry = {
+            "timestamp": datetime.now().isoformat(),
+            "phase": self.name,
+            "round": round_num,
+            "response_length": len(response),
+            "response": response,
+        }
+        with journal_path.open("a") as f:
+            f.write(json.dumps(entry) + "\n")

multi_model_debate/phases/baseline.py

@@ -0,0 +1,117 @@
+"""Phase 1: Independent baseline critiques from critics."""
+
+from __future__ import annotations
+
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from rich.console import Console
+
+from multi_model_debate.phases.base import Phase, PhaseArtifact
+
+if TYPE_CHECKING:
+    from multi_model_debate.config import Config
+    from multi_model_debate.models.protocols import ModelBackend
+
+console = Console()
+
+
+class BaselinePhase(Phase):
+    """Phase 1: Critics independently critique the proposal.
+
+    Each critic uses an adversarial "assume flawed" persona with different
+    focus areas. Critics do NOT see each other's critiques.
+    """
+
+    def __init__(
+        self,
+        run_dir: Path,
+        config: Config,
+        *,
+        critic_a: ModelBackend,
+        critic_b: ModelBackend,
+        critic_a_name: str,
+        critic_b_name: str,
+    ) -> None:
+        """Initialize the baseline phase.
+
+        Args:
+            run_dir: Directory for this run's artifacts.
+            config: Configuration settings.
+            critic_a: First critic model backend.
+            critic_b: Second critic model backend.
+            critic_a_name: Display name for first critic (e.g., "codex").
+            critic_b_name: Display name for second critic (e.g., "gemini").
+        """
+        super().__init__(run_dir, config)
+        self.critic_a = critic_a
+        self.critic_b = critic_b
+        self.critic_a_name = critic_a_name
+        self.critic_b_name = critic_b_name
+
+    @property
+    def name(self) -> str:
+        """Phase identifier."""
+        return "PHASE_1"
+
+    @property
+    def display_name(self) -> str:
+        """Human-readable phase name."""
+        return "Baseline Critiques"
+
+    def required_artifacts(self) -> list[PhaseArtifact]:
+        """Artifacts required for phase completion."""
+        return [
+            self.artifact(f"p1_{self.critic_a_name}_baseline", is_json=True),
+            self.artifact(f"p1_{self.critic_b_name}_baseline", is_json=True),
+        ]
+
+    def run(self) -> None:
+        """Execute the baseline phase.
+
+        Generates independent critiques from both critics in parallel.
+        Skips already-completed artifacts on resume.
+        """
+        game_plan = self.get_game_plan()
+        # Dynamic lens selection based on model
+        critic_a_lens = self.render_template("critic_1_lens.md.j2")
+        critic_b_lens = self.render_template("critic_2_lens.md.j2")
+
+        critic_a_artifact = self.artifact(f"p1_{self.critic_a_name}_baseline", is_json=True)
+        critic_b_artifact = self.artifact(f"p1_{self.critic_b_name}_baseline", is_json=True)
+
+        # Track which critics need to run
+        futures: dict[Any, tuple[str, PhaseArtifact]] = {}
+
+        with ThreadPoolExecutor(max_workers=2) as executor:
+            if not critic_a_artifact.is_valid():
+                console.print(f"  [cyan]{self.critic_a_name} baseline critique...[/cyan]")
+                critic_a_prompt = self.render_template(
+                    "baseline_critique.md.j2",
+                    lens_prompt=critic_a_lens,
+                    game_plan=game_plan,
+                )
+                future = executor.submit(self.critic_a.generate, critic_a_prompt)
+                futures[future] = (self.critic_a_name, critic_a_artifact)
+            else:
+                console.print(f"  [dim]{self.critic_a_name} baseline (cached)[/dim]")
+
+            if not critic_b_artifact.is_valid():
+                console.print(f"  [cyan]{self.critic_b_name} baseline critique...[/cyan]")
+                critic_b_prompt = self.render_template(
+                    "baseline_critique.md.j2",
+                    lens_prompt=critic_b_lens,
+                    game_plan=game_plan,
+                )
+                future = executor.submit(self.critic_b.generate, critic_b_prompt)
+                futures[future] = (self.critic_b_name, critic_b_artifact)
+            else:
+                console.print(f"  [dim]{self.critic_b_name} baseline (cached)[/dim]")
+
+            # Wait for parallel calls to complete
+            for future in as_completed(futures):
+                name, artifact = futures[future]
+                response: str = future.result()
+                artifact.write(response)
+                console.print(f"  [green]{name} baseline complete[/green]")

multi_model_debate/phases/debate.py

@@ -0,0 +1,154 @@
+"""Phase 2: Critic vs Critic adversarial debate."""
+
+from __future__ import annotations
+
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from rich.console import Console
+
+from multi_model_debate.phases.base import Phase, PhaseArtifact
+
+if TYPE_CHECKING:
+    from multi_model_debate.config import Config
+    from multi_model_debate.models.protocols import ModelBackend
+
+console = Console()
+
+
+class DebatePhase(Phase):
+    """Phase 2: Critics debate each other.
+
+    Multiple rounds of alternating critiques. Each round only sees
+    the previous round's response (rolling window) to prevent token saturation.
+    """
+
+    def __init__(
+        self,
+        run_dir: Path,
+        config: Config,
+        *,
+        critic_a: ModelBackend,
+        critic_b: ModelBackend,
+        critic_a_name: str,
+        critic_b_name: str,
+    ) -> None:
+        """Initialize the debate phase.
+
+        Args:
+            run_dir: Directory for this run's artifacts.
+            config: Configuration settings.
+            critic_a: First critic model backend.
+            critic_b: Second critic model backend.
+            critic_a_name: Display name for first critic (e.g., "codex").
+            critic_b_name: Display name for second critic (e.g., "gemini").
+        """
+        super().__init__(run_dir, config)
+        self.critic_a = critic_a
+        self.critic_b = critic_b
+        self.critic_a_name = critic_a_name
+        self.critic_b_name = critic_b_name
+        self._rounds = config.debate.critic_rounds
+
+    @property
+    def name(self) -> str:
+        """Phase identifier."""
+        return "PHASE_2"
+
+    @property
+    def display_name(self) -> str:
+        """Human-readable phase name with dynamic model names."""
+        return f"{self.critic_a_name} vs {self.critic_b_name} Debate"
+
+    def required_artifacts(self) -> list[PhaseArtifact]:
+        """Artifacts required for phase completion."""
+        artifacts = []
+        for r in range(1, self._rounds + 1):
+            artifacts.append(self.artifact(f"p2_r{r}_{self.critic_a_name}", is_json=True))
+            artifacts.append(self.artifact(f"p2_r{r}_{self.critic_b_name}", is_json=True))
+        return artifacts
+
+    def run(self) -> None:
+        """Execute the debate phase.
+
+        Runs multiple rounds where both critics respond in parallel.
+        Each critic responds to the other's PREVIOUS round output.
+        Uses rolling window context (only previous round's response).
+        """
+        game_plan = self.get_game_plan()
+        # Dynamic lens selection based on model
+        critic_a_lens = self.render_template("critic_1_lens.md.j2")
+        critic_b_lens = self.render_template("critic_2_lens.md.j2")
+
+        # Load baselines as starting point
+        critic_a_last = self.artifact(f"p1_{self.critic_a_name}_baseline", is_json=True).read()
+        critic_b_last = self.artifact(f"p1_{self.critic_b_name}_baseline", is_json=True).read()
+
+        for round_num in range(1, self._rounds + 1):
+            console.print(f"  [bold]Round {round_num}/{self._rounds}[/bold]")
+
+            critic_a_artifact = self.artifact(f"p2_r{round_num}_{self.critic_a_name}", is_json=True)
+            critic_b_artifact = self.artifact(f"p2_r{round_num}_{self.critic_b_name}", is_json=True)
+
+            # Track which critics need to run
+            futures: dict[Any, tuple[str, PhaseArtifact]] = {}
+            round_label = "Baseline" if round_num == 1 else f"Round {round_num - 1}"
+
+            with ThreadPoolExecutor(max_workers=2) as executor:
+                # Critic A responds to Critic B's previous output
+                if not critic_a_artifact.is_valid():
+                    console.print(f"    [cyan]{self.critic_a_name} responding...[/cyan]")
+                    critic_a_prompt = self.render_template(
+                        "debate_round.md.j2",
+                        lens_prompt=critic_a_lens,
+                        game_plan=game_plan,
+                        opponent_name=self.critic_b_name.upper(),
+                        round_label=round_label,
+                        opponent_response=critic_b_last,
+                    )
+                    future = executor.submit(self.critic_a.generate, critic_a_prompt)
+                    futures[future] = (self.critic_a_name, critic_a_artifact)
+                else:
+                    console.print(f"    [dim]{self.critic_a_name} (cached)[/dim]")
+
+                # Critic B responds to Critic A's previous output
+                if not critic_b_artifact.is_valid():
+                    console.print(f"    [cyan]{self.critic_b_name} responding...[/cyan]")
+                    critic_b_prompt = self.render_template(
+                        "debate_round.md.j2",
+                        lens_prompt=critic_b_lens,
+                        game_plan=game_plan,
+                        opponent_name=self.critic_a_name.upper(),
+                        round_label=round_label,
+                        opponent_response=critic_a_last,
+                    )
+                    future = executor.submit(self.critic_b.generate, critic_b_prompt)
+                    futures[future] = (self.critic_b_name, critic_b_artifact)
+                else:
+                    console.print(f"    [dim]{self.critic_b_name} (cached)[/dim]")
+
+                # Wait for parallel calls to complete
+                for future in as_completed(futures):
+                    name, artifact = futures[future]
+                    response: str = future.result()
+                    artifact.write(response)
+                    console.print(f"    [green]{name} done[/green]")
+
+            # Update "last" responses for next round
+            critic_a_last = critic_a_artifact.read()
+            critic_b_last = critic_b_artifact.read()
+
+    def get_final_positions(self) -> tuple[str, str]:
+        """Get the final positions from the debate.
+
+        Returns:
+            Tuple of (critic_a_final, critic_b_final) responses.
+        """
+        critic_a_final = self.artifact(
+            f"p2_r{self._rounds}_{self.critic_a_name}", is_json=True
+        ).read()
+        critic_b_final = self.artifact(
+            f"p2_r{self._rounds}_{self.critic_b_name}", is_json=True
+        ).read()
+        return critic_a_final, critic_b_final

multi_model_debate/phases/defense.py

@@ -0,0 +1,186 @@
+"""Phase 5: Strategist defends the proposal against the winner's Peer Review."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+from rich.console import Console
+
+from multi_model_debate.exceptions import PhaseError
+from multi_model_debate.phases.base import Phase, PhaseArtifact
+
+if TYPE_CHECKING:
+    from multi_model_debate.config import Config
+    from multi_model_debate.models.protocols import ModelBackend
+
+console = Console()
+
+
+class DefensePhase(Phase):
+    """Phase 5: Strategist defends the proposal against the debate winner.
+
+    Fully automated phase where Strategist responds via CLI invocation.
+    The Strategist is the AI that authored the game plan and has full context.
+    """
+
+    def __init__(
+        self,
+        run_dir: Path,
+        config: Config,
+        strategist: ModelBackend,
+        *,
+        critic_a: ModelBackend,
+        critic_b: ModelBackend,
+        critic_a_name: str,
+        critic_b_name: str,
+    ) -> None:
+        """Initialize the defense phase.
+
+        Args:
+            run_dir: Directory for this run's artifacts.
+            config: Configuration settings.
+            strategist: Strategist model backend (uses CLI invocation).
+            critic_a: First critic model backend.
+            critic_b: Second critic model backend.
+            critic_a_name: Display name for first critic (e.g., "codex").
+            critic_b_name: Display name for second critic (e.g., "gemini").
+        """
+        super().__init__(run_dir, config)
+        self.strategist = strategist
+        self.critic_a = critic_a
+        self.critic_b = critic_b
+        self.critic_a_name = critic_a_name
+        self.critic_b_name = critic_b_name
+        self._rounds = config.debate.strategist_rounds
+
+    @property
+    def name(self) -> str:
+        """Phase identifier."""
+        return "PHASE_5"
+
+    @property
+    def display_name(self) -> str:
+        """Human-readable phase name."""
+        return "Strategist Defense"
+
+    def required_artifacts(self) -> list[PhaseArtifact]:
+        """Artifacts required for phase completion.
+
+        Phase 5 requires:
+        - p5_r0_strategist.md (initial Strategist defense)
+        - p5_r{1..N}_winner.md (winner's responses)
+        - p5_r{1..N}_strategist.md (Strategist's responses)
+        """
+        artifacts = [self.artifact("p5_r0_strategist")]
+        for r in range(1, self._rounds + 1):
+            artifacts.append(self.artifact(f"p5_r{r}_winner"))
+            artifacts.append(self.artifact(f"p5_r{r}_strategist"))
+        return artifacts
+
+    def run(self) -> None:
+        """Execute the defense phase.
+
+        Strategist defends against the winner's Peer Review, then multiple
+        rounds of back-and-forth debate. All responses are automated via CLI.
+        """
+        winner_name = self._get_winner_name()
+        winner_model = self.critic_a if winner_name == self.critic_a_name else self.critic_b
+        winner_lens = self._get_winner_lens(winner_name)
+
+        game_plan = self.get_game_plan()
+        strategist_lens = self.render_template("strategist_proxy_lens.md.j2")
+        peer_review = self.artifact("p4_peer_review").read()
+
+        # Initial Strategist defense against Peer Review
+        strategist_initial = self.artifact("p5_r0_strategist")
+        if not strategist_initial.is_valid():
+            console.print("  [bold cyan]Round 0: Strategist Initial Defense[/bold cyan]")
+            prompt = self.render_template(
+                "defense_initial.md.j2",
+                strategist_lens=strategist_lens,
+                peer_review=peer_review,
+            )
+            response = self.strategist.generate(prompt)  # Uses per-model timeout
+            strategist_initial.write(response)
+            # Journal the Strategist response for audit trail
+            self.journal_response(round_num=0, response=response)
+            console.print("  [green]Strategist initial defense complete[/green]")
+        else:
+            console.print("  [dim]Round 0: Strategist Initial (cached)[/dim]")
+
+        strategist_last = strategist_initial.read()
+
+        # Debate rounds
+        for round_num in range(1, self._rounds + 1):
+            console.print(f"  [bold]Round {round_num}/{self._rounds}[/bold]")
+
+            # Winner responds to Strategist
+            winner_artifact = self.artifact(f"p5_r{round_num}_winner")
+            if not winner_artifact.is_valid():
+                console.print(f"    [cyan]{winner_name} responding...[/cyan]")
+                round_label = "Initial" if round_num == 1 else f"Round {round_num - 1}"
+                prompt = self.render_template(
+                    "winner_response.md.j2",
+                    winner_lens=winner_lens,
+                    game_plan=game_plan,
+                    peer_review=peer_review,
+                    round_label=round_label,
+                    strategist_response=strategist_last,
+                )
+                response = winner_model.generate(prompt)  # Uses per-model timeout
+                winner_artifact.write(response)
+                console.print(f"    [green]{winner_name} done[/green]")
+            else:
+                console.print(f"    [dim]{winner_name} (cached)[/dim]")
+
+            winner_last = winner_artifact.read()
+
+            # Strategist responds to winner
+            strategist_artifact = self.artifact(f"p5_r{round_num}_strategist")
+            if not strategist_artifact.is_valid():
+                console.print("    [bold cyan]Strategist responding...[/bold cyan]")
+                prompt = self.render_template(
+                    "defense_round.md.j2",
+                    strategist_lens=strategist_lens,
+                    round_number=round_num,
+                    winner_response=winner_last,
+                )
+                response = self.strategist.generate(prompt)  # Uses per-model timeout
+                strategist_artifact.write(response)
+                # Journal the Strategist response for audit trail
+                self.journal_response(round_num=round_num, response=response)
+                console.print("    [green]Strategist done[/green]")
+            else:
+                console.print("    [dim]Strategist (cached)[/dim]")
+
+            strategist_last = strategist_artifact.read()
+
+    def _get_winner_name(self) -> str:
+        """Get the winner's model name from Phase 3."""
+        winner_path = self.run_dir / "p3_winner.txt"
+        if not winner_path.exists():
+            raise PhaseError("Winner file not found")
+        content = winner_path.read_text().strip()
+        if content.startswith("WINNER="):
+            return content.split("=")[1].strip()
+        raise PhaseError(f"Invalid winner file: {content}")
+
+    def _get_winner_lens(self, winner_name: str) -> str:
+        """Get the winner's lens prompt."""
+        # Use critic A's lens for critic A, critic B's lens for critic B
+        if winner_name == self.critic_a_name:
+            template = "critic_1_lens.md.j2"
+        else:
+            template = "critic_2_lens.md.j2"
+        return self.render_template(template)
+
+    def get_final_responses(self) -> tuple[str, str]:
+        """Get the final responses from the defense.
+
+        Returns:
+            Tuple of (final_winner_response, final_strategist_response).
+        """
+        final_winner = self.artifact(f"p5_r{self._rounds}_winner").read()
+        final_strategist = self.artifact(f"p5_r{self._rounds}_strategist").read()
+        return final_winner, final_strategist