symphony-code 0.0.1__py3-none-any.whl

coding_agent/addons/compaction/transcript.py

@@ -0,0 +1,118 @@
+"""Render dropped turns for the model and build the compacted-context message."""
+
+from __future__ import annotations
+
+import json
+from typing import List, Sequence
+
+from core_ai.content import text_from_content
+from core_ai.types import Message
+from core_harness.addons.compaction import (
+    COMPACTION_CONTINUATION,
+    compaction_header,
+    dropped_turn_facts,
+)
+
+DEFAULT_TRANSCRIPT_MAX_CHARS = 24_000
+_MESSAGE_PREVIEW_CHARS = 700
+_TOOL_RESULT_PREVIEW_CHARS = 400
+_TOOL_ARGS_PREVIEW_CHARS = 160
+
+
+def _clamp(text: str, limit: int) -> str:
+    text = text.strip()
+    if len(text) <= limit:
+        return text
+    return text[: limit - 1].rstrip() + "…"
+
+
+def _tool_call_line(call: dict) -> str:
+    function = call.get("function") or {}
+    name = str(function.get("name") or "tool")
+    raw_arguments = function.get("arguments") or ""
+    if isinstance(raw_arguments, str):
+        try:
+            decoded = json.loads(raw_arguments) if raw_arguments else {}
+        except json.JSONDecodeError:
+            decoded = raw_arguments
+    else:
+        decoded = raw_arguments
+    arguments = decoded if isinstance(decoded, str) else json.dumps(decoded, sort_keys=True)
+    return f"assistant → {name}({_clamp(arguments, _TOOL_ARGS_PREVIEW_CHARS)})"
+
+
+def render_dropped_turns(
+    turns: Sequence[Sequence[Message]],
+    *,
+    max_chars: int = DEFAULT_TRANSCRIPT_MAX_CHARS,
+) -> str:
+    """Plain-text transcript of the turns about to be dropped.
+
+    Each message is clamped individually; if the whole transcript is still too
+    long the middle is elided so both the earliest asks and the most recent
+    work reach the model.
+    """
+    names: dict[str, str] = {}
+    lines: List[str] = []
+    for turn in turns:
+        for message in turn:
+            if message.role == "assistant" and message.tool_calls:
+                for call in message.tool_calls:
+                    function = call.get("function") or {}
+                    names[str(call.get("id") or "")] = str(function.get("name") or "tool")
+                text = text_from_content(message.content).strip()
+                if text:
+                    lines.append(f"assistant: {_clamp(text, _MESSAGE_PREVIEW_CHARS)}")
+                lines.extend(_tool_call_line(call) for call in message.tool_calls)
+            elif message.role == "tool":
+                name = names.get(str(message.tool_call_id or ""), "tool")
+                body = text_from_content(message.content)
+                lines.append(f"tool[{name}]: {_clamp(body, _TOOL_RESULT_PREVIEW_CHARS)}")
+            else:
+                text = text_from_content(message.content)
+                lines.append(f"{message.role}: {_clamp(text, _MESSAGE_PREVIEW_CHARS)}")
+
+    transcript = "\n".join(lines)
+    if len(transcript) <= max_chars:
+        return transcript
+    head_budget = max_chars // 2
+    tail_budget = max_chars - head_budget
+    head: List[str] = []
+    used = 0
+    for line in lines:
+        if used + len(line) + 1 > head_budget:
+            break
+        head.append(line)
+        used += len(line) + 1
+    tail: List[str] = []
+    used = 0
+    for line in reversed(lines[len(head):]):
+        if used + len(line) + 1 > tail_budget:
+            break
+        tail.insert(0, line)
+        used += len(line) + 1
+    omitted = len(lines) - len(head) - len(tail)
+    return "\n".join([*head, f"… ({omitted} lines omitted) …", *tail])
+
+
+def build_compacted_message(
+    turns: Sequence[Sequence[Message]],
+    *,
+    narrative: str,
+    tokens: int,
+) -> Message:
+    """Compacted-context message: header, model narrative, deterministic facts."""
+    dropped_messages = sum(len(turn) for turn in turns)
+    lines = compaction_header(dropped_messages, tokens=tokens)
+    lines.append("Summary of the dropped work:")
+    lines.append(narrative.strip())
+    lines.extend(dropped_turn_facts(turns))
+    lines.append(COMPACTION_CONTINUATION)
+    return Message(role="user", content="\n".join(lines))
+
+
+__all__ = [
+    "DEFAULT_TRANSCRIPT_MAX_CHARS",
+    "build_compacted_message",
+    "render_dropped_turns",
+]

coding_agent/addons/evaluation/__init__.py

@@ -0,0 +1,68 @@
+"""Optional Jev critic mode for coding-agent runs."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from coding_agent._lazy import resolve
+from coding_agent.addons.evaluation.evaluators import (
+    LLMEvaluator,
+    MockEvaluator,
+    StubEvaluator,
+    VercelJevEvaluator,
+)
+from coding_agent.addons.evaluation.handlers import HandlerResult, dispatch
+from coding_agent.addons.evaluation.policy import (
+    JEV_SYSTEM_SEGMENT,
+    NUDGE_MARKER,
+    PolicyDecision,
+    apply_findings_gate,
+    critic_message,
+    decide_next_step,
+)
+from coding_agent.addons.evaluation.protocol import (
+    CONTINUE_NORMALLY,
+    EvaluationDecision,
+    EvaluationFinding,
+    EvaluationResult,
+    Evaluator,
+    FINISH_QUESTIONS,
+    START_QUESTIONS,
+)
+from coding_agent.addons.evaluation.state import RunState, build_run_state, clip_text
+
+_ADDON_EXPORTS = {
+    "JevAddon": ("coding_agent.addons.evaluation.addon", "JevAddon"),
+    "jev_from_config": ("coding_agent.addons.evaluation.addon", "jev_from_config"),
+}
+
+__all__ = [
+    "CONTINUE_NORMALLY",
+    "EvaluationDecision",
+    "EvaluationFinding",
+    "EvaluationResult",
+    "Evaluator",
+    "FINISH_QUESTIONS",
+    "HandlerResult",
+    "JEV_SYSTEM_SEGMENT",
+    "JevAddon",
+    "LLMEvaluator",
+    "NUDGE_MARKER",
+    "PolicyDecision",
+    "MockEvaluator",
+    "RunState",
+    "START_QUESTIONS",
+    "StubEvaluator",
+    "VercelJevEvaluator",
+    "apply_findings_gate",
+    "build_run_state",
+    "clip_text",
+    "critic_message",
+    "decide_next_step",
+    "dispatch",
+    "jev_from_config",
+]
+
+
+def __getattr__(name: str) -> Any:
+    return resolve(globals(), name, _ADDON_EXPORTS)

coding_agent/addons/evaluation/addon.py

@@ -0,0 +1,199 @@
+"""Jev critic add-on. Evaluates completed runs and suggests revisions."""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Optional
+
+from core_ai.types import Message
+from core_harness.addons.addon import Addon
+
+from coding_agent.config import EvaluationConfig
+from coding_agent.addons.evaluation.evaluators import VercelJevEvaluator
+from coding_agent.addons.evaluation.handlers import dispatch
+from coding_agent.addons.evaluation.policy import (
+    MAX_HONOURED_FOLLOW_UPS,
+    PolicyDecision,
+    critic_message,
+    decide_next_step,
+    is_jev_nudge,
+)
+from coding_agent.addons.evaluation.protocol import (
+    CONTINUE_NORMALLY,
+    EvaluationResult,
+    EvaluationPhase,
+    Evaluator,
+    error_result,
+)
+from coding_agent.addons.evaluation.state import bound_text, build_run_state
+
+logger = logging.getLogger(__name__)
+
+
+class JevAddon(Addon):
+    """Evaluate completed runs against the user's rule and task.
+
+    Dispatches named handlers from ``last_decision.action``. Does not override
+    ``before_turn`` or ``on_tool`` and never gates tools. A violated user rule
+    can trigger one automatic follow-up. Evaluation uses only the conversation,
+    tool results, and final response; it does not read workspace files.
+    """
+
+    name = "jev"
+
+    def __init__(
+        self,
+        config: EvaluationConfig,
+        *,
+        evaluator: Optional[Evaluator] = None,
+    ) -> None:
+        self.config = config
+        self.evaluator = evaluator or VercelJevEvaluator(config)
+        self.last_start: Optional[EvaluationResult] = None
+        self.last_finish: Optional[EvaluationResult] = None
+        self.last_decision: Optional[PolicyDecision] = None
+        self.follow_ups = 0
+        self._harness: Any = None
+
+    def attach(self, harness: Any) -> None:
+        self._harness = harness
+
+    def fork_for_child(self, parent_harness: Any) -> None:
+        """Skip inherit: child runs are not evaluated by Jev."""
+        del parent_harness
+        return None
+
+    async def _call(
+        self, method_name: str, phase: EvaluationPhase, **state_kwargs: Any
+    ) -> EvaluationResult:
+        state = build_run_state(config=self.config, phase=phase, **state_kwargs)
+        method = getattr(self.evaluator, method_name)
+        try:
+            result = await method(state)
+        except Exception as exc:
+            logger.warning("Jev %s failed open: %s", method_name, exc)
+            result = error_result(phase, str(exc)[:280])
+        decision = decide_next_step(result)
+        if (
+            (self.config.honour_follow_up or self.config.rule.strip())
+            and decision.should_act
+            and self.follow_ups >= MAX_HONOURED_FOLLOW_UPS
+            and decision.action == "retry"
+        ):
+            decision = PolicyDecision(
+                action=CONTINUE_NORMALLY,
+                reason="follow-up budget exhausted",
+                finding=decision.finding,
+            )
+        self.last_decision = decision
+        logger.info(
+            "jev %s status=%s findings=%s action=%s reason=%s",
+            phase,
+            result.status,
+            [finding.question for finding in result.findings],
+            decision.action,
+            decision.reason,
+        )
+        if self._harness is not None:
+            for addon in self._harness.addons:
+                hook = getattr(addon, "on_evaluation", None)
+                if callable(hook):
+                    await hook(phase=phase, result=result, evaluator="jev")
+        return result
+
+    async def before_run(self, **payload: Any) -> None:
+        self._clear_nudge(payload)
+
+    async def after_run(self, **payload: Any) -> None:
+        result = payload.get("result")
+        messages = payload.get("messages")
+        if not isinstance(messages, list):
+            messages = getattr(result, "messages", None) or []
+        tool_calls = getattr(result, "tool_calls", None) or ()
+        final = getattr(result, "output_text", None) or ""
+        self.last_finish = await self._call(
+            "on_finish",
+            "finish",
+            request=payload.get("task") or "",
+            messages=messages,
+            final=final,
+            tool_calls=tool_calls,
+        )
+        self._dispatch(payload, self.last_decision)
+
+    def consume_follow_up(self) -> Optional[str]:
+        """Return one revision prompt for a violated rule or opt-in retry."""
+        if not (self.config.honour_follow_up or self.config.rule.strip()):
+            return None
+        decision = self.last_decision
+        if decision is None or decision.action != "retry":
+            return None
+        if not self.config.honour_follow_up and (
+            decision.finding is None or decision.finding.question != "rule_satisfied"
+        ):
+            return None
+        if self.follow_ups >= MAX_HONOURED_FOLLOW_UPS:
+            return None
+        self.follow_ups += 1
+        if decision.finding is not None and decision.finding.question == "rule_satisfied":
+            return bound_text(
+                f"Revise the completed work to satisfy this user rule: {self.config.rule}. "
+                f"Jev found: {decision.reason}",
+                self.config.request_max_chars,
+            )
+        return critic_message(decision)
+
+    def _dispatch(
+        self,
+        payload: dict[str, Any],
+        decision: Optional[PolicyDecision],
+        *,
+        queued: Optional[str] = None,
+    ) -> Optional[str]:
+        effect = dispatch(decision or PolicyDecision())
+        if effect.note:
+            self._apply_nudge(payload, effect.note)
+            return effect.note
+        if queued:
+            self._apply_nudge(payload, queued)
+            return queued
+        self._clear_nudge(payload)
+        return None
+
+    def _message_lists(self, payload: dict[str, Any]) -> list[list[Any]]:
+        result = payload.get("result")
+        seen: set[int] = set()
+        lists: list[list[Any]] = []
+        for target in (payload.get("messages"), getattr(result, "messages", None)):
+            if not isinstance(target, list) or id(target) in seen:
+                continue
+            seen.add(id(target))
+            lists.append(target)
+        return lists
+
+    def _apply_nudge(self, payload: dict[str, Any], note: str) -> None:
+        """Write the note onto live messages and ``HarnessResult.messages``."""
+        for target in self._message_lists(payload):
+            target[:] = [item for item in target if not is_jev_nudge(item)]
+            target.append(Message(role="user", content=note))
+
+    def _clear_nudge(self, payload: dict[str, Any]) -> None:
+        for target in self._message_lists(payload):
+            target[:] = [item for item in target if not is_jev_nudge(item)]
+
+
+def jev_from_config(
+    config: EvaluationConfig,
+    *,
+    evaluator: Optional[Evaluator] = None,
+) -> Optional[JevAddon]:
+    """Mount the add-on only when critic mode is enabled."""
+    if not config.enabled:
+        return None
+    return JevAddon(
+        config,
+        evaluator=evaluator,
+    )
+
+
+__all__ = ["JevAddon", "jev_from_config"]

coding_agent/addons/evaluation/evaluators.py

@@ -0,0 +1,245 @@
+"""Evaluator implementations. Vercel helpers are reused, not forked."""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+from typing import Any, Optional
+
+import httpx
+
+from core_ai.providers.catalog import find_provider, provider_api_key
+from core_ai.providers.vercel_evaluation import (
+    evaluation_model_url,
+    evaluation_request_body,
+    evaluation_request_headers,
+    is_evaluation_model,
+)
+from core_ai.types import Message
+
+from coding_agent.config import EvaluationConfig
+from coding_agent.addons.evaluation.protocol import (
+    EvaluationDecision,
+    EvaluationPhase,
+    EvaluationResult,
+    FINISH_QUESTIONS,
+    START_QUESTIONS,
+    error_result,
+    findings_from_decisions,
+    skipped_result,
+)
+from coding_agent.addons.evaluation.state import RunState, clip_text
+
+logger = logging.getLogger(__name__)
+
+RATIONALE_MAX_CHARS = 280
+
+
+def vercel_credentials(config: EvaluationConfig) -> tuple[str, str]:
+    """Return ``(api_key, base_url)`` for the configured evaluation provider."""
+    spec = find_provider(config.provider)
+    api_key = provider_api_key(spec) if spec is not None else ""
+    env_base = (os.environ.get("AI_GATEWAY_BASE_URL") or "").strip()
+    default_base = (spec.default_base_url or "").strip() if spec is not None else ""
+    return (api_key or "", env_base or default_base)
+
+
+def decision_from_answer(name: str, answer: object) -> EvaluationDecision:
+    if not isinstance(answer, dict):
+        return EvaluationDecision(
+            name=name,
+            kind="error",
+            label="invalid",
+            rationale="answer was not an object",
+        )
+    kind = str(answer.get("type") or "").strip() or "error"
+    rationale = clip_text(
+        str(answer.get("rationale") or answer.get("reason") or answer.get("explanation") or ""),
+        RATIONALE_MAX_CHARS,
+    )
+    if kind == "boolean":
+        probability = answer.get("probability")
+        probs: Optional[dict[str, float]] = None
+        if isinstance(probability, (int, float)):
+            true_p = float(probability)
+            probs = {"true": true_p, "false": float(1.0 - true_p)}
+            value = true_p >= 0.5
+        else:
+            value = bool(answer.get("value"))
+        return EvaluationDecision(
+            name=name,
+            kind="boolean",
+            value=value,
+            probs=probs,
+            label="true" if value else "false",
+            rationale=rationale,
+        )
+    if kind == "choice":
+        choice = str(answer.get("choice") or "").strip()
+        return EvaluationDecision(
+            name=name,
+            kind="choice",
+            value=choice,
+            label=choice,
+            rationale=rationale,
+        )
+    if kind == "score":
+        score = answer.get("score")
+        return EvaluationDecision(
+            name=name,
+            kind="score",
+            value=score,
+            label="" if score is None else str(score),
+            rationale=rationale,
+        )
+    return EvaluationDecision(
+        name=name,
+        kind="error",
+        value=answer,
+        label=kind or "invalid",
+        rationale=rationale or "unsupported answer type",
+    )
+
+
+def result_from_answers(phase: EvaluationPhase, answers: object) -> EvaluationResult:
+    if not isinstance(answers, dict) or not answers:
+        return skipped_result(phase, "evaluation model returned no answers")
+    decisions = [decision_from_answer(str(name), answer) for name, answer in answers.items()]
+    return EvaluationResult(
+        phase=phase,
+        status="ok",
+        decisions=decisions,
+        findings=findings_from_decisions(decisions),
+    )
+
+
+class StubEvaluator:
+    """Offline evaluator: records nothing and returns skipped."""
+
+    async def on_start(self, state: RunState) -> EvaluationResult:
+        del state
+        return skipped_result("start", "stub evaluator")
+
+    async def on_finish(self, state: RunState) -> EvaluationResult:
+        del state
+        return skipped_result("finish", "stub evaluator")
+
+
+class MockEvaluator:
+    """Test double that records each start/finish call."""
+
+    def __init__(
+        self,
+        *,
+        start: Optional[EvaluationResult] = None,
+        finish: Optional[EvaluationResult] = None,
+        error: Optional[BaseException] = None,
+    ) -> None:
+        self.start_result = start
+        self.finish_result = finish
+        self.error = error
+        self.starts: list[RunState] = []
+        self.finishes: list[RunState] = []
+
+    async def on_start(self, state: RunState) -> EvaluationResult:
+        self.starts.append(state)
+        if self.error is not None:
+            raise self.error
+        return self.start_result or skipped_result("start", "mock")
+
+    async def on_finish(self, state: RunState) -> EvaluationResult:
+        self.finishes.append(state)
+        if self.error is not None:
+            raise self.error
+        return self.finish_result or skipped_result("finish", "mock")
+
+
+class LLMEvaluator:
+    """Placeholder chat-model critic. Not wired; Jev never becomes the primary model."""
+
+    async def on_start(self, state: RunState) -> EvaluationResult:
+        del state
+        return skipped_result("start", "LLMEvaluator is a stub; use VercelJevEvaluator")
+
+    async def on_finish(self, state: RunState) -> EvaluationResult:
+        del state
+        return skipped_result("finish", "LLMEvaluator is a stub; use VercelJevEvaluator")
+
+
+class VercelJevEvaluator:
+    """POST ``/v4/ai/evaluation-model`` using ``core_ai.providers.vercel_evaluation``."""
+
+    def __init__(
+        self,
+        config: EvaluationConfig,
+        *,
+        api_key: Optional[str] = None,
+        base_url: str = "",
+        transport: Optional[httpx.AsyncBaseTransport] = None,
+    ) -> None:
+        self.config = config
+        resolved_key, resolved_base = vercel_credentials(config)
+        self.api_key = resolved_key if api_key is None else api_key
+        self.base_url = base_url or resolved_base
+        self.transport = transport
+
+    async def on_start(self, state: RunState) -> EvaluationResult:
+        return await self._evaluate("start", state, START_QUESTIONS)
+
+    async def on_finish(self, state: RunState) -> EvaluationResult:
+        questions = dict(FINISH_QUESTIONS)
+        if state.rule:
+            questions["rule_satisfied"] = {
+                "type": "boolean",
+                "instructions": (
+                    "Did the work across this run satisfy the user's rule in state.rule? "
+                    "Judge the observed work and final result. Explain any concrete violation."
+                ),
+            }
+        return await self._evaluate("finish", state, questions)
+
+    async def _evaluate(
+        self,
+        phase: EvaluationPhase,
+        state: RunState,
+        questions: dict[str, Any],
+    ) -> EvaluationResult:
+        if self.config.on_error != "fail-open":
+            return skipped_result(phase, "unsupported on_error policy")
+        if not self.api_key:
+            return skipped_result(phase, "missing Vercel AI Gateway API key")
+        model = self.config.model
+        if not is_evaluation_model(model):
+            return error_result(phase, f"{model} is not an evaluation model")
+        payload = {"state": state.as_eval_state(), "questions": questions}
+        body = evaluation_request_body(
+            [Message(role="user", content=json.dumps(payload))]
+        )
+        try:
+            async with httpx.AsyncClient(transport=self.transport) as client:
+                response = await client.post(
+                    evaluation_model_url(self.base_url),
+                    json=body,
+                    headers=evaluation_request_headers(model, api_key=self.api_key),
+                    timeout=60.0,
+                )
+                response.raise_for_status()
+                data = response.json()
+        except Exception as exc:
+            logger.warning("Jev evaluator %s failed open: %s", phase, exc)
+            return error_result(phase, clip_text(str(exc), RATIONALE_MAX_CHARS))
+        if not isinstance(data, dict):
+            return error_result(phase, "evaluation model returned a non-object response")
+        return result_from_answers(phase, data.get("answers"))
+
+
+__all__ = [
+    "LLMEvaluator",
+    "MockEvaluator",
+    "StubEvaluator",
+    "VercelJevEvaluator",
+    "decision_from_answer",
+    "result_from_answers",
+    "vercel_credentials",
+]

coding_agent/addons/evaluation/handlers.py

@@ -0,0 +1,66 @@
+"""Per-action Jev handlers. Policy chooses ``action``; this module actuates."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Callable, Optional
+
+from coding_agent.addons.evaluation.policy import PolicyDecision, critic_message
+from coding_agent.addons.evaluation.protocol import CONTINUE_NORMALLY
+
+# Read-only allowlist: write/edit/bash (and other implement tools) stay gated.
+READ_ONLY_TOOLS = frozenset({"read_file", "search", "ask_user", "memory"})
+
+
+@dataclass(frozen=True)
+class HandlerResult:
+    note: Optional[str] = None
+    allow: Optional[frozenset[str]] = None
+
+
+def handle_replan(decision: PolicyDecision) -> HandlerResult:
+    return HandlerResult(note=critic_message(decision), allow=READ_ONLY_TOOLS)
+
+
+def handle_retry(decision: PolicyDecision) -> HandlerResult:
+    return HandlerResult(note=critic_message(decision))
+
+
+def handle_review(decision: PolicyDecision) -> HandlerResult:
+    return HandlerResult(note=critic_message(decision))
+
+
+def handle_gather(decision: PolicyDecision) -> HandlerResult:
+    return HandlerResult(note=critic_message(decision), allow=READ_ONLY_TOOLS)
+
+
+def handle_ask(decision: PolicyDecision) -> HandlerResult:
+    return HandlerResult(note=critic_message(decision), allow=READ_ONLY_TOOLS)
+
+
+def handle_finish(decision: PolicyDecision) -> HandlerResult:
+    del decision
+    return HandlerResult()
+
+
+def handle_unsupported_claims(decision: PolicyDecision) -> HandlerResult:
+    return handle_review(decision)
+
+
+HANDLERS: dict[str, Callable[[PolicyDecision], HandlerResult]] = {
+    "replan": handle_replan,
+    "retry": handle_retry,
+    "review": handle_review,
+    "gather": handle_gather,
+    "ask": handle_ask,
+    CONTINUE_NORMALLY: handle_finish,
+    "unsupported_claims": handle_unsupported_claims,
+}
+
+
+def dispatch(decision: PolicyDecision) -> HandlerResult:
+    """Named handler for ``last_decision.action`` (plus unsupported_claims)."""
+    finding = decision.finding
+    if decision.action == "review" and finding is not None and finding.question == "unsupported_claims":
+        return handle_unsupported_claims(decision)
+    return HANDLERS.get(decision.action, handle_finish)(decision)