premanmcp 1.0.4 → 1.1.0

package/bin/eval_harness.py

@@ -0,0 +1,231 @@
+"""Run assert-ai against Anthropic, working around four upstream gaps.
+
+All four workarounds live here rather than in the installed package so they
+survive `uv sync`, which re-downloads the pinned ASSERT rev and discards any
+edit made under .venv/.
+
+1. Workspace header. Identity-linked Anthropic API keys carry no workspace
+   binding, so the Messages API rejects them unless the request names the
+   workspace it acts in. assert-ai has no header passthrough, so set LiteLLM's
+   global header dict before any stage runs.
+
+2. summary_items validator. assert-ai's systematize prompt was migrated to a
+   JSON contract that no longer emits `summary_items`, but
+   `_validate_summary_items` still rejects an empty list -- while the only
+   downstream consumer, systematization_convert, treats them as optional.
+   Anthropic returns the empty list the prompt implies and the stage dies.
+
+3. extra_body on the multi-turn tester. inference.py hardcodes
+   `extra_kwargs={"extra_body": {"store": True}}` on every tester call --
+   an OpenAI response-retention flag. Anthropic rejects the unknown field with
+   `invalid_request_error: extra_body: Extra inputs are not permitted`, which
+   assert-ai catches as LLMInputError and mislabels `tester_input_refused`, as
+   if the model had refused on content grounds. It had not. Every scenario case
+   dies this way, deterministically, on any Anthropic tester -- silently
+   removing the multi-turn half of every test bank while blaming the model.
+   Only the chat path needs this: the responses path is OpenAI-only.
+
+4. The systematization response schema, which contradicts its own prompt.
+   `systematization_single.md` tells the model to return a behavior spec --
+   `scope`, `concept_spec.patterns` with slot components, `impact_analysis`,
+   `references` -- and says "no additional fields". The code then forces a
+   different schema through structured output: `{systematization: str,
+   summary_items: [...]}`, `extra="forbid"`. Structured output wins, so the
+   model cannot return what it was asked for. It puts something in the one
+   string field it was given and the spec is never produced.
+
+   Observed on three OpenAI configurations: the field came back as `""`, as
+   the literal word `"systematization"`, and as the behavior's own name. Two
+   of those pass `_validate_systematization`, which only checks
+   non-emptiness -- so the pipeline proceeds on a hollow taxonomy and scores
+   an agent against nothing. The gpt-4o run that burned its full 16,000-token
+   budget was the same fault wearing a different hat: the model trying to
+   write the whole document into the single string it had.
+
+   It is a leftover from the pre-migration markdown era, and the same
+   migration workaround 2 exists for. `systematization_convert` reads
+   `data["systematization"]` as text and its documented input contract is
+   exactly the object the prompt describes -- so the value was always meant to
+   be that document, serialised.
+
+All four go away upstream: (1) with a workspace-scoped API key, (2), (3) and
+(4) with a fix in AleWang16/ASSERT plus a bumped rev in pyproject.toml.
+
+This file exists twice. ``preman-mcp/bin/eval_harness.py`` is a byte-identical
+copy, shipped in the npm package so a run leased to a paired device goes through
+the same three workarounds a server-side run does -- otherwise an Anthropic
+tester would silently lose the multi-turn half of every bank on a customer's
+machine and keep it on ours, and the two runs would not be comparable while
+looking like they were. ``tests/test_eval_harness_shim.py`` fails if the copies
+drift, and byte-identical is what makes that check possible: edit this file and
+copy it over, do not maintain two.
+"""
+
+import os
+import sys
+from importlib.metadata import entry_points
+
+
+def _config_mentions_anthropic() -> bool:
+    """Whether this invocation routes any stage to Anthropic.
+
+    Workaround 1 is Anthropic-only, and `litellm.headers` is global -- it would
+    otherwise attach a vendor header to every provider's requests. Reading the
+    config is cheap and keeps a DeepSeek or OpenAI run from being blocked by a
+    requirement that does not apply to it.
+    """
+    argv = sys.argv
+    if "--config" not in argv:
+        return True  # cannot tell; keep the stricter behaviour
+    try:
+        path = argv[argv.index("--config") + 1]
+        return "anthropic/" in open(path, encoding="utf-8").read()
+    except (IndexError, OSError):
+        return True
+
+
+WORKSPACE_ID = os.environ.get("ANTHROPIC_WORKSPACE_ID")
+if _config_mentions_anthropic() and not WORKSPACE_ID:
+    sys.exit("ANTHROPIC_WORKSPACE_ID is not set (expected a wrkspc_... value)")
+
+import litellm
+
+if WORKSPACE_ID and _config_mentions_anthropic():
+    litellm.headers = {"anthropic-workspace-id": WORKSPACE_ID}
+
+from pydantic import BaseModel, ConfigDict
+
+from assert_ai.core import model_client
+from assert_ai.stages import systematization
+
+
+def _validate_summary_items(summary_items) -> None:
+    for item in summary_items:
+        if not item.description.strip():
+            raise ValueError("systematization summary_items.description must be non-empty")
+        if not item.example.strip():
+            raise ValueError("systematization summary_items.example must be non-empty")
+
+
+systematization._validate_summary_items = _validate_summary_items
+
+
+class _KeyTerm(BaseModel):
+    term: str
+    definition: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class _SlotValue(BaseModel):
+    slot_value: str
+    definition: str
+    example_phrase: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class _SlotComponent(BaseModel):
+    component: str
+    slot_values: list[_SlotValue]
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class _Pattern(BaseModel):
+    pattern: str
+    pattern_role: str
+    primary_theory: str
+    related_theory: str
+    key_terms: list[_KeyTerm]
+    slot_components: list[_SlotComponent]
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class _ConceptSpec(BaseModel):
+    behavior: str
+    patterns: list[_Pattern]
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class _StakeholderLens(BaseModel):
+    label: str
+    expertise: str
+
+    model_config = ConfigDict(extra="forbid")
+
+
+class _SystematizationDocument(BaseModel):
+    """The output contract ``systematization_single.md`` actually asks for.
+
+    Stands in for assert-ai's ``SystematizationResponse``, which asks for
+    something else -- see workaround 4. The call site reads
+    ``.model_json_schema()`` to build the request and then validates the reply
+    with the same class, so replacing the class replaces both halves at once
+    and the two cannot drift apart the way the schema and the prompt did.
+
+    ``nested_slot_components`` is the one contract field left out. OpenAI's
+    strict structured output allows no unconstrained objects and no more than
+    five levels of nesting, and this schema already reaches five at
+    ``slot_values``. The convert prompt treats nested slots as conditional --
+    "when a slot component contains nested_slot_components" -- so their absence
+    costs a refinement rather than breaking the stage, which an invalid schema
+    would.
+    """
+
+    behavior: str
+    scope: str
+    impact_analysis: str
+    alternative_systematizations: str
+    references: list[str]
+    stakeholder_lenses: list[_StakeholderLens]
+    reasoning_summary: str
+    concept_spec: _ConceptSpec
+
+    model_config = ConfigDict(extra="forbid")
+
+    @property
+    def systematization(self) -> str:
+        """The document as the text ``systematization_convert`` expects.
+
+        Serialised rather than summarised: the convert prompt's input contract
+        names these fields and reads them out of this string, so anything
+        lossy here is a taxonomy built from less than was researched.
+        """
+        return self.model_dump_json(indent=2)
+
+    @property
+    def summary_items(self) -> list:
+        """Always empty, and legitimately so.
+
+        The migrated prompt does not ask for them and the convert stage treats
+        them as optional. Workaround 2 is what stops the validator upstream of
+        it from rejecting the empty list.
+        """
+        return []
+
+
+systematization.SystematizationResponse = _SystematizationDocument
+
+# Providers that understand `extra_body`. Everything else rejects it outright.
+_EXTRA_BODY_PROVIDERS = ("openai/", "azure/", "azure_ai/")
+_build_chat_payload = model_client._build_chat_payload
+
+
+def _build_chat_payload_without_extra_body(model, messages, options):
+    payload = _build_chat_payload(model, messages, options)
+    if not str(model).startswith(_EXTRA_BODY_PROVIDERS):
+        payload.pop("extra_body", None)
+    return payload
+
+
+# Patched as a module attribute because the call sites resolve it as a global
+# at call time, so rebinding here reaches all of them.
+model_client._build_chat_payload = _build_chat_payload_without_extra_body
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+_cli = next(e for e in entry_points(group="console_scripts") if e.name == "assert-ai")
+sys.exit(_cli.load()())