agent-loss-map 0.18.0__py3-none-any.whl

agent_loss_map/__init__.py

@@ -0,0 +1,47 @@
+"""UACP interop conformance harness.
+
+Purpose: find where a UACP agent definition and a framework-native agent
+definition actually diverge, by computing the divergences rather than asserting
+them, and by machine-checking UACP's own JSON Schemas against its normative text.
+
+Provenance discipline: every foreign-format entry in `corpus` records where it
+came from and how confident we are. Nothing here is presented as a wire format
+unless a real serialization was observed.
+"""
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+
+def _detect_version() -> str:
+    """Single source of truth: the installed distribution's version.
+
+    This used to be a hardcoded string here alongside the one in pyproject.toml,
+    and the two drifted: 0.2.0 was published while the package reported itself as
+    0.1.0, and nothing caught it because the release gate only compares the git
+    tag against pyproject.toml. An installed package is the normal case, so ask
+    the installed metadata; a bare source checkout falls back to pyproject.
+    """
+    try:
+        from importlib.metadata import PackageNotFoundError, version
+        try:
+            return version("agent-loss-map")
+        except PackageNotFoundError:
+            pass
+    except ImportError:  # pragma: no cover - importlib.metadata is stdlib >=3.8
+        pass
+
+    for candidate in (Path(__file__).resolve().parent.parent / "pyproject.toml",
+                      Path(__file__).resolve().parent / "pyproject.toml"):
+        try:
+            text = candidate.read_text(encoding="utf-8")
+        except OSError:
+            continue
+        match = re.search(r'^version\s*=\s*"([^"]+)"', text, re.MULTILINE)
+        if match:
+            return match.group(1)
+    return "0.0.0+unknown"
+
+
+__version__ = _detect_version()

agent_loss_map/adapters.py

@@ -0,0 +1,272 @@
+"""Adapters: framework-native agent description -> UACP agent descriptor.
+
+Each adapter returns (descriptor, divergences). Divergences are *computed* from
+the structure of the source, and the produced descriptor is then validated
+against UACP's real JSON Schema, so a SCHEMA_VIOLATION is an observed fact
+rather than a prediction.
+"""
+from __future__ import annotations
+
+from typing import Any
+
+from .schemas import load_schema, validator
+from .types import Divergence, ForeignAgent
+
+# Keywords permitted in an OpenAI function-calling `parameters` schema.
+# Anything outside this set has no representation in that format.
+OPENAI_PARAM_SUBSET = {
+    "type", "properties", "required", "additionalProperties",
+    "description", "enum", "default", "items",
+}
+OPENAI_DISALLOWED = {
+    "oneOf", "anyOf", "allOf", "not", "$ref", "$defs", "definitions",
+    "if", "then", "else", "patternProperties", "dependentSchemas",
+    "dependentRequired", "propertyNames", "const", "pattern", "format",
+    "minimum", "maximum", "minLength", "maxLength", "multipleOf",
+    "minItems", "maxItems", "uniqueItems", "contentEncoding", "prefixItems",
+}
+
+# Deliberately absent: a hand-copied copy of the id and capability-name
+# patterns from the JSON Schemas. Predicting a SCHEMA_VIOLATION from a regex
+# here duplicated what validate_descriptor() already observes from the real
+# validator, so every such finding was counted twice in the totals -- once
+# predicted, once observed. A copied pattern is also a second source of truth
+# that can drift from the schema it was copied from, and CI only pins the
+# schemas, not this file. The schemas are the authority. See
+# validate_descriptor(), which is the sole source of SCHEMA_VIOLATION.
+UACP_NAMESPACES = ("csv", "llm", "web", "deploy", "data")
+
+
+def collect_disallowed(node: Any, path: str = "$", found: dict[str, str] | None = None) -> dict[str, str]:
+    """Walk a JSON Schema and record every keyword OpenAI's subset rejects."""
+    if found is None:
+        found = {}
+    if isinstance(node, dict):
+        for key, value in node.items():
+            here = f"{path}.{key}"
+            if key in OPENAI_DISALLOWED:
+                found.setdefault(key, here)
+            collect_disallowed(value, here, found)
+    elif isinstance(node, list):
+        for i, value in enumerate(node):
+            collect_disallowed(value, f"{path}[{i}]", found)
+    return found
+
+
+def to_uacp_descriptor(agent: ForeignAgent) -> tuple[dict, list[Divergence]]:
+    divs: list[Divergence] = []
+    capabilities: list[dict] = []
+
+    if not agent.has_machine_readable_capabilities:
+        divs.append(Divergence(
+            "UNVERIFIABLE_CLAIM", "blocker", "capabilities",
+            "This framework exposes no machine-readable capability list. A UACP "
+            "registry consumer asking 'who can do csv.analyze' can only "
+            "substring-match free text, so a capability answer derived from this "
+            "agent is a guess, not a verifiable claim.",
+            f"UACP SPEC.md section 5 (Registry / Discovery) depends on "
+            f"{agent.framework} exposing structured capabilities; "
+            f"source shape: {agent.source_shape}",
+        ))
+
+    for tool in agent.capabilities:
+        name = tool.get("name", "")
+        params = tool.get("parameters") or tool.get("inputSchema")
+        out = tool.get("outputSchema")
+
+        # 1. output schema: UACP requires it on every capability
+        if out is None:
+            divs.append(Divergence(
+                "LOSSY", "major", f"capabilities[{name}].outputSchema",
+                "UACP requires outputSchema on every capability. The source "
+                "declares no output contract, so the adapter must fabricate a "
+                "permissive one. Consumers will believe a guarantee the source "
+                "never made.",
+                "UACP capability.schema.json requires "
+                "['name','description','inputSchema','outputSchema']; "
+                f"{agent.framework} exposes no output schema for this tool",
+            ))
+
+        # 2. OpenAI 'strict' has no JSON Schema equivalent
+        if "strict" in tool:
+            divs.append(Divergence(
+                "LOSSY", "minor", f"capabilities[{name}].strict",
+                f"The source sets strict={tool['strict']!r}. JSON Schema has no "
+                "equivalent of the OpenAI 'strict' flag, so the guarantee is "
+                "dropped in either direction and cannot be re-expressed as a "
+                "UACP consumer check.",
+                "OpenAI function-calling tool shape vs JSON Schema (draft 2020-12); "
+                "no 'strict' keyword exists in JSON Schema",
+            ))
+
+        # 3. dot-notation namespace convention
+        if "." not in name:
+            divs.append(Divergence(
+                "LOSSY", "minor", f"capabilities[{name}].name",
+                f"Capability {name!r} carries no namespace prefix, so the "
+                "adapter must rename it. Registry queries scoped to a "
+                "namespace (for example 'data.*') will not match the original "
+                "name.",
+                f"UACP SPEC.md section 2 requires dot-notation namespaces "
+                f"({', '.join(UACP_NAMESPACES)}.*, custom allowed); "
+                f"source name {name!r} is unnamespaced",
+            ))
+
+        capabilities.append({
+            "name": name,
+            "description": tool.get("description", ""),
+            "inputSchema": params if params is not None else {"type": "object"},
+            "outputSchema": out if out is not None else {"type": "object"},
+        })
+
+    # 4. statefulness has no UACP representation
+    if agent.is_stateful:
+        divs.append(Divergence(
+            "SEMANTIC_MISMATCH", "major", "lifecycle",
+            "The source agent is stateful: run() mutates internal history and is "
+            "documented as being called with new messages rather than complete "
+            "history. A UACP descriptor has no lifecycle or state field, and the "
+            "UACP envelope is stateless message passing, so two calls that look "
+            "identical on the wire mean different things on either side.",
+            f"{agent.framework} documents run() as stateful "
+            "(microsoft.github.io/autogen AgentChat agents tutorial); "
+            "UACP SPEC.md section 1 agent descriptor has no state field",
+        ))
+
+    # 5. token streaming vs result streaming
+    if agent.streams_tokens:
+        divs.append(Divergence(
+            "SEMANTIC_MISMATCH", "major", "streaming",
+            "The source streams per-token chunks (ModelClientStreamingChunkEvent) "
+            "that are part of the agent's own message trace. UACP stream messages "
+            "are partial *results* correlated to a request via replyTo and "
+            "sequence numbers. Mapping one onto the other conflates 'the agent's "
+            "reasoning trace' with 'the result stream', so a UACP consumer would "
+            "receive internal reasoning it never asked for and could not tell the "
+            "two apart.",
+            f"{agent.framework} emits per-token chunks in its output stream; "
+            "UACP SPEC.md section 13 defines stream/stream.end as correlated "
+            "partial results",
+        ))
+
+    # 6. multimodal content parts
+    if agent.supports_multimodal:
+        divs.append(Divergence(
+            "UNREPRESENTABLE", "major", "payload",
+            "The source accepts heterogeneous content parts (for example "
+            "MultiModalMessage(content=[str, Image])). The UACP envelope's payload "
+            "is untyped, so a list would validate, but the spec defines no "
+            "modality, media type or part ordering, and an in-memory Image object "
+            "has no wire form. Nothing in UACP distinguishes this from an "
+            "ordinary array payload.",
+            f"{agent.framework} MultiModalMessage; UACP SPEC.md section 3 "
+            "envelope fields define no content-part or modality construct",
+        ))
+
+    # The agent id pattern is deliberately *not* checked here. validate_descriptor()
+    # runs the real validator over the descriptor built below and reports whatever
+    # it rejects, which is an observation rather than a prediction. Predicting here
+    # as well double-counted every such finding in the report totals.
+
+    descriptor = {
+        "id": agent.agent_id,
+        "name": agent.agent_id,
+        "version": "0.0.0-probe",
+        "description": agent.description,
+        "author": "agent-loss-map-probe",
+        "capabilities": capabilities,
+        "transports": ["stdio"],
+        "metadata": {
+            "framework": agent.framework,
+            "probeConfidence": agent.confidence,
+        },
+    }
+    return descriptor, divs
+
+
+def validate_descriptor(descriptor: dict) -> list[Divergence]:
+    """Validate against UACP's real schema. Failures are observed, not predicted."""
+    divs: list[Divergence] = []
+    schema = load_schema("agent-descriptor")
+    for err in sorted(validator(schema).iter_errors(descriptor), key=lambda e: e.path):
+        loc = "$" + "".join(f"[{p!r}]" for p in err.absolute_path)
+        divs.append(Divergence(
+            "SCHEMA_VIOLATION", "blocker", loc,
+            f"UACP agent-descriptor.schema.json rejected the produced descriptor: "
+            f"{err.message}",
+            "agent_loss_map/schemas/agent-descriptor.schema.json",
+        ))
+    return divs
+
+
+def tool_parameter_completeness(agent: ForeignAgent) -> list[Divergence]:
+    """Report capability schemas that a strict tool-calling client will refuse.
+
+    An OpenAI-compatible `function` block is `parameters: JSONSchema`, and the
+    clients that validate it strictly require `parameters.properties` to be
+    present. A schema of exactly `{"type": "object"}` has no `properties` key, so
+    those clients reject the whole tool list with a 400 before the request runs.
+    This was not reachable from the existing checks: the capability schema
+    itself accepts `{"type": "object"}` perfectly well, so the constraint lives in
+    the *target format's* supported subset rather than in JSON Schema.
+
+    What this check can and cannot claim matters, so:
+
+    - It OBSERVES that the schema is rejected. That is computed against the
+      target format's rule, not predicted from a copied regex.
+    - It CANNOT observe how many arguments the tool actually accepts. A schema
+      that declares no properties is indistinguishable from a tool that genuinely
+      takes none. If the tool does take arguments, they are unreachable, but
+      that needs the server's source to establish and is not asserted here.
+      The detail says so rather than guessing, which is the same rule the corpus
+      provenance tiers follow.
+    """
+    divs: list[Divergence] = []
+    for cap in agent.capabilities:
+        schema = cap.get("inputSchema") or cap.get("parameters")
+        if not isinstance(schema, dict):
+            continue
+        if schema.get("type") != "object":
+            continue
+        if "properties" in schema:
+            continue
+        divs.append(Divergence(
+            "UNREPRESENTABLE", "blocker", f"capabilities[{cap.get('name', '?')}].inputSchema",
+            "This capability declares an object schema with no `properties` key. "
+            "Clients that validate an OpenAI-compatible function block strictly "
+            "require `parameters.properties` to be present and reject the entire "
+            "tool list with a 400 before any request runs, so the capability "
+            "cannot be offered to them at all. Whether the tool actually accepts "
+            "arguments is not decidable from the schema: a tool that genuinely "
+            "takes none publishes the same thing, and in that case the fix is a "
+            "present-but-empty `properties` rather than a different schema.",
+            "OpenAI function-calling `parameters` block requires the `properties` "
+            "key for object-typed schemas; this one has keys "
+            f"{sorted(schema) or 'none'}",
+        ))
+    return divs
+
+
+def uacp_to_openai_probes(agent: ForeignAgent) -> list[Divergence]:
+    """Test the UACP -> foreign direction, where the unrepresentable losses live."""
+    divs: list[Divergence] = []
+    if agent.framework != "uacp-native":
+        return divs
+    for cap in agent.capabilities:
+        for field_name in ("inputSchema", "outputSchema"):
+            schema = cap.get(field_name)
+            if not schema:
+                continue
+            bad = collect_disallowed(schema)
+            for keyword, where in sorted(bad.items()):
+                divs.append(Divergence(
+                    "UNREPRESENTABLE", "blocker", f"{cap['name']}.{field_name}.{keyword}",
+                    f"UACP capability uses JSON Schema keyword {keyword!r} at "
+                    f"{where}, which has no representation in an OpenAI "
+                    "function-calling `parameters` block. The capability cannot be "
+                    "exposed to an OpenAI-style tool consumer without either "
+                    "dropping the constraint or flattening it into prose.",
+                    f"{where} in the UACP capability schema vs the OpenAI "
+                    "function-calling parameter subset",
+                ))
+    return divs

agent_loss_map/cli.py

@@ -0,0 +1,134 @@
+"""CLI: run the spec audit and the cross-format divergence probe, print a report."""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+
+from . import __version__, impl_audit, report, spec_audit
+from .adapters import (
+    load_schema, to_uacp_descriptor, tool_parameter_completeness,
+    uacp_to_openai_probes, validate_descriptor,
+)
+from . import extra_corpus
+from .corpus import CORPUS
+from .types import Divergence, sort_divergences
+
+SEV_MARK = {"blocker": "[BLOCKER]", "major": "[major]  ", "minor": "[minor]  "}
+
+
+def render(divs, title: str) -> None:
+    print()
+    print("=" * 78)
+    print(title)
+    print("=" * 78)
+    if not divs:
+        print("  no divergences found")
+        return
+    for d in sort_divergences(divs):
+        print(f"  {SEV_MARK[d.severity]} {d.code:<18} {d.field}")
+        print(f"      {d.detail}")
+        print(f"      evidence: {d.evidence}")
+        print()
+
+
+def main(argv=None) -> int:
+    ap = argparse.ArgumentParser(prog="agent-loss-map")
+    ap.add_argument("--json", action="store_true", help="emit machine-readable JSON")
+    ap.add_argument("--markdown", action="store_true",
+                    help="emit the full report as Markdown, for an issue or a PR")
+    ap.add_argument("--scorecard", action="store_true",
+                    help="emit the one-glance per-section table only")
+    ap.add_argument("--badge", action="store_true",
+                    help="emit badge JSON for the worst finding against UACP itself")
+    ap.add_argument("--corpus", action="append", default=[], metavar="PATH",
+                    help="extra corpus entry file, or a directory of them; "
+                         "repeatable. Lets a format be measured without editing "
+                         "the package.")
+    ap.add_argument("--fail-on-blocker", action="store_true",
+                    help="exit 1 if any blocker is found")
+    args = ap.parse_args(argv)
+
+    try:
+        active_corpus = extra_corpus.merge_with_builtin(
+            extra_corpus.load_extra_corpus(args.corpus)
+        ) if args.corpus else list(CORPUS)
+    except extra_corpus.CorpusError as e:
+        print(f"agent-loss-map: {e}", file=sys.stderr)
+        return 2
+
+    envelope_schema = load_schema("envelope")
+    capability_schema = load_schema("capability")
+    descriptor_schema = load_schema("agent-descriptor")
+
+    spec_divs = spec_audit.run_all(capability_schema, envelope_schema)
+    spec_divs += impl_audit.run_all(envelope_schema)
+
+    per_agent = {}
+    all_divs = []
+    for agent in active_corpus:
+        descriptor, divs = to_uacp_descriptor(agent)
+        # observed, not predicted: validate what we actually produced
+        divs += validate_descriptor(descriptor)
+        divs += uacp_to_openai_probes(agent)
+        divs += tool_parameter_completeness(agent)
+        per_agent[agent.framework] = {
+            "descriptor": descriptor,
+            "divergences": [d.as_dict() for d in sort_divergences(divs)],
+            "provenance": {
+                "confidence": agent.confidence,
+                "source_shape": agent.source_shape,
+                "provenance": agent.provenance,
+            },
+        }
+        all_divs += divs
+
+    if args.badge:
+        print(json.dumps(report.badge(spec_divs, per_agent)))
+    elif args.json:
+        print(json.dumps({
+            "harness": f"agent-loss-map {__version__}",
+            "spec_audit": [d.as_dict() for d in sort_divergences(spec_divs)],
+            "cross_format": {k: {"divergences": v["divergences"],
+                                 "provenance": v["provenance"]}
+                             for k, v in per_agent.items()},
+        }, indent=2))
+    elif args.markdown:
+        print(report.markdown(spec_divs, per_agent, __version__))
+    elif args.scorecard:
+        print(report.scorecard(spec_divs, per_agent, __version__))
+    else:
+        print(report.scorecard(spec_divs, per_agent, __version__))
+        print("Provenance of corpus entries:")
+        for agent in active_corpus:
+            print(f"  {agent.framework:<24} confidence={agent.confidence}")
+        render(spec_divs, "PART 1  UACP schema vs normative text, and schema vs the two reference implementations")
+        for framework, data in per_agent.items():
+            render(
+                [d for d in (Divergence(**x) for x in data["divergences"])],
+                f"PART 2  {framework}  (confidence: {data['provenance']['confidence']})",
+            )
+
+        counts = report.tally(all_divs + spec_divs)
+        print()
+        print("=" * 78)
+        print("TOTALS")
+        print("=" * 78)
+        print(f"  spec/schema conflicts : {len(spec_divs)}")
+        print(f"  cross-format findings : {len(all_divs)}")
+        print(f"  {'-' * 76}")
+        print(f"  all findings          : {len(all_divs) + len(spec_divs)}")
+        for sev in ("blocker", "major", "minor"):
+            print(f"    {sev:<8} {counts.get(sev, 0)}")
+        print()
+        print("Caveat: the AutoGen entry models a documented constructor surface, not an")
+        print("observed serialization. Findings derived from it are weaker evidence than")
+        print("findings derived from the UACP-native entry, which is quoted from SPEC.md.")
+
+    if args.fail_on_blocker and any(d.severity == "blocker" for d in all_divs + spec_divs):
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

agent_loss_map/corpus.py

@@ -0,0 +1,121 @@
+"""Corpus of framework-native agent descriptions.
+
+PROVENANCE RULES (learned the hard way once already):
+  observed-serialization  we read a real serialized artifact
+  documented-api          taken from official documentation of the public API
+  inferred                our modelling choice, not a documented shape
+
+Where a framework has no documented portable serialization of an agent
+definition, we say so in `source_shape` rather than implying one exists. The
+adapter then models the constructor/API surface, and every finding derived from
+such an entry is weaker evidence than one derived from an observed artifact.
+"""
+from __future__ import annotations
+
+from .types import ForeignAgent
+
+# AutoGen AgentChat AssistantAgent.
+# Source: microsoft.github.io/autogen/stable AgentChat "Agents" tutorial.
+# The FunctionTool.schema output below is quoted verbatim from that page.
+# NOTE: this models the documented constructor surface. We did NOT observe an
+# AutoGen agent serialized to a portable artifact, so anything about *wire*
+# portability for AutoGen is unverified.
+AUTOGEN_ASSISTANT = ForeignAgent(
+    framework="autogen-agentchat",
+    agent_id="assistant",
+    description="Use tools to solve tasks.",
+    capabilities=[
+        {
+            # verbatim from the docs' FunctionTool.schema output
+            "name": "web_search_func",
+            "description": "web_search_func",
+            "parameters": {
+                "type": "object",
+                "properties": {"query": {"description": "query", "type": "string"}},
+                "required": ["query"],
+                "additionalProperties": False,
+            },
+            "strict": False,
+        }
+    ],
+    prose_claims=["Use tools to solve tasks."],
+    is_stateful=True,
+    streams_tokens=True,
+    supports_multimodal=True,
+    provenance="Official AutoGen AgentChat agents tutorial, read 2026-09-27",
+    confidence="documented-api",
+    source_shape=(
+        "Constructor surface only: name, description, system_message, tools, "
+        "workbench, model_client. No observed portable serialization of an agent "
+        "definition."
+    ),
+)
+
+# Same tool shape reached through the generic OpenAI function-calling convention.
+# This is the lowest common denominator that most frameworks agree on, so it is
+# the most likely thing a UACP capability has to land on in practice.
+OPENAI_FUNCTION_TOOL = ForeignAgent(
+    framework="openai-function-calling",
+    agent_id="tool-caller",
+    description="Calls OpenAI-style function tools.",
+    capabilities=[
+        {
+            "name": "get_weather",
+            "description": "Get current weather for a city.",
+            "parameters": {
+                "type": "object",
+                "properties": {"city": {"type": "string"}},
+                "required": ["city"],
+                "additionalProperties": False,
+            },
+            "strict": True,
+        }
+    ],
+    prose_claims=["Calls OpenAI-style function tools."],
+    is_stateful=None,
+    provenance="OpenAI function-calling tool shape, as emitted by AutoGen FunctionTool.schema",
+    confidence="documented-api",
+    source_shape="Function/tool declaration only. No agent identity, no lifecycle.",
+)
+
+# A hypothetical-but-realistic case: a UACP capability that uses JSON Schema
+# features outside the OpenAI function-calling subset. Used to test the
+# UACP -> foreign direction, which is where the hard losses live.
+UACP_RICH_CAPABILITY = ForeignAgent(
+    framework="uacp-native",
+    agent_id="csv_analyzer_01",
+    description="Analyzes CSV files and outputs structured JSON schemas.",
+    capabilities=[
+        {
+            "name": "csv.analyze",
+            "description": "Parse a CSV and return column types and statistics.",
+            "inputSchema": {
+                "type": "object",
+                "properties": {
+                    "csv": {"type": "string"},
+                    "mode": {"enum": ["strict", "lenient"]},
+                },
+                "required": ["csv"],
+                "oneOf": [
+                    {"properties": {"mode": {"const": "strict"}}},
+                    {"properties": {"mode": {"const": "lenient"}}},
+                ],
+            },
+            "outputSchema": {
+                "type": "object",
+                "properties": {"rowCount": {"type": "integer"}},
+                "required": ["rowCount"],
+            },
+            "async": True,
+        }
+    ],
+    provenance="UACP SPEC.md section 1 and 2, agent descriptor example",
+    confidence="observed-serialization",
+    source_shape="UACP agent descriptor, quoted from SPEC.md",
+)
+
+CORPUS: list[ForeignAgent] = [
+    AUTOGEN_ASSISTANT,
+    OPENAI_FUNCTION_TOOL,
+    UACP_RICH_CAPABILITY,
+]