asm-protocol 0.5.0__py3-none-any.whl

scorer/test_langchain_adapter.py

@@ -0,0 +1,54 @@
+"""Tests for the LangChain adapter (ASMToolSelectorTool). Skipped if langchain-core absent."""
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import pytest
+
+pytest.importorskip("langchain_core")
+
+ROOT = Path(__file__).resolve().parent.parent
+if str(ROOT) not in sys.path:
+    sys.path.insert(0, str(ROOT))
+sys.path.insert(0, str(ROOT / "integrations" / "langchain"))
+
+from asm_tools import ASMToolSelectorTool  # noqa: E402
+
+
+def test_selector_tool_booking_flow():
+    tool = ASMToolSelectorTool()
+    out = tool._run(
+        task="find and book a refundable flight",
+        taxonomy="tool.booking.travel",
+        user_platform="windows",
+        required_functions="flight_search,flight_order_create",
+    )
+    assert "Amadeus" in out
+    assert "risk_class: critical" in out
+    assert "approval_required: True" in out
+
+
+def test_selector_tool_setup_gate():
+    tool = ASMToolSelectorTool()
+    out = tool._run(
+        task="get property data",
+        taxonomy="tool.data.real_estate",
+        required_functions="real_estate_data",
+        require_agent_completable_setup=True,
+    )
+    assert "US Census Bureau Data API" in out
+    assert "setup not agent-completable" in out
+
+
+def test_selector_tool_is_valid_langchain_tool():
+    tool = ASMToolSelectorTool()
+    assert tool.name == "asm_tool_selector"
+    schema = tool.args_schema.model_json_schema()
+    assert "task" in schema["properties"]
+    # invoke via the LangChain interface, not just _run
+    out = tool.invoke({"task": "store a study plan with daily reminders",
+                       "taxonomy": "tool.productivity.task_management",
+                       "user_platform": "windows",
+                       "required_functions": "reminders,recurring_tasks"})
+    assert "Selected tool:" in out

scorer/test_library_select.py

@@ -0,0 +1,104 @@
+"""Tests for the shared tool selector (library_select) and the hosted select API."""
+from __future__ import annotations
+
+import json
+import sys
+import threading
+import urllib.request
+from http.server import ThreadingHTTPServer
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+if str(ROOT) not in sys.path:
+    sys.path.insert(0, str(ROOT))
+
+from library_select import load_library, monthly_cost, select  # noqa: E402
+
+
+def test_library_loads_and_validates_shape():
+    lib = load_library()
+    assert len(lib) >= 30
+    assert all("service_id" in m and "taxonomy" in m for m in lib)
+
+
+def test_local_device_tools_filtered_for_cloud_agent():
+    r = select("store a study plan", taxonomy="tool.productivity.task_management",
+               agent_reach="cloud", user_platform="windows",
+               required_functions=["reminders", "recurring_tasks"])
+    assert r["selected"] is not None
+    rejected_names = {x["service"] for x in r["rejected"]}
+    assert "Things 3" in rejected_names and "Apple Reminders" in rejected_names
+
+
+def test_booking_surfaces_critical_risk_and_approval():
+    r = select("book a flight", taxonomy="tool.booking.travel",
+               agent_reach="cloud", user_platform="windows",
+               required_functions=["flight_search", "flight_order_create"],
+               require_approval_for=["financial_charge"])
+    assert r["selected"] is not None
+    assert r["risk_class"] == "critical"
+    assert r["approval_required"] is True
+    assert "financial_charge" in r["side_effects"]
+
+
+def test_agent_completable_setup_gate():
+    r = select("property data", taxonomy="tool.data.real_estate",
+               agent_reach="cloud", user_platform="windows",
+               required_functions=["real_estate_data"],
+               require_agent_completable_setup=True)
+    assert r["selected"]["display_name"] == "US Census Bureau Data API"
+    reasons = " ".join(x["reason"] for x in r["rejected"])
+    assert "setup not agent-completable" in reasons
+
+
+def test_monthly_cost_free_tier_is_zero():
+    lib = load_library()
+    todoist = next(m for m in lib if m["service_id"].startswith("todoist/"))
+    assert monthly_cost(todoist) == 0.0
+
+
+def test_select_api_endpoints():
+    import asm_select_api as api
+
+    srv = ThreadingHTTPServer(("127.0.0.1", 0), api.Handler)
+    port = srv.server_address[1]
+    threading.Thread(target=srv.serve_forever, daemon=True).start()
+    try:
+        h = json.loads(urllib.request.urlopen(f"http://127.0.0.1:{port}/healthz").read())
+        assert h["ok"] and h["tools"] >= 30
+
+        tools = json.loads(urllib.request.urlopen(
+            f"http://127.0.0.1:{port}/tools?taxonomy=tool.data.real_estate").read())
+        assert len(tools) == 4
+
+        body = json.dumps({"task": "book a flight", "taxonomy": "tool.booking.travel",
+                           "user_platform": "windows",
+                           "required_functions": ["flight_search", "flight_order_create"]}).encode()
+        req = urllib.request.Request(f"http://127.0.0.1:{port}/select", data=body,
+                                     headers={"Content-Type": "application/json"})
+        r = json.loads(urllib.request.urlopen(req).read())
+        assert r["selected"] is not None and r["risk_class"] == "critical"
+
+        bad = urllib.request.Request(f"http://127.0.0.1:{port}/select", data=b"{}",
+                                     headers={"Content-Type": "application/json"})
+        try:
+            urllib.request.urlopen(bad)
+            assert False, "expected 400"
+        except urllib.error.HTTPError as e:
+            assert e.code == 400
+
+        # .well-known/asm catalog: one generated_at, per-manifest links
+        cat = json.loads(urllib.request.urlopen(
+            f"http://127.0.0.1:{port}/.well-known/asm").read())
+        assert cat["count"] >= 30 and cat["generated_at"]
+        assert all("service_id" in e and e["url"].startswith("/manifest/")
+                   for e in cat["manifests"])
+
+        # follow a catalog link to a full manifest
+        first = cat["manifests"][0]
+        man = json.loads(urllib.request.urlopen(
+            f"http://127.0.0.1:{port}{first['url']}").read())
+        assert man["service_id"] == first["service_id"]
+        assert man.get("asm_version") == "0.3"
+    finally:
+        srv.shutdown()

scorer/test_manifests_schema.py

@@ -0,0 +1,55 @@
+"""Schema validation: every manifest under manifests/ and library/ must validate against
+schema/asm-v0.3.schema.json. This test catches the class of regression where
+new manifests use enum values not in the schema (e.g. an invalid
+verification_status string).
+
+Run:
+    pip install jsonschema pytest
+    python -m pytest scorer/test_manifests_schema.py -v
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+try:
+    import jsonschema
+except ImportError:  # pragma: no cover
+    jsonschema = None
+
+ROOT = Path(__file__).resolve().parent.parent
+SCHEMA_PATH = ROOT / "schema" / "asm-v0.3.schema.json"
+MANIFESTS_DIR = ROOT / "manifests"
+LIBRARY_DIR = ROOT / "library"
+
+
+@pytest.fixture(scope="module")
+def schema():
+    if jsonschema is None:
+        pytest.skip("jsonschema not installed; pip install jsonschema")
+    return json.loads(SCHEMA_PATH.read_text(encoding="utf-8"))
+
+
+def manifest_paths():
+    return sorted(MANIFESTS_DIR.glob("*.asm.json")) + sorted(LIBRARY_DIR.rglob("*.asm.json"))
+
+
+def test_at_least_one_manifest():
+    paths = manifest_paths()
+    assert len(paths) > 0, "no manifests found under manifests/"
+
+
+@pytest.mark.parametrize("manifest_path", manifest_paths(), ids=lambda p: p.name)
+def test_manifest_validates(manifest_path: Path, schema):
+    document = json.loads(manifest_path.read_text(encoding="utf-8"))
+    jsonschema.validate(document, schema)
+
+
+def test_no_extra_unknown_versions():
+    """Sanity: every manifest declares asm_version 0.3."""
+    for path in manifest_paths():
+        d = json.loads(path.read_text(encoding="utf-8"))
+        assert d.get("asm_version") == "0.3", f"{path.name} declares asm_version={d.get('asm_version')}"

scorer/test_mcp_server_json_asm.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+"""Tests for ASM metadata embedded in MCP Registry server.json files."""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+if str(ROOT) not in sys.path:
+    sys.path.insert(0, str(ROOT))
+
+from mcp_server_json_asm import PUBLISHER_META_KEY, inspect_server_json, validate_manifest
+
+
+EXAMPLES = ROOT / "examples" / "mcp-server-json"
+
+
+def test_valid_server_json_with_inline_asm_passes():
+    result = inspect_server_json(EXAMPLES / "remote-with-asm.server.json")
+
+    assert result.has_asm is True
+    assert result.valid_asm is True
+    assert result.can_convert is True
+    assert result.asm_service_id == "example/remote-search@1.0"
+    assert result.asm_taxonomy == "tool.data.search"
+    assert result.missing_recommended_fields == []
+
+
+def test_missing_asm_block_is_warning_not_error(tmp_path):
+    server_json = tmp_path / "server.json"
+    server_json.write_text(
+        json.dumps({
+            "name": "io.asm.example/no-asm",
+            "description": "No ASM metadata yet",
+            "_meta": {
+                PUBLISHER_META_KEY: {
+                    "maintainer": "example"
+                }
+            },
+        }),
+        encoding="utf-8",
+    )
+
+    result = inspect_server_json(server_json)
+
+    assert result.has_asm is False
+    assert result.valid_asm is False
+    assert result.errors == []
+    assert result.warnings
+
+
+def test_malformed_asm_reports_schema_errors(tmp_path):
+    server_json = tmp_path / "server.json"
+    server_json.write_text(
+        json.dumps({
+            "name": "io.asm.example/bad-asm",
+            "_meta": {
+                PUBLISHER_META_KEY: {
+                    "asm": {
+                        "asm_version": "0.3",
+                        "service_id": "example/bad@1.0"
+                    }
+                }
+            },
+        }),
+        encoding="utf-8",
+    )
+
+    result = inspect_server_json(server_json)
+
+    assert result.has_asm is True
+    assert result.valid_asm is False
+    assert result.can_convert is False
+    assert any("taxonomy" in error for error in result.errors)
+
+
+def test_extracted_manifest_validates_against_schema():
+    result = inspect_server_json(EXAMPLES / "package-with-asm.server.json")
+
+    assert result.asm_manifest is not None
+    assert validate_manifest(result.asm_manifest) == []
+    assert result.asm_manifest["service_id"] == "example/vector-db@1.0"

scorer/test_openrouter_adapter.py

@@ -0,0 +1,238 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+from jsonschema import Draft202012Validator
+
+from asm_cli import main as asm_main
+from openrouter_adapter import (
+    load_openrouter_manifests,
+    openrouter_models_to_manifests,
+)
+from scorer import parse_manifest
+from scorer import Preferences, ServiceVector, score_topsis
+
+
+SAMPLE_MODELS = {
+    "data": [
+        {
+            "id": "openai/gpt-test",
+            "name": "OpenAI: GPT Test",
+            "context_length": 128000,
+            "pricing": {"prompt": "0.00000015", "completion": "0.0000006"},
+            "top_provider": {"context_length": 128000, "max_completion_tokens": 16000},
+            "architecture": {
+                "input_modalities": ["text", "image"],
+                "output_modalities": ["text"],
+            },
+        },
+        {
+            "id": "cheap/free-model:free",
+            "name": "Cheap: Free Model",
+            "context_length": 8192,
+            "pricing": {"prompt": "0", "completion": "0"},
+            "architecture": {
+                "input_modalities": ["text"],
+                "output_modalities": ["text"],
+            },
+        },
+        {
+            "id": "router/special",
+            "name": "Router: Special",
+            "pricing": {"prompt": "-1", "completion": "-1"},
+            "architecture": {
+                "input_modalities": ["text"],
+                "output_modalities": ["text"],
+            },
+        },
+    ]
+}
+
+
+SAMPLE_RANKINGS = {
+    "generated_at": "2026-05-04T00:00:00Z",
+    "source": "fixture",
+    "n_models": 2,
+    "models": [
+        {
+            "permaslug": "openai/gpt-test-20260501",
+            "slug": "openai/gpt-test",
+            "author": "openai",
+            "prompt_tokens": 1000,
+            "completion_tokens": 500,
+            "total_tokens": 1500,
+            "count": 10,
+            "rank_by_prompt_tokens": 1,
+        },
+        {
+            "permaslug": "other/model-20260501",
+            "slug": "other/model",
+            "author": "other",
+            "prompt_tokens": 100,
+            "completion_tokens": 50,
+            "total_tokens": 150,
+            "count": 2,
+            "rank_by_prompt_tokens": 2,
+        },
+    ],
+}
+
+
+def test_openrouter_models_to_ephemeral_manifests():
+    manifests = openrouter_models_to_manifests(
+        SAMPLE_MODELS["data"],
+        models_source="https://openrouter.ai/api/v1/models",
+        retrieved_at="2026-05-10T00:00:00Z",
+        ranking_by_slug={"openai/gpt-test": SAMPLE_RANKINGS["models"][0]},
+        ranking_count=2,
+        ranking_generated_at=SAMPLE_RANKINGS["generated_at"],
+    )
+
+    assert len(manifests) == 2
+    first = manifests[0]
+    schema = json.loads((Path(__file__).resolve().parent.parent / "schema" / "asm-v0.3.schema.json").read_text(encoding="utf-8"))
+    validator = Draft202012Validator(schema)
+    for manifest in manifests:
+        assert list(validator.iter_errors(manifest)) == []
+
+    assert first["service_id"] == "openrouter/openai/gpt-test@current"
+    assert first["taxonomy"] == "ai.llm.chat"
+    assert first["ttl"] == 300
+    assert first["provenance"]["verification_status"] == "self_reported"
+    assert "usage signal" in first["provenance"]["notes"]
+    assert first["pricing"]["billing_dimensions"][0]["cost_per_unit"] == pytest.approx(0.15)
+    assert first["pricing"]["billing_dimensions"][1]["cost_per_unit"] == pytest.approx(0.6)
+    assert first["quality"]["metrics"][0]["name"] == "openrouter_usage_signal"
+    assert first["quality"]["metrics"][0]["score"] == pytest.approx(1.0)
+
+    vector = parse_manifest(first, io_ratio=0.5)
+    assert vector.cost_per_unit == pytest.approx(0.375 / 1_000_000)
+    assert vector.quality_score == pytest.approx(1.0)
+
+
+def test_load_openrouter_manifests_from_cached_json(tmp_path: Path):
+    models_path = tmp_path / "models.json"
+    rankings_path = tmp_path / "rankings.json"
+    models_path.write_text(json.dumps(SAMPLE_MODELS), encoding="utf-8")
+    rankings_path.write_text(json.dumps(SAMPLE_RANKINGS), encoding="utf-8")
+
+    manifests, metadata = load_openrouter_manifests(
+        models_json=models_path,
+        rankings_json=rankings_path,
+    )
+
+    assert metadata["n_models"] == 3
+    assert metadata["n_manifests"] == 2
+    assert metadata["ranking_snapshot"] == "2026-05-04T00:00:00Z"
+    assert manifests[1]["quality"]["metrics"][0]["score"] == pytest.approx(0.5)
+
+
+def test_cli_openrouter_source_with_cached_json(tmp_path: Path, capsys):
+    models_path = tmp_path / "models.json"
+    rankings_path = tmp_path / "rankings.json"
+    models_path.write_text(json.dumps(SAMPLE_MODELS), encoding="utf-8")
+    rankings_path.write_text(json.dumps(SAMPLE_RANKINGS), encoding="utf-8")
+
+    code = asm_main([
+        "score",
+        "--source",
+        "openrouter",
+        "--openrouter-models-json",
+        str(models_path),
+        "--openrouter-rankings-json",
+        str(rankings_path),
+        "cheap LLM under $1 per 1M tokens under 1s",
+    ])
+    output = capsys.readouterr().out
+
+    assert code == 0
+    assert "OpenRouter ephemeral manifests" in output
+    assert "ignored latency hard constraint" in output
+    assert "representative cost <= $1.0000/1M blended tokens" in output
+    assert "Selected:" in output
+
+
+def test_cli_openrouter_subcommand_with_cached_json(tmp_path: Path, capsys):
+    models_path = tmp_path / "models.json"
+    rankings_path = tmp_path / "rankings.json"
+    models_path.write_text(json.dumps(SAMPLE_MODELS), encoding="utf-8")
+    rankings_path.write_text(json.dumps(SAMPLE_RANKINGS), encoding="utf-8")
+
+    code = asm_main([
+        "openrouter",
+        "--openrouter-models-json",
+        str(models_path),
+        "--openrouter-rankings-json",
+        str(rankings_path),
+        "cheap coding model under $1 per 1M tokens",
+    ])
+    output = capsys.readouterr().out
+
+    assert code == 0
+    assert "OpenRouter ephemeral manifests" in output
+    assert "Model: cheap/free-model:free" in output
+    assert "Selected:" in output
+
+
+def test_cli_openrouter_json_format(tmp_path: Path, capsys):
+    models_path = tmp_path / "models.json"
+    rankings_path = tmp_path / "rankings.json"
+    models_path.write_text(json.dumps(SAMPLE_MODELS), encoding="utf-8")
+    rankings_path.write_text(json.dumps(SAMPLE_RANKINGS), encoding="utf-8")
+
+    code = asm_main([
+        "openrouter",
+        "--format",
+        "json",
+        "--openrouter-models-json",
+        str(models_path),
+        "--openrouter-rankings-json",
+        str(rankings_path),
+        "cheap coding model under $1 per 1M tokens",
+    ])
+    payload = json.loads(capsys.readouterr().out)
+
+    assert code == 0
+    assert payload["selected"]["model"] == "cheap/free-model:free"
+    assert payload["ranked"][0]["cost_per_1m_blended_tokens"] == 0
+    assert payload["source"]["n_manifests"] == 2
+
+
+def test_cli_openrouter_route_litellm_format(tmp_path: Path, capsys):
+    models_path = tmp_path / "models.json"
+    rankings_path = tmp_path / "rankings.json"
+    models_path.write_text(json.dumps(SAMPLE_MODELS), encoding="utf-8")
+    rankings_path.write_text(json.dumps(SAMPLE_RANKINGS), encoding="utf-8")
+
+    code = asm_main([
+        "openrouter",
+        "route",
+        "--format",
+        "litellm",
+        "--openrouter-models-json",
+        str(models_path),
+        "--openrouter-rankings-json",
+        str(rankings_path),
+        "cheap coding model under $1 per 1M tokens",
+    ])
+    output = capsys.readouterr().out
+
+    assert code == 0
+    assert "model_list:" in output
+    assert "model: openrouter/cheap/free-model:free" in output
+    assert "asm_score:" in output
+
+
+def test_topsis_handles_unknown_latency_without_nan():
+    services = [
+        ServiceVector("a", "A", "ai.llm.chat", 1.0, 0.5, float("inf"), 0.5),
+        ServiceVector("b", "B", "ai.llm.chat", 2.0, 0.8, float("inf"), 0.5),
+    ]
+    results = score_topsis(services, Preferences(cost=0.4, quality=0.3, speed=0.2, reliability=0.1))
+
+    assert len(results) == 2
+    for result in results:
+        assert result.total_score == result.total_score
+        assert result.breakdown["speed"] == 1.0